def doCleanupDeletedFiles(cursor = None): maxIterCount = 2 # How often a file will be checked for deletion gotAtLeastOne = False query = db.Query("viur-deleted-files").cursor( cursor ) for file in query.run(100): gotAtLeastOne = True if not "dlkey" in file.keys(): db.Delete(file.key()) elif db.Query("viur-blob-locks").filter("active_blob_references =", file["dlkey"]).get(): logging.info("is referenced, %s" % file["dlkey"]) db.Delete(file.key()) else: if file["itercount"] > maxIterCount: logging.info("Finally deleting, %s" % file["dlkey"]) blobstore.delete(file["dlkey"]) db.Delete(file.key()) # There should be exactly 1 or 0 of these for f in db.Query("file").filter("dlkey =", file["dlkey"]).iter(keysOnly=True): db.Delete(f) else: logging.error("Increasing count, %s" % file["dlkey"]) file["itercount"] += 1 db.Put(file) newCursor = query.getCursor() if gotAtLeastOne and newCursor and newCursor.urlsafe() != cursor: doCleanupDeletedFiles(newCursor.urlsafe())
def flushCache(prefix="/*"): """ Flushes the cache. Its possible the flush only a part of the cache by specifying the path-prefix. :param prefix: Path or prefix that should be flushed. :type prefix: str Examples: - "/" would flush the main page (and only that), - "/*" everything from the cache, "/page/*" everything from the page-module (default render), - and "/page/view/*" only that specific subset of the page-module. """ items = db.Query(viurCacheName).filter( "path =", prefix.rstrip("*")).iter(keysOnly=True) for item in items: db.Delete(item) if prefix.endswith("*"): items = db.Query(viurCacheName).filter( "path >", prefix.rstrip("*")).filter( "path <", prefix.rstrip("*") + u"\ufffd").iter(keysOnly=True) for item in items: db.Delete(item) logging.debug( "Flushing cache succeeded. Everything matching \"%s\" is gone." % prefix)
def validate(key, acceptSessionKey=False): """ Validates a onetime securitykey :type key: str :param key: The key to validate :type acceptSessionKey: Bool :param acceptSessionKey: If True, we also accept the session's skey :returns: False if the key was not valid for whatever reasons, the data (given during createSecurityKey) as dictionary or True if the dict is empty. """ if acceptSessionKey: if key == currentSession.getSessionSecurityKey(): return (True) try: dbObj = db.Get(db.Key.from_path(securityKeyKindName, key)) except: return (False) if dbObj: if "session" in dbObj and dbObj["session"] is not None: if dbObj["session"] != currentSession.getSessionKey(): return (False) db.Delete(dbObj.key()) if dbObj["until"] < datetime.now(): #This key has expired return (False) res = {} for k in dbObj.keys(): res[k] = dbObj[k] del res["session"] del res["until"] if not res: return (True) return (res) return (False)
def doClearSessions(timeStamp, cursor): gotAtLeastOne = False query = db.Query(GaeSession.kindName).filter("lastseen <", timeStamp) for oldKey in query.run(100, keysOnly=True): gotAtLeastOne = True db.Delete(oldKey) newCursor = query.getCursor() if gotAtLeastOne and newCursor and newCursor.urlsafe() != cursor: doClearSessions(timeStamp, newCursor.urlsafe())
def getOldBlobKeysTxn(dbKey): obj = db.Get(dbKey) res = obj["old_blob_references"] or [] if obj["is_stale"]: db.Delete(dbKey) else: obj["has_old_blob_references"] = False obj["old_blob_references"] = [] db.Put(obj) return res
def doClearSKeys(timeStamp, cursor): gotAtLeastOne = False query = db.Query(securityKeyKindName).filter( "until <", datetime.strptime(timeStamp, "%d.%m.%Y %H:%M:%S")) for oldKey in query.run(100, keysOnly=True): gotAtLeastOne = True db.Delete(oldKey) newCursor = query.getCursor() if gotAtLeastOne and newCursor and newCursor.urlsafe() != cursor: doClearSKeys(timeStamp, newCursor.urlsafe())
def txnDelete(key, skel): dbObj = db.Get(db.Key( key)) # Fetch the raw object as we might have to clear locks for boneName, bone in skel.items(): # Ensure that we delete any value-lock objects remaining for this entry if bone.unique: try: logging.error("x1") logging.error(dbObj.keys()) if "%s.uniqueIndexValue" % boneName in dbObj.keys(): logging.error("x2") db.Delete( db.Key.from_path( "%s_%s_uniquePropertyIndex" % (skel.kindName, boneName), dbObj["%s.uniqueIndexValue" % boneName])) except db.EntityNotFoundError: raise pass # Delete the blob-key lock object try: lockObj = db.Get(db.Key.from_path("viur-blob-locks", str(key))) except: lockObj = None if lockObj is not None: if lockObj["old_blob_references"] is None and lockObj[ "active_blob_references"] is None: db.Delete(lockObj) # Nothing to do here else: if lockObj["old_blob_references"] is None: # No old stale entries, move active_blob_references -> old_blob_references lockObj["old_blob_references"] = lockObj[ "active_blob_references"] elif lockObj["active_blob_references"] is not None: # Append the current references to the list of old & stale references lockObj["old_blob_references"] += lockObj[ "active_blob_references"] lockObj["active_blob_references"] = [ ] # There are no active ones left lockObj["is_stale"] = True lockObj["has_old_blob_references"] = True db.Put(lockObj) db.Delete(db.Key(key))
def index(self, *args, **kwargs): global _callableTasks, _periodicTasks logging.debug("Starting maintenance-run") checkUpdate() #Let the update-module verify the database layout first logging.debug("Updatecheck complete") for task, intervall in _periodicTasks.items( ): #Call all periodic tasks if intervall: #Ensure this task doesn't get called to often try: lastCall = db.Get( db.Key.from_path("viur-task-interval", task.periodicTaskName)) if lastCall["date"] > datetime.now() - timedelta( minutes=intervall): logging.debug( "Skipping task %s - Has already run recently." % task.periodicTaskName) continue except db.EntityNotFoundError: pass res = self.findBoundTask(task) if res: #Its bound, call it this way :) res[0]() else: task() #It seems it wasnt bound - call it as a static method logging.debug("Successfully called task %s" % task.periodicTaskName) if intervall: # Update its last-call timestamp entry = db.Entity("viur-task-interval", name=task.periodicTaskName) entry["date"] = datetime.now() db.Put(entry) logging.debug("Periodic tasks complete") for currentTask in db.Query( "viur-queued-tasks").iter(): #Look for queued tasks db.Delete(currentTask.key()) if currentTask["taskid"] in _callableTasks: task = _callableTasks[currentTask["taskid"]]() tmpDict = {} for k in currentTask.keys(): if k == "taskid": continue tmpDict[k] = json.loads(currentTask[k]) try: task.execute(**tmpDict) except Exception as e: logging.error("Error executing Task") logging.exception(e) logging.debug("Scheduled tasks complete")
def refreshIndex(self, query): """ Refreshes the Index for the given query (Actually it removes it from the db so it gets rebuild on next use) :param query: Query for which the index should be refreshed :type query: db.Query """ key = self.keyFromQuery(query) try: db.Delete(db.Key.from_path(self._dbType, key)) except db.EntityNotFoundError: pass if key in self._cache.keys(): del self._cache[key]
def killSessionByUser(user=None): """ Invalidates all active sessions for the given *user*. This means that this user is instantly logged out. If no user is given, it tries to invalidate **all** active sessions. Use "guest" as to kill all sessions not associated with an user. :param user: UserID, "guest" or None. :type user: str | None """ logging.error("Invalidating all sessions for %s" % user) query = db.Query(GaeSession.kindName) if user is not None: query.filter("user =", str(user)) for key in query.iter(keysOnly=True): db.Delete(key)
def reset(self): """ Invalids the current session and starts a new one. This function is especially useful at login, where we might need to create an SSL-capable session. :warning: Everything (except the current language) is flushed. """ lang = self.session.get("language") if self.key: db.Delete(db.Key.from_path(self.kindName, self.key)) self.key = None self.sslKey = None self.sessionSecurityKey = None self.changed = True self.session = {} if lang: self.session["language"] = lang
def postDeletedHandler(self, skel, key, id): db.Delete([ x for x in db.Query("viur-relations").ancestor(db.Key(id)).run( keysOnly=True) ])
def postSavedHandler(self, valuesCache, boneName, skel, key, dbfields): if not valuesCache[boneName]: values = [] elif isinstance(valuesCache[boneName], dict): values = [dict((k, v) for k, v in valuesCache[boneName].items())] else: values = [ dict((k, v) for k, v in x.items()) for x in valuesCache[boneName] ] parentValues = {} for parentKey in dbfields.keys(): if parentKey in self.parentKeys or any( [parentKey.startswith(x + ".") for x in self.parentKeys]): parentValues[parentKey] = dbfields[parentKey] dbVals = db.Query("viur-relations").ancestor( db.Key(key)) #skel.kindName+"_"+self.kind+"_"+key dbVals.filter("viur_src_kind =", skel.kindName) dbVals.filter("viur_dest_kind =", self.kind) dbVals.filter("viur_src_property =", boneName) for dbObj in dbVals.iter(): try: if not dbObj["dest.key"] in [x["dest"]["key"] for x in values ]: #Relation has been removed db.Delete(dbObj.key()) continue except: #This entry is corrupt db.Delete(dbObj.key()) else: # Relation: Updated data = [ x for x in values if x["dest"]["key"] == dbObj["dest.key"] ][0] if self.indexed: #We dont store more than key and kinds, and these dont change #Write our (updated) values in refSkel = self._refSkelCache refSkel.setValuesCache(data["dest"]) for k, v in refSkel.serialize().items(): dbObj["dest." + k] = v for k, v in parentValues.items(): dbObj["src." + k] = v if self.using is not None: usingSkel = self._usingSkelCache usingSkel.setValuesCache(data["rel"]) for k, v in usingSkel.serialize().items(): dbObj["rel." + k] = v dbObj["viur_delayed_update_tag"] = time() db.Put(dbObj) values.remove(data) # Add any new Relation for val in values: dbObj = db.Entity( "viur-relations", parent=db.Key(key)) #skel.kindName+"_"+self.kind+"_"+key if not self.indexed: #Dont store more than key and kinds, as they aren't used anyway dbObj["dest.key"] = val["dest"]["key"] dbObj["src.key"] = key else: refSkel = self._refSkelCache refSkel.setValuesCache(val["dest"]) for k, v in refSkel.serialize().items(): dbObj["dest." + k] = v for k, v in parentValues.items(): dbObj["src." + k] = v if self.using is not None: usingSkel = self._usingSkelCache usingSkel.setValuesCache(val["rel"]) for k, v in usingSkel.serialize().items(): dbObj["rel." + k] = v dbObj["viur_delayed_update_tag"] = time() dbObj[ "viur_src_kind"] = skel.kindName #The kind of the entry referencing #dbObj[ "viur_src_key" ] = str( key ) #The key of the entry referencing dbObj[ "viur_src_property"] = boneName #The key of the bone referencing #dbObj[ "viur_dest_key" ] = val["key"] dbObj["viur_dest_kind"] = self.kind db.Put(dbObj)
def txnUpdate(key, mergeFrom, clearUpdateTag): blobList = set() skel = type(mergeFrom)() # Load the current values from Datastore or create a new, empty db.Entity if not key: dbObj = db.Entity(skel.kindName) oldBlobLockObj = None else: k = db.Key(key) assert k.kind() == skel.kindName, "Cannot write to invalid kind!" try: dbObj = db.Get(k) except db.EntityNotFoundError: dbObj = db.Entity(k.kind(), id=k.id(), name=k.name(), parent=k.parent()) else: skel.setValues(dbObj) try: oldBlobLockObj = db.Get(db.Key.from_path("viur-blob-locks", str(k))) except: oldBlobLockObj = None # Remember old hashes for bones that must have an unique value oldUniqeValues = {} for boneName, boneInstance in skel.items(): if boneInstance.unique: if "%s.uniqueIndexValue" % boneName in dbObj: oldUniqeValues[boneName] = dbObj["%s.uniqueIndexValue" % boneName] ## Merge the values from mergeFrom in for key, bone in skel.items(): if key in mergeFrom: bone.mergeFrom(skel.valuesCache, key, mergeFrom) for key, _bone in skel.items(): dbObj = _bone.serialize(skel.valuesCache, key, dbObj) blobList.update(_bone.getReferencedBlobs(self.valuesCache, key)) if clearUpdateTag: dbObj["viur_delayed_update_tag"] = 0 # Mark this entity as Up-to-date. else: dbObj[ "viur_delayed_update_tag"] = time() # Mark this entity as dirty, so the background-task will catch it up and update its references. dbObj = skel.preProcessSerializedData(dbObj) try: ourKey = str(dbObj.key()) except: # Its not an update but an insert, no key yet ourKey = None # Lock hashes from bones that must have unique values newUniqeValues = {} for boneName, boneInstance in skel.items(): if boneInstance.unique: # Check if the property is really unique newUniqeValues[boneName] = boneInstance.getUniquePropertyIndexValue( self.valuesCache, boneName) if newUniqeValues[boneName] is not None: try: lockObj = db.Get(db.Key.from_path( "%s_%s_uniquePropertyIndex" % (skel.kindName, boneName), newUniqeValues[boneName])) if lockObj["references"] != ourKey: # This value has been claimed, and that not by us raise ValueError( "The unique value '%s' of bone '%s' has been recently claimed!" % (self.valuesCache[boneName], boneName)) except db.EntityNotFoundError: # No lockObj found for that value, we can use that pass dbObj["%s.uniqueIndexValue" % boneName] = newUniqeValues[boneName] else: if "%s.uniqueIndexValue" % boneName in dbObj: del dbObj["%s.uniqueIndexValue" % boneName] if not skel.searchIndex: # We generate the searchindex using the full skel, not this (maybe incomplete one) tags = [] for key, _bone in skel.items(): if _bone.searchable: tags += [tag for tag in _bone.getSearchTags(self.valuesCache, key) if (tag not in tags and len(tag) < 400)] dbObj["viur_tags"] = tags db.Put(dbObj) # Write the core entry back # Now write the blob-lock object blobList = skel.preProcessBlobLocks(blobList) if blobList is None: raise ValueError( "Did you forget to return the bloblist somewhere inside getReferencedBlobs()?") if None in blobList: raise ValueError("None is not a valid blobKey.") if oldBlobLockObj is not None: oldBlobs = set(oldBlobLockObj["active_blob_references"] if oldBlobLockObj[ "active_blob_references"] is not None else []) removedBlobs = oldBlobs - blobList oldBlobLockObj["active_blob_references"] = list(blobList) if oldBlobLockObj["old_blob_references"] is None: oldBlobLockObj["old_blob_references"] = [x for x in removedBlobs] else: tmp = set(oldBlobLockObj["old_blob_references"] + [x for x in removedBlobs]) oldBlobLockObj["old_blob_references"] = [x for x in (tmp - blobList)] oldBlobLockObj["has_old_blob_references"] = oldBlobLockObj[ "old_blob_references"] is not None and len( oldBlobLockObj["old_blob_references"]) > 0 oldBlobLockObj["is_stale"] = False db.Put(oldBlobLockObj) else: # We need to create a new blob-lock-object blobLockObj = db.Entity("viur-blob-locks", name=str(dbObj.key())) blobLockObj["active_blob_references"] = list(blobList) blobLockObj["old_blob_references"] = [] blobLockObj["has_old_blob_references"] = False blobLockObj["is_stale"] = False db.Put(blobLockObj) for boneName, boneInstance in skel.items(): if boneInstance.unique: # Update/create/delete missing lock-objects if boneName in oldUniqeValues and oldUniqeValues[boneName] != \ newUniqeValues[boneName]: # We had an old lock and its value changed try: # Try to delete the old lock oldLockObj = db.Get(db.Key.from_path( "%s_%s_uniquePropertyIndex" % (skel.kindName, boneName), oldUniqeValues[boneName])) if oldLockObj["references"] != ourKey: # We've been supposed to have that lock - but we don't. # Don't remove that lock as it now belongs to a different entry logging.critical( "Detected Database corruption! A Value-Lock had been reassigned!") else: # It's our lock which we don't need anymore db.Delete(db.Key.from_path( "%s_%s_uniquePropertyIndex" % ( skel.kindName, boneName), oldUniqeValues[boneName])) except db.EntityNotFoundError as e: logging.critical( "Detected Database corruption! Could not delete stale lock-object!") if newUniqeValues[boneName] is not None: # Lock the new value newLockObj = db.Entity( "%s_%s_uniquePropertyIndex" % (skel.kindName, boneName), name=newUniqeValues[boneName]) newLockObj["references"] = str(dbObj.key()) db.Put(newLockObj) return (str(dbObj.key()), dbObj, skel)