def flushCache(prefix="/*"): """ Flushes the cache. Its possible the flush only a part of the cache by specifying the path-prefix. :param prefix: Path or prefix that should be flushed. :type prefix: str Examples: - "/" would flush the main page (and only that), - "/*" everything from the cache, "/page/*" everything from the page-module (default render), - and "/page/view/*" only that specific subset of the page-module. """ items = db.Query(viurCacheName).filter( "path =", prefix.rstrip("*")).iter(keysOnly=True) for item in items: db.Delete(item) if prefix.endswith("*"): items = db.Query(viurCacheName).filter( "path >", prefix.rstrip("*")).filter( "path <", prefix.rstrip("*") + u"\ufffd").iter(keysOnly=True) for item in items: db.Delete(item) logging.debug( "Flushing cache succeeded. Everything matching \"%s\" is gone." % prefix)
def doCleanupDeletedFiles(cursor=None): maxIterCount = 2 # How often a file will be checked for deletion query = db.Query("viur-deleted-files") if cursor: query.setCursor(cursor) for file in query.run(100): if not "dlkey" in file: db.Delete(file.key) elif db.Query("viur-blob-locks").filter("active_blob_references =", file["dlkey"]).getEntry(): logging.info("is referenced, %s" % file["dlkey"]) db.Delete(file.key) else: if file["itercount"] > maxIterCount: logging.info("Finally deleting, %s" % file["dlkey"]) blobs = bucket.list_blobs(prefix="%s/" % file["dlkey"]) for blob in blobs: blob.delete() db.Delete(file.key) # There should be exactly 1 or 0 of these for f in skeletonByKind("file")().all().filter( "dlkey =", file["dlkey"]).fetch(99): f.delete() else: logging.debug("Increasing count, %s" % file["dlkey"]) file["itercount"] += 1 db.Put(file) newCursor = query.getCursor() if newCursor: doCleanupDeletedFiles(newCursor)
def flushCache(prefix: str = None, key: Union[db.Key, None] = None, kind: Union[str, None] = None): """ Flushes the cache. Its possible the flush only a part of the cache by specifying the path-prefix. The path is equal to the url that caused it to be cached (eg /page/view) and must be one listed in the 'url' param of :meth:`viur.core.cache.enableCache`. :param prefix: Path or prefix that should be flushed. :param key: Flush all cache entries which may contain this key. Also flushes entries which executed a query over that kind. :param kind: Flush all cache entries which executed a query over that kind. Examples: - "/" would flush the main page (and only that), - "/*" everything from the cache, "/page/*" everything from the page-module (default render), - and "/page/view/*" only that specific subset of the page-module. """ if prefix is None and key is None and kind is None: prefix = "/*" if prefix is not None: items = db.Query(viurCacheName).filter("path =", prefix.rstrip("*")).iter() for item in items: db.Delete(item) if prefix.endswith("*"): items = db.Query(viurCacheName) \ .filter("path >", prefix.rstrip("*")) \ .filter("path <", prefix.rstrip("*") + u"\ufffd") \ .iter() for item in items: db.Delete(item) logging.debug( "Flushing cache succeeded. Everything matching \"%s\" is gone." % prefix) if key is not None: items = db.Query(viurCacheName).filter("accessedEntries =", key).iter() for item in items: logging.info("Deleted cache entry %s", item["path"]) db.Delete(item.key) if not isinstance(key, db.Key): key = db.Key(encoded=key) items = db.Query(viurCacheName).filter("accessedEntries =", key.kind).iter() for item in items: logging.info("Deleted cache entry %s", item["path"]) db.Delete(item.key) if kind is not None: items = db.Query(viurCacheName).filter("accessedEntries =", kind).iter() for item in items: logging.info("Deleted cache entry %s", item["path"]) db.Delete(item.key)
def doClearSessions(timeStamp, cursor): query = db.Query(GaeSession.kindName).filter("lastseen <", timeStamp) for oldKey in query.run(100, keysOnly=True): db.Delete(oldKey) newCursor = query.getCursor() if newCursor: doClearSessions(timeStamp, newCursor)
def validate(key: str, useSessionKey: bool) -> Union[bool, db.Entity]: """ Validates a onetime securitykey :type key: str :param key: The key to validate :type useSessionKey: Bool :param useSessionKey: If True, we validate against the session's skey, otherwise we'll lookup an unbound key :returns: False if the key was not valid for whatever reasons, the data (given during createSecurityKey) as dictionary or True if the dict is empty. """ if useSessionKey: if key == "staticSessionKey": skeyHeaderValue = request.current.get().request.headers.get( "Sec-X-ViUR-StaticSKey") if skeyHeaderValue and currentSession.validateStaticSecurityKey( skeyHeaderValue): return True elif currentSession.validateSecurityKey(key): return True return False if not key or "/" in key: return False dbObj = db.Get((securityKeyKindName, key)) if dbObj: db.Delete((securityKeyKindName, key)) if dbObj["until"] < datetime.now(): # This key has expired return False del dbObj["until"] if not dbObj: return True return dbObj return False
def validate(key: str, useSessionKey: bool) -> Union[bool, db.Entity]: """ Validates a security key. If useSessionKey is true, the key is expected to be the sessions current security key (or it's static security key). Otherwise it must be a key created with a duration (so it's not session dependent) :param key: The key to validate :param useSessionKey: If True, we validate against the session's skey, otherwise we'll lookup an unbound key :returns: False if the key was not valid for whatever reasons, the data (given during createSecurityKey) as dictionary or True if the dict is empty (or :param:useSessionKey was true). """ if useSessionKey: if key == "staticSessionKey": skeyHeaderValue = currentRequest.get().request.headers.get( "Sec-X-ViUR-StaticSKey") if skeyHeaderValue and currentSession.get( ).validateStaticSecurityKey(skeyHeaderValue): return True elif currentSession.get().validateSecurityKey(key): return True return False if not key: return False dbKey = db.Key(securityKeyKindName, key) dbObj = db.Get(dbKey) if dbObj: db.Delete(dbKey) until = dbObj["until"] if until < utcNow(): # This key has expired return False del dbObj["until"] if not dbObj: return True return dbObj return False
def postDeletedHandler(self, skel, boneName, key): dbVals = db.Query("viur-relations") # skel.kindName+"_"+self.kind+"_"+key dbVals.filter("viur_src_kind =", skel.kindName) dbVals.filter("viur_dest_kind =", self.kind) dbVals.filter("viur_src_property =", boneName) dbVals.filter("src.key =", key) db.Delete([x for x in dbVals.run(keysOnly=True)])
def doClearSKeys(timeStamp, cursor): query = db.Query(securityKeyKindName).filter( "until <", datetime.strptime(timeStamp, "%d.%m.%Y %H:%M:%S")) for oldKey in query.run(100, keysOnly=True): db.Delete(oldKey) newCursor = query.getCursor() if newCursor: doClearSKeys(timeStamp, newCursor)
def doClearSessions(timeStamp, cursor): gotAtLeastOne = False query = db.Query(GaeSession.kindName).filter("lastseen <", timeStamp) for oldKey in query.run(100, keysOnly=True): gotAtLeastOne = True db.Delete(oldKey) newCursor = query.getCursor() if gotAtLeastOne and newCursor and newCursor.urlsafe() != cursor: doClearSessions(timeStamp, newCursor.urlsafe())
def doClearSKeys(timeStamp, cursor): gotAtLeastOne = False query = db.Query(securityKeyKindName).filter( "until <", datetime.strptime(timeStamp, "%d.%m.%Y %H:%M:%S")) for oldKey in query.run(100, keysOnly=True): gotAtLeastOne = True db.Delete(oldKey) newCursor = query.getCursor() if gotAtLeastOne and newCursor and newCursor.urlsafe() != cursor: doClearSKeys(timeStamp, newCursor.urlsafe())
def getOldBlobKeysTxn(dbKey): obj = db.Get(dbKey) res = obj["old_blob_references"] or [] if obj["is_stale"]: db.Delete(dbKey) else: obj["has_old_blob_references"] = False obj["old_blob_references"] = [] db.Put(obj) return res
def refreshIndex(self, query: db.Query): """ Refreshes the Index for the given query (Actually it removes it from the db so it gets rebuild on next use) :param query: Query for which the index should be refreshed :type query: db.Query """ key = self.keyFromQuery(query) try: db.Delete(db.Key.from_path(self._dbType, key)) except: pass
def killSessionByUser(user=None): """ Invalidates all active sessions for the given *user*. This means that this user is instantly logged out. If no user is given, it tries to invalidate **all** active sessions. Use "guest" as to kill all sessions not associated with an user. :param user: UserID, "guest" or None. :type user: str | None """ logging.error("Invalidating all sessions for %s" % user) query = db.Query(GaeSession.kindName) if user is not None: query.filter("user =", str(user)) for key in query.iter(keysOnly=True): db.Delete(key)
def refreshIndex(self, query): """ Refreshes the Index for the given query (Actually it removes it from the db so it gets rebuild on next use) :param query: Query for which the index should be refreshed :type query: db.Query """ key = self.keyFromQuery(query) try: db.Delete(db.Key.from_path(self._dbType, key)) except db.EntityNotFoundError: pass # try/except is faster than if clause try: del self._cache[key] except: pass
def reset(self): """ Invalids the current session and starts a new one. This function is especially useful at login, where we might need to create an SSL-capable session. :warning: Everything (except the current language) is flushed. """ lang = self.session.get("language") if self.cookieKey: db.Delete(db.Key(self.kindName, self.cookieKey)) self.cookieKey = utils.generateRandomString(42) self.staticSecurityKey = utils.generateRandomString(13) self.securityKey = utils.generateRandomString(13) self.changed = True self.isInitial = True self.session = db.Entity() if lang: self.session["language"] = lang
def postSavedHandler(self, skel, boneName, key): if not skel[boneName]: values = [] elif self.multiple and self.languages: values = chain(*skel[boneName].values()) elif self.languages: values = list(skel[boneName].values()) elif self.multiple: values = skel[boneName] else: values = [skel[boneName]] values = [x for x in values if x is not None] #elif isinstance(skel[boneName], dict): # values = [dict((k, v) for k, v in skel[boneName].items())] #else: # values = [dict((k, v) for k, v in x.items()) for x in skel[boneName]] parentValues = db.Entity() srcEntity = skel.dbEntity parentValues.key = srcEntity.key for boneKey in (self.parentKeys or []): parentValues[boneKey] = srcEntity.get(boneKey) dbVals = db.Query("viur-relations") dbVals.filter("viur_src_kind =", skel.kindName) dbVals.filter("viur_dest_kind =", self.kind) dbVals.filter("viur_src_property =", boneName) dbVals.filter("src.__key__ =", key) for dbObj in dbVals.iter(): try: if not dbObj["dest"].key in [x["dest"]["key"] for x in values]: # Relation has been removed db.Delete(dbObj.key) continue except: # This entry is corrupt db.Delete(dbObj.key) else: # Relation: Updated data = [x for x in values if x["dest"]["key"] == dbObj["dest"].key][0] # Write our (updated) values in refSkel = data["dest"] dbObj["dest"] = refSkel.serialize(parentIndexed=True) dbObj["src"] = parentValues if self.using is not None: usingSkel = data["rel"] dbObj["rel"] = usingSkel.serialize(parentIndexed=True) dbObj["viur_delayed_update_tag"] = time() dbObj["viur_relational_updateLevel"] = self.updateLevel dbObj["viur_relational_consistency"] = self.consistency.value dbObj["viur_foreign_keys"] = self.refKeys dbObj["viurTags"] = srcEntity.get("viurTags") # Copy tags over so we can still use our searchengine db.Put(dbObj) values.remove(data) # Add any new Relation for val in values: dbObj = db.Entity(db.Key("viur-relations", parent=key)) refSkel = val["dest"] dbObj["dest"] = refSkel.serialize(parentIndexed=True) dbObj["src"] = parentValues if self.using is not None: usingSkel = val["rel"] dbObj["rel"] = usingSkel.serialize(parentIndexed=True) dbObj["viur_delayed_update_tag"] = time() dbObj["viur_src_kind"] = skel.kindName # The kind of the entry referencing dbObj["viur_src_property"] = boneName # The key of the bone referencing dbObj["viur_dest_kind"] = self.kind dbObj["viur_relational_updateLevel"] = self.updateLevel dbObj["viur_relational_consistency"] = self.consistency.value dbObj["viur_foreign_keys"] = self.refKeys db.Put(dbObj)
def doClearSessions(timeStamp: str) -> None: query = db.Query(GaeSession.kindName).filter("lastseen <", timeStamp) for oldKey in query.run(100): db.Delete(oldKey) if query.getCursor(): doClearSessions(timeStamp)
def cron(self, cronName="default", *args, **kwargs): global _callableTasks, _periodicTasks, _appengineServiceIPs req = currentRequest.get() if not req.isDevServer: if 'X-Appengine-Cron' not in req.request.headers: logging.critical( 'Detected an attempted XSRF attack. The header "X-AppEngine-Cron" was not set.' ) raise errors.Forbidden() if req.request.environ.get( "HTTP_X_APPENGINE_USER_IP") not in _appengineServiceIPs: logging.critical( 'Detected an attempted XSRF attack. This request did not originate from Cron.' ) raise errors.Forbidden() if cronName not in _periodicTasks: logging.warning( "Got Cron request '%s' which doesn't have any tasks" % cronName) for task, interval in _periodicTasks[cronName].items( ): # Call all periodic tasks bound to that queue periodicTaskName = task.periodicTaskName.lower() if interval: # Ensure this task doesn't get called to often lastCall = db.Get( db.Key("viur-task-interval", periodicTaskName)) if lastCall and utils.utcNow() - lastCall["date"] < timedelta( minutes=interval): logging.debug( "Skipping task %s - Has already run recently." % periodicTaskName) continue res = self.findBoundTask(task) try: if res: # Its bound, call it this way :) res[0]() else: task( ) # It seems it wasn't bound - call it as a static method except Exception as e: logging.error("Error calling periodic task %s", periodicTaskName) logging.exception(e) else: logging.debug("Successfully called task %s", periodicTaskName) if interval: # Update its last-call timestamp entry = db.Entity( db.Key("viur-task-interval", periodicTaskName)) entry["date"] = utils.utcNow() db.Put(entry) logging.debug("Periodic tasks complete") for currentTask in db.Query( "viur-queued-tasks").iter(): # Look for queued tasks db.Delete(currentTask.key()) if currentTask["taskid"] in _callableTasks: task = _callableTasks[currentTask["taskid"]]() tmpDict = {} for k in currentTask.keys(): if k == "taskid": continue tmpDict[k] = json.loads(currentTask[k]) try: task.execute(**tmpDict) except Exception as e: logging.error("Error executing Task") logging.exception(e) logging.debug("Scheduled tasks complete")
def handleEntry(cls, entry, customData): db.Delete(entry.key)
def doClearSKeys(timeStamp: str) -> None: query = db.Query(securityKeyKindName).filter("until <", datetime.strptime(timeStamp, "%d.%m.%Y %H:%M:%S")) for oldKey in query.run(100): db.Delete(oldKey) if query.getCursor(): doClearSKeys(timeStamp)