def flushCache(prefix="/*"):
"""
Flushes the cache. Its possible the flush only a part of the cache by specifying
the path-prefix.
:param prefix: Path or prefix that should be flushed.
:type prefix: str
Examples:
- "/" would flush the main page (and only that),
- "/*" everything from the cache, "/page/*" everything from the page-module (default render),
- and "/page/view/*" only that specific subset of the page-module.
"""
items = db.Query(viurCacheName).filter(
"path =", prefix.rstrip("*")).iter(keysOnly=True)
for item in items:
db.Delete(item)
if prefix.endswith("*"):
items = db.Query(viurCacheName).filter(
"path >", prefix.rstrip("*")).filter(
"path <",
prefix.rstrip("*") + u"\ufffd").iter(keysOnly=True)
for item in items:
db.Delete(item)
logging.debug(
"Flushing cache succeeded. Everything matching \"%s\" is gone." %
prefix)
def doCleanupDeletedFiles(cursor=None):
maxIterCount = 2 # How often a file will be checked for deletion
query = db.Query("viur-deleted-files")
if cursor:
query.setCursor(cursor)
for file in query.run(100):
if not "dlkey" in file:
db.Delete(file.key)
elif db.Query("viur-blob-locks").filter("active_blob_references =",
file["dlkey"]).getEntry():
logging.info("is referenced, %s" % file["dlkey"])
db.Delete(file.key)
else:
if file["itercount"] > maxIterCount:
logging.info("Finally deleting, %s" % file["dlkey"])
blobs = bucket.list_blobs(prefix="%s/" % file["dlkey"])
for blob in blobs:
blob.delete()
db.Delete(file.key)
# There should be exactly 1 or 0 of these
for f in skeletonByKind("file")().all().filter(
"dlkey =", file["dlkey"]).fetch(99):
f.delete()
else:
logging.debug("Increasing count, %s" % file["dlkey"])
file["itercount"] += 1
db.Put(file)
newCursor = query.getCursor()
if newCursor:
doCleanupDeletedFiles(newCursor)
def flushCache(prefix: str = None,
key: Union[db.Key, None] = None,
kind: Union[str, None] = None):
"""
Flushes the cache. Its possible the flush only a part of the cache by specifying
the path-prefix. The path is equal to the url that caused it to be cached (eg /page/view) and must be one
listed in the 'url' param of :meth:`viur.core.cache.enableCache`.
:param prefix: Path or prefix that should be flushed.
:param key: Flush all cache entries which may contain this key. Also flushes entries
which executed a query over that kind.
:param kind: Flush all cache entries which executed a query over that kind.
Examples:
- "/" would flush the main page (and only that),
- "/*" everything from the cache, "/page/*" everything from the page-module (default render),
- and "/page/view/*" only that specific subset of the page-module.
"""
if prefix is None and key is None and kind is None:
prefix = "/*"
if prefix is not None:
items = db.Query(viurCacheName).filter("path =",
prefix.rstrip("*")).iter()
for item in items:
db.Delete(item)
if prefix.endswith("*"):
items = db.Query(viurCacheName) \
.filter("path >", prefix.rstrip("*")) \
.filter("path <", prefix.rstrip("*") + u"\ufffd") \
.iter()
for item in items:
db.Delete(item)
logging.debug(
"Flushing cache succeeded. Everything matching \"%s\" is gone." %
prefix)
if key is not None:
items = db.Query(viurCacheName).filter("accessedEntries =", key).iter()
for item in items:
logging.info("Deleted cache entry %s", item["path"])
db.Delete(item.key)
if not isinstance(key, db.Key):
key = db.Key(encoded=key)
items = db.Query(viurCacheName).filter("accessedEntries =",
key.kind).iter()
for item in items:
logging.info("Deleted cache entry %s", item["path"])
db.Delete(item.key)
if kind is not None:
items = db.Query(viurCacheName).filter("accessedEntries =",
kind).iter()
for item in items:
logging.info("Deleted cache entry %s", item["path"])
db.Delete(item.key)
def doClearSessions(timeStamp, cursor):
query = db.Query(GaeSession.kindName).filter("lastseen <", timeStamp)
for oldKey in query.run(100, keysOnly=True):
db.Delete(oldKey)
newCursor = query.getCursor()
if newCursor:
doClearSessions(timeStamp, newCursor)
def validate(key: str, useSessionKey: bool) -> Union[bool, db.Entity]:
"""
Validates a onetime securitykey
:type key: str
:param key: The key to validate
:type useSessionKey: Bool
:param useSessionKey: If True, we validate against the session's skey, otherwise we'll lookup an unbound key
:returns: False if the key was not valid for whatever reasons, the data (given during createSecurityKey) as dictionary or True if the dict is empty.
"""
if useSessionKey:
if key == "staticSessionKey":
skeyHeaderValue = request.current.get().request.headers.get(
"Sec-X-ViUR-StaticSKey")
if skeyHeaderValue and currentSession.validateStaticSecurityKey(
skeyHeaderValue):
return True
elif currentSession.validateSecurityKey(key):
return True
return False
if not key or "/" in key:
return False
dbObj = db.Get((securityKeyKindName, key))
if dbObj:
db.Delete((securityKeyKindName, key))
if dbObj["until"] < datetime.now(): # This key has expired
return False
del dbObj["until"]
if not dbObj:
return True
return dbObj
return False
def validate(key: str, useSessionKey: bool) -> Union[bool, db.Entity]:
"""
Validates a security key. If useSessionKey is true, the key is expected to be the sessions current security key
(or it's static security key). Otherwise it must be a key created with a duration (so it's not session
dependent)
:param key: The key to validate
:param useSessionKey: If True, we validate against the session's skey, otherwise we'll lookup an unbound key
:returns: False if the key was not valid for whatever reasons, the data (given during createSecurityKey) as
dictionary or True if the dict is empty (or :param:useSessionKey was true).
"""
if useSessionKey:
if key == "staticSessionKey":
skeyHeaderValue = currentRequest.get().request.headers.get(
"Sec-X-ViUR-StaticSKey")
if skeyHeaderValue and currentSession.get(
).validateStaticSecurityKey(skeyHeaderValue):
return True
elif currentSession.get().validateSecurityKey(key):
return True
return False
if not key:
return False
dbKey = db.Key(securityKeyKindName, key)
dbObj = db.Get(dbKey)
if dbObj:
db.Delete(dbKey)
until = dbObj["until"]
if until < utcNow(): # This key has expired
return False
del dbObj["until"]
if not dbObj:
return True
return dbObj
return False
def postDeletedHandler(self, skel, boneName, key):
dbVals = db.Query("viur-relations") # skel.kindName+"_"+self.kind+"_"+key
dbVals.filter("viur_src_kind =", skel.kindName)
dbVals.filter("viur_dest_kind =", self.kind)
dbVals.filter("viur_src_property =", boneName)
dbVals.filter("src.key =", key)
db.Delete([x for x in dbVals.run(keysOnly=True)])
def doClearSKeys(timeStamp, cursor):
query = db.Query(securityKeyKindName).filter(
"until <", datetime.strptime(timeStamp, "%d.%m.%Y %H:%M:%S"))
for oldKey in query.run(100, keysOnly=True):
db.Delete(oldKey)
newCursor = query.getCursor()
if newCursor:
doClearSKeys(timeStamp, newCursor)
def doClearSessions(timeStamp, cursor):
gotAtLeastOne = False
query = db.Query(GaeSession.kindName).filter("lastseen <", timeStamp)
for oldKey in query.run(100, keysOnly=True):
gotAtLeastOne = True
db.Delete(oldKey)
newCursor = query.getCursor()
if gotAtLeastOne and newCursor and newCursor.urlsafe() != cursor:
doClearSessions(timeStamp, newCursor.urlsafe())
def doClearSKeys(timeStamp, cursor):
gotAtLeastOne = False
query = db.Query(securityKeyKindName).filter(
"until <", datetime.strptime(timeStamp, "%d.%m.%Y %H:%M:%S"))
for oldKey in query.run(100, keysOnly=True):
gotAtLeastOne = True
db.Delete(oldKey)
newCursor = query.getCursor()
if gotAtLeastOne and newCursor and newCursor.urlsafe() != cursor:
doClearSKeys(timeStamp, newCursor.urlsafe())
def getOldBlobKeysTxn(dbKey):
obj = db.Get(dbKey)
res = obj["old_blob_references"] or []
if obj["is_stale"]:
db.Delete(dbKey)
else:
obj["has_old_blob_references"] = False
obj["old_blob_references"] = []
db.Put(obj)
return res
def refreshIndex(self, query: db.Query):
"""
Refreshes the Index for the given query
(Actually it removes it from the db so it gets rebuild on next use)
:param query: Query for which the index should be refreshed
:type query: db.Query
"""
key = self.keyFromQuery(query)
try:
db.Delete(db.Key.from_path(self._dbType, key))
except:
pass
def killSessionByUser(user=None):
"""
Invalidates all active sessions for the given *user*.
This means that this user is instantly logged out.
If no user is given, it tries to invalidate **all** active sessions.
Use "guest" as to kill all sessions not associated with an user.
:param user: UserID, "guest" or None.
:type user: str | None
"""
logging.error("Invalidating all sessions for %s" % user)
query = db.Query(GaeSession.kindName)
if user is not None:
query.filter("user =", str(user))
for key in query.iter(keysOnly=True):
db.Delete(key)
def refreshIndex(self, query):
"""
Refreshes the Index for the given query
(Actually it removes it from the db so it gets rebuild on next use)
:param query: Query for which the index should be refreshed
:type query: db.Query
"""
key = self.keyFromQuery(query)
try:
db.Delete(db.Key.from_path(self._dbType, key))
except db.EntityNotFoundError:
pass
# try/except is faster than if clause
try:
del self._cache[key]
except:
pass
def reset(self):
"""
Invalids the current session and starts a new one.
This function is especially useful at login, where
we might need to create an SSL-capable session.
:warning: Everything (except the current language) is flushed.
"""
lang = self.session.get("language")
if self.cookieKey:
db.Delete(db.Key(self.kindName, self.cookieKey))
self.cookieKey = utils.generateRandomString(42)
self.staticSecurityKey = utils.generateRandomString(13)
self.securityKey = utils.generateRandomString(13)
self.changed = True
self.isInitial = True
self.session = db.Entity()
if lang:
self.session["language"] = lang
def postSavedHandler(self, skel, boneName, key):
if not skel[boneName]:
values = []
elif self.multiple and self.languages:
values = chain(*skel[boneName].values())
elif self.languages:
values = list(skel[boneName].values())
elif self.multiple:
values = skel[boneName]
else:
values = [skel[boneName]]
values = [x for x in values if x is not None]
#elif isinstance(skel[boneName], dict):
# values = [dict((k, v) for k, v in skel[boneName].items())]
#else:
# values = [dict((k, v) for k, v in x.items()) for x in skel[boneName]]
parentValues = db.Entity()
srcEntity = skel.dbEntity
parentValues.key = srcEntity.key
for boneKey in (self.parentKeys or []):
parentValues[boneKey] = srcEntity.get(boneKey)
dbVals = db.Query("viur-relations")
dbVals.filter("viur_src_kind =", skel.kindName)
dbVals.filter("viur_dest_kind =", self.kind)
dbVals.filter("viur_src_property =", boneName)
dbVals.filter("src.__key__ =", key)
for dbObj in dbVals.iter():
try:
if not dbObj["dest"].key in [x["dest"]["key"] for x in values]: # Relation has been removed
db.Delete(dbObj.key)
continue
except: # This entry is corrupt
db.Delete(dbObj.key)
else: # Relation: Updated
data = [x for x in values if x["dest"]["key"] == dbObj["dest"].key][0]
# Write our (updated) values in
refSkel = data["dest"]
dbObj["dest"] = refSkel.serialize(parentIndexed=True)
dbObj["src"] = parentValues
if self.using is not None:
usingSkel = data["rel"]
dbObj["rel"] = usingSkel.serialize(parentIndexed=True)
dbObj["viur_delayed_update_tag"] = time()
dbObj["viur_relational_updateLevel"] = self.updateLevel
dbObj["viur_relational_consistency"] = self.consistency.value
dbObj["viur_foreign_keys"] = self.refKeys
dbObj["viurTags"] = srcEntity.get("viurTags") # Copy tags over so we can still use our searchengine
db.Put(dbObj)
values.remove(data)
# Add any new Relation
for val in values:
dbObj = db.Entity(db.Key("viur-relations", parent=key))
refSkel = val["dest"]
dbObj["dest"] = refSkel.serialize(parentIndexed=True)
dbObj["src"] = parentValues
if self.using is not None:
usingSkel = val["rel"]
dbObj["rel"] = usingSkel.serialize(parentIndexed=True)
dbObj["viur_delayed_update_tag"] = time()
dbObj["viur_src_kind"] = skel.kindName # The kind of the entry referencing
dbObj["viur_src_property"] = boneName # The key of the bone referencing
dbObj["viur_dest_kind"] = self.kind
dbObj["viur_relational_updateLevel"] = self.updateLevel
dbObj["viur_relational_consistency"] = self.consistency.value
dbObj["viur_foreign_keys"] = self.refKeys
db.Put(dbObj)
def doClearSessions(timeStamp: str) -> None:
query = db.Query(GaeSession.kindName).filter("lastseen <", timeStamp)
for oldKey in query.run(100):
db.Delete(oldKey)
if query.getCursor():
doClearSessions(timeStamp)
def cron(self, cronName="default", *args, **kwargs):
global _callableTasks, _periodicTasks, _appengineServiceIPs
req = currentRequest.get()
if not req.isDevServer:
if 'X-Appengine-Cron' not in req.request.headers:
logging.critical(
'Detected an attempted XSRF attack. The header "X-AppEngine-Cron" was not set.'
)
raise errors.Forbidden()
if req.request.environ.get(
"HTTP_X_APPENGINE_USER_IP") not in _appengineServiceIPs:
logging.critical(
'Detected an attempted XSRF attack. This request did not originate from Cron.'
)
raise errors.Forbidden()
if cronName not in _periodicTasks:
logging.warning(
"Got Cron request '%s' which doesn't have any tasks" %
cronName)
for task, interval in _periodicTasks[cronName].items(
): # Call all periodic tasks bound to that queue
periodicTaskName = task.periodicTaskName.lower()
if interval: # Ensure this task doesn't get called to often
lastCall = db.Get(
db.Key("viur-task-interval", periodicTaskName))
if lastCall and utils.utcNow() - lastCall["date"] < timedelta(
minutes=interval):
logging.debug(
"Skipping task %s - Has already run recently." %
periodicTaskName)
continue
res = self.findBoundTask(task)
try:
if res: # Its bound, call it this way :)
res[0]()
else:
task(
) # It seems it wasn't bound - call it as a static method
except Exception as e:
logging.error("Error calling periodic task %s",
periodicTaskName)
logging.exception(e)
else:
logging.debug("Successfully called task %s", periodicTaskName)
if interval:
# Update its last-call timestamp
entry = db.Entity(
db.Key("viur-task-interval", periodicTaskName))
entry["date"] = utils.utcNow()
db.Put(entry)
logging.debug("Periodic tasks complete")
for currentTask in db.Query(
"viur-queued-tasks").iter(): # Look for queued tasks
db.Delete(currentTask.key())
if currentTask["taskid"] in _callableTasks:
task = _callableTasks[currentTask["taskid"]]()
tmpDict = {}
for k in currentTask.keys():
if k == "taskid":
continue
tmpDict[k] = json.loads(currentTask[k])
try:
task.execute(**tmpDict)
except Exception as e:
logging.error("Error executing Task")
logging.exception(e)
logging.debug("Scheduled tasks complete")
def handleEntry(cls, entry, customData):
db.Delete(entry.key)
def doClearSKeys(timeStamp: str) -> None:
query = db.Query(securityKeyKindName).filter("until <", datetime.strptime(timeStamp, "%d.%m.%Y %H:%M:%S"))
for oldKey in query.run(100):
db.Delete(oldKey)
if query.getCursor():
doClearSKeys(timeStamp)
