def doCleanupDeletedFiles(cursor = None):
maxIterCount = 2 # How often a file will be checked for deletion
gotAtLeastOne = False
query = db.Query("viur-deleted-files").cursor( cursor )
for file in query.run(100):
gotAtLeastOne = True
if not "dlkey" in file.keys():
db.Delete(file.key())
elif db.Query("viur-blob-locks").filter("active_blob_references =", file["dlkey"]).get():
logging.info("is referenced, %s" % file["dlkey"])
db.Delete(file.key())
else:
if file["itercount"] > maxIterCount:
logging.info("Finally deleting, %s" % file["dlkey"])
blobstore.delete(file["dlkey"])
db.Delete(file.key())
# There should be exactly 1 or 0 of these
for f in db.Query("file").filter("dlkey =", file["dlkey"]).iter(keysOnly=True):
db.Delete(f)
else:
logging.error("Increasing count, %s" % file["dlkey"])
file["itercount"] += 1
db.Put(file)
newCursor = query.getCursor()
if gotAtLeastOne and newCursor and newCursor.urlsafe() != cursor:
doCleanupDeletedFiles(newCursor.urlsafe())
def flushCache(prefix="/*"):
"""
Flushes the cache. Its possible the flush only a part of the cache by specifying
the path-prefix.
:param prefix: Path or prefix that should be flushed.
:type prefix: str
Examples:
- "/" would flush the main page (and only that),
- "/*" everything from the cache, "/page/*" everything from the page-module (default render),
- and "/page/view/*" only that specific subset of the page-module.
"""
items = db.Query(viurCacheName).filter(
"path =", prefix.rstrip("*")).iter(keysOnly=True)
for item in items:
db.Delete(item)
if prefix.endswith("*"):
items = db.Query(viurCacheName).filter(
"path >", prefix.rstrip("*")).filter(
"path <",
prefix.rstrip("*") + u"\ufffd").iter(keysOnly=True)
for item in items:
db.Delete(item)
logging.debug(
"Flushing cache succeeded. Everything matching \"%s\" is gone." %
prefix)
def validate(key, acceptSessionKey=False):
"""
Validates a onetime securitykey
:type key: str
:param key: The key to validate
:type acceptSessionKey: Bool
:param acceptSessionKey: If True, we also accept the session's skey
:returns: False if the key was not valid for whatever reasons, the data (given during createSecurityKey) as dictionary or True if the dict is empty.
"""
if acceptSessionKey:
if key == currentSession.getSessionSecurityKey():
return (True)
try:
dbObj = db.Get(db.Key.from_path(securityKeyKindName, key))
except:
return (False)
if dbObj:
if "session" in dbObj and dbObj["session"] is not None:
if dbObj["session"] != currentSession.getSessionKey():
return (False)
db.Delete(dbObj.key())
if dbObj["until"] < datetime.now(): #This key has expired
return (False)
res = {}
for k in dbObj.keys():
res[k] = dbObj[k]
del res["session"]
del res["until"]
if not res:
return (True)
return (res)
return (False)
def doClearSessions(timeStamp, cursor):
gotAtLeastOne = False
query = db.Query(GaeSession.kindName).filter("lastseen <", timeStamp)
for oldKey in query.run(100, keysOnly=True):
gotAtLeastOne = True
db.Delete(oldKey)
newCursor = query.getCursor()
if gotAtLeastOne and newCursor and newCursor.urlsafe() != cursor:
doClearSessions(timeStamp, newCursor.urlsafe())
def getOldBlobKeysTxn(dbKey): obj = db.Get(dbKey) res = obj["old_blob_references"] or [] if obj["is_stale"]: db.Delete(dbKey) else: obj["has_old_blob_references"] = False obj["old_blob_references"] = [] db.Put(obj) return res
def doClearSKeys(timeStamp, cursor):
gotAtLeastOne = False
query = db.Query(securityKeyKindName).filter(
"until <", datetime.strptime(timeStamp, "%d.%m.%Y %H:%M:%S"))
for oldKey in query.run(100, keysOnly=True):
gotAtLeastOne = True
db.Delete(oldKey)
newCursor = query.getCursor()
if gotAtLeastOne and newCursor and newCursor.urlsafe() != cursor:
doClearSKeys(timeStamp, newCursor.urlsafe())
def txnDelete(key, skel):
dbObj = db.Get(db.Key(
key)) # Fetch the raw object as we might have to clear locks
for boneName, bone in skel.items():
# Ensure that we delete any value-lock objects remaining for this entry
if bone.unique:
try:
logging.error("x1")
logging.error(dbObj.keys())
if "%s.uniqueIndexValue" % boneName in dbObj.keys():
logging.error("x2")
db.Delete(
db.Key.from_path(
"%s_%s_uniquePropertyIndex" %
(skel.kindName, boneName),
dbObj["%s.uniqueIndexValue" % boneName]))
except db.EntityNotFoundError:
raise
pass
# Delete the blob-key lock object
try:
lockObj = db.Get(db.Key.from_path("viur-blob-locks", str(key)))
except:
lockObj = None
if lockObj is not None:
if lockObj["old_blob_references"] is None and lockObj[
"active_blob_references"] is None:
db.Delete(lockObj) # Nothing to do here
else:
if lockObj["old_blob_references"] is None:
# No old stale entries, move active_blob_references -> old_blob_references
lockObj["old_blob_references"] = lockObj[
"active_blob_references"]
elif lockObj["active_blob_references"] is not None:
# Append the current references to the list of old & stale references
lockObj["old_blob_references"] += lockObj[
"active_blob_references"]
lockObj["active_blob_references"] = [
] # There are no active ones left
lockObj["is_stale"] = True
lockObj["has_old_blob_references"] = True
db.Put(lockObj)
db.Delete(db.Key(key))
def index(self, *args, **kwargs):
global _callableTasks, _periodicTasks
logging.debug("Starting maintenance-run")
checkUpdate() #Let the update-module verify the database layout first
logging.debug("Updatecheck complete")
for task, intervall in _periodicTasks.items(
): #Call all periodic tasks
if intervall: #Ensure this task doesn't get called to often
try:
lastCall = db.Get(
db.Key.from_path("viur-task-interval",
task.periodicTaskName))
if lastCall["date"] > datetime.now() - timedelta(
minutes=intervall):
logging.debug(
"Skipping task %s - Has already run recently." %
task.periodicTaskName)
continue
except db.EntityNotFoundError:
pass
res = self.findBoundTask(task)
if res: #Its bound, call it this way :)
res[0]()
else:
task() #It seems it wasnt bound - call it as a static method
logging.debug("Successfully called task %s" %
task.periodicTaskName)
if intervall:
# Update its last-call timestamp
entry = db.Entity("viur-task-interval",
name=task.periodicTaskName)
entry["date"] = datetime.now()
db.Put(entry)
logging.debug("Periodic tasks complete")
for currentTask in db.Query(
"viur-queued-tasks").iter(): #Look for queued tasks
db.Delete(currentTask.key())
if currentTask["taskid"] in _callableTasks:
task = _callableTasks[currentTask["taskid"]]()
tmpDict = {}
for k in currentTask.keys():
if k == "taskid":
continue
tmpDict[k] = json.loads(currentTask[k])
try:
task.execute(**tmpDict)
except Exception as e:
logging.error("Error executing Task")
logging.exception(e)
logging.debug("Scheduled tasks complete")
def refreshIndex(self, query):
"""
Refreshes the Index for the given query
(Actually it removes it from the db so it gets rebuild on next use)
:param query: Query for which the index should be refreshed
:type query: db.Query
"""
key = self.keyFromQuery(query)
try:
db.Delete(db.Key.from_path(self._dbType, key))
except db.EntityNotFoundError:
pass
if key in self._cache.keys():
del self._cache[key]
def killSessionByUser(user=None):
"""
Invalidates all active sessions for the given *user*.
This means that this user is instantly logged out.
If no user is given, it tries to invalidate **all** active sessions.
Use "guest" as to kill all sessions not associated with an user.
:param user: UserID, "guest" or None.
:type user: str | None
"""
logging.error("Invalidating all sessions for %s" % user)
query = db.Query(GaeSession.kindName)
if user is not None:
query.filter("user =", str(user))
for key in query.iter(keysOnly=True):
db.Delete(key)
def reset(self):
"""
Invalids the current session and starts a new one.
This function is especially useful at login, where
we might need to create an SSL-capable session.
:warning: Everything (except the current language) is flushed.
"""
lang = self.session.get("language")
if self.key:
db.Delete(db.Key.from_path(self.kindName, self.key))
self.key = None
self.sslKey = None
self.sessionSecurityKey = None
self.changed = True
self.session = {}
if lang:
self.session["language"] = lang
def postDeletedHandler(self, skel, key, id):
db.Delete([
x for x in db.Query("viur-relations").ancestor(db.Key(id)).run(
keysOnly=True)
])
def postSavedHandler(self, valuesCache, boneName, skel, key, dbfields):
if not valuesCache[boneName]:
values = []
elif isinstance(valuesCache[boneName], dict):
values = [dict((k, v) for k, v in valuesCache[boneName].items())]
else:
values = [
dict((k, v) for k, v in x.items())
for x in valuesCache[boneName]
]
parentValues = {}
for parentKey in dbfields.keys():
if parentKey in self.parentKeys or any(
[parentKey.startswith(x + ".") for x in self.parentKeys]):
parentValues[parentKey] = dbfields[parentKey]
dbVals = db.Query("viur-relations").ancestor(
db.Key(key)) #skel.kindName+"_"+self.kind+"_"+key
dbVals.filter("viur_src_kind =", skel.kindName)
dbVals.filter("viur_dest_kind =", self.kind)
dbVals.filter("viur_src_property =", boneName)
for dbObj in dbVals.iter():
try:
if not dbObj["dest.key"] in [x["dest"]["key"] for x in values
]: #Relation has been removed
db.Delete(dbObj.key())
continue
except: #This entry is corrupt
db.Delete(dbObj.key())
else: # Relation: Updated
data = [
x for x in values if x["dest"]["key"] == dbObj["dest.key"]
][0]
if self.indexed: #We dont store more than key and kinds, and these dont change
#Write our (updated) values in
refSkel = self._refSkelCache
refSkel.setValuesCache(data["dest"])
for k, v in refSkel.serialize().items():
dbObj["dest." + k] = v
for k, v in parentValues.items():
dbObj["src." + k] = v
if self.using is not None:
usingSkel = self._usingSkelCache
usingSkel.setValuesCache(data["rel"])
for k, v in usingSkel.serialize().items():
dbObj["rel." + k] = v
dbObj["viur_delayed_update_tag"] = time()
db.Put(dbObj)
values.remove(data)
# Add any new Relation
for val in values:
dbObj = db.Entity(
"viur-relations",
parent=db.Key(key)) #skel.kindName+"_"+self.kind+"_"+key
if not self.indexed: #Dont store more than key and kinds, as they aren't used anyway
dbObj["dest.key"] = val["dest"]["key"]
dbObj["src.key"] = key
else:
refSkel = self._refSkelCache
refSkel.setValuesCache(val["dest"])
for k, v in refSkel.serialize().items():
dbObj["dest." + k] = v
for k, v in parentValues.items():
dbObj["src." + k] = v
if self.using is not None:
usingSkel = self._usingSkelCache
usingSkel.setValuesCache(val["rel"])
for k, v in usingSkel.serialize().items():
dbObj["rel." + k] = v
dbObj["viur_delayed_update_tag"] = time()
dbObj[
"viur_src_kind"] = skel.kindName #The kind of the entry referencing
#dbObj[ "viur_src_key" ] = str( key ) #The key of the entry referencing
dbObj[
"viur_src_property"] = boneName #The key of the bone referencing
#dbObj[ "viur_dest_key" ] = val["key"]
dbObj["viur_dest_kind"] = self.kind
db.Put(dbObj)
def txnUpdate(key, mergeFrom, clearUpdateTag):
blobList = set()
skel = type(mergeFrom)()
# Load the current values from Datastore or create a new, empty db.Entity
if not key:
dbObj = db.Entity(skel.kindName)
oldBlobLockObj = None
else:
k = db.Key(key)
assert k.kind() == skel.kindName, "Cannot write to invalid kind!"
try:
dbObj = db.Get(k)
except db.EntityNotFoundError:
dbObj = db.Entity(k.kind(), id=k.id(), name=k.name(), parent=k.parent())
else:
skel.setValues(dbObj)
try:
oldBlobLockObj = db.Get(db.Key.from_path("viur-blob-locks", str(k)))
except:
oldBlobLockObj = None
# Remember old hashes for bones that must have an unique value
oldUniqeValues = {}
for boneName, boneInstance in skel.items():
if boneInstance.unique:
if "%s.uniqueIndexValue" % boneName in dbObj:
oldUniqeValues[boneName] = dbObj["%s.uniqueIndexValue" % boneName]
## Merge the values from mergeFrom in
for key, bone in skel.items():
if key in mergeFrom:
bone.mergeFrom(skel.valuesCache, key, mergeFrom)
for key, _bone in skel.items():
dbObj = _bone.serialize(skel.valuesCache, key, dbObj)
blobList.update(_bone.getReferencedBlobs(self.valuesCache, key))
if clearUpdateTag:
dbObj["viur_delayed_update_tag"] = 0 # Mark this entity as Up-to-date.
else:
dbObj[
"viur_delayed_update_tag"] = time() # Mark this entity as dirty, so the background-task will catch it up and update its references.
dbObj = skel.preProcessSerializedData(dbObj)
try:
ourKey = str(dbObj.key())
except: # Its not an update but an insert, no key yet
ourKey = None
# Lock hashes from bones that must have unique values
newUniqeValues = {}
for boneName, boneInstance in skel.items():
if boneInstance.unique:
# Check if the property is really unique
newUniqeValues[boneName] = boneInstance.getUniquePropertyIndexValue(
self.valuesCache, boneName)
if newUniqeValues[boneName] is not None:
try:
lockObj = db.Get(db.Key.from_path(
"%s_%s_uniquePropertyIndex" % (skel.kindName, boneName),
newUniqeValues[boneName]))
if lockObj["references"] != ourKey:
# This value has been claimed, and that not by us
raise ValueError(
"The unique value '%s' of bone '%s' has been recently claimed!" %
(self.valuesCache[boneName], boneName))
except db.EntityNotFoundError: # No lockObj found for that value, we can use that
pass
dbObj["%s.uniqueIndexValue" % boneName] = newUniqeValues[boneName]
else:
if "%s.uniqueIndexValue" % boneName in dbObj:
del dbObj["%s.uniqueIndexValue" % boneName]
if not skel.searchIndex:
# We generate the searchindex using the full skel, not this (maybe incomplete one)
tags = []
for key, _bone in skel.items():
if _bone.searchable:
tags += [tag for tag in _bone.getSearchTags(self.valuesCache, key) if
(tag not in tags and len(tag) < 400)]
dbObj["viur_tags"] = tags
db.Put(dbObj) # Write the core entry back
# Now write the blob-lock object
blobList = skel.preProcessBlobLocks(blobList)
if blobList is None:
raise ValueError(
"Did you forget to return the bloblist somewhere inside getReferencedBlobs()?")
if None in blobList:
raise ValueError("None is not a valid blobKey.")
if oldBlobLockObj is not None:
oldBlobs = set(oldBlobLockObj["active_blob_references"] if oldBlobLockObj[
"active_blob_references"] is not None else [])
removedBlobs = oldBlobs - blobList
oldBlobLockObj["active_blob_references"] = list(blobList)
if oldBlobLockObj["old_blob_references"] is None:
oldBlobLockObj["old_blob_references"] = [x for x in removedBlobs]
else:
tmp = set(oldBlobLockObj["old_blob_references"] + [x for x in removedBlobs])
oldBlobLockObj["old_blob_references"] = [x for x in (tmp - blobList)]
oldBlobLockObj["has_old_blob_references"] = oldBlobLockObj[
"old_blob_references"] is not None and len(
oldBlobLockObj["old_blob_references"]) > 0
oldBlobLockObj["is_stale"] = False
db.Put(oldBlobLockObj)
else: # We need to create a new blob-lock-object
blobLockObj = db.Entity("viur-blob-locks", name=str(dbObj.key()))
blobLockObj["active_blob_references"] = list(blobList)
blobLockObj["old_blob_references"] = []
blobLockObj["has_old_blob_references"] = False
blobLockObj["is_stale"] = False
db.Put(blobLockObj)
for boneName, boneInstance in skel.items():
if boneInstance.unique:
# Update/create/delete missing lock-objects
if boneName in oldUniqeValues and oldUniqeValues[boneName] != \
newUniqeValues[boneName]:
# We had an old lock and its value changed
try:
# Try to delete the old lock
oldLockObj = db.Get(db.Key.from_path(
"%s_%s_uniquePropertyIndex" % (skel.kindName, boneName),
oldUniqeValues[boneName]))
if oldLockObj["references"] != ourKey:
# We've been supposed to have that lock - but we don't.
# Don't remove that lock as it now belongs to a different entry
logging.critical(
"Detected Database corruption! A Value-Lock had been reassigned!")
else:
# It's our lock which we don't need anymore
db.Delete(db.Key.from_path(
"%s_%s_uniquePropertyIndex" % (
skel.kindName, boneName),
oldUniqeValues[boneName]))
except db.EntityNotFoundError as e:
logging.critical(
"Detected Database corruption! Could not delete stale lock-object!")
if newUniqeValues[boneName] is not None:
# Lock the new value
newLockObj = db.Entity(
"%s_%s_uniquePropertyIndex" % (skel.kindName, boneName),
name=newUniqeValues[boneName])
newLockObj["references"] = str(dbObj.key())
db.Put(newLockObj)
return (str(dbObj.key()), dbObj, skel)
