def storeEntry(self, e, key):
if not self._checkKey(key, export=False):
raise errors.Forbidden()
entry = pickle.loads(e)
for k in list(entry.keys())[:]:
if isinstance(entry[k], str):
entry[k] = entry[k].decode("UTF-8")
if "key" in entry.keys():
key = db.Key(encoded=entry["key"])
elif "id" in entry.keys():
key = db.Key(encoded=entry["id"])
else:
raise AttributeError()
logging.error(key.kind())
logging.error(key.id())
logging.error(key.name())
dbEntry = db.Entity(kind=key.kind(),
parent=key.parent(),
id=key.id(),
_app=key.app(),
name=key.name()) #maybe some more fixes here ?
for k in entry.keys():
if k != "key":
val = entry[k]
#if isinstance(val, dict) or isinstance(val, list):
# val = pickle.dumps( val )
dbEntry[k] = val
if dbEntry.key().id():
# Ensure the Datastore knows that it's id is in use
datastore._GetConnection()._reserve_keys([dbEntry.key()])
db.Put(dbEntry)
def doCheckForUnreferencedBlobs(cursor = None):
def getOldBlobKeysTxn(dbKey):
obj = db.Get(dbKey)
res = obj["old_blob_references"] or []
if obj["is_stale"]:
db.Delete(dbKey)
else:
obj["has_old_blob_references"] = False
obj["old_blob_references"] = []
db.Put(obj)
return res
gotAtLeastOne = False
query = db.Query("viur-blob-locks").filter("has_old_blob_references", True).cursor(cursor)
for lockKey in query.run(100, keysOnly=True):
gotAtLeastOne = True
oldBlobKeys = db.RunInTransaction( getOldBlobKeysTxn, lockKey )
for blobKey in oldBlobKeys:
if db.Query("viur-blob-locks").filter("active_blob_references =", blobKey).get():
#This blob is referenced elsewhere
logging.info("Stale blob is still referenced, %s" % blobKey)
continue
# Add a marker and schedule it for deletion
fileObj = db.Query( "viur-deleted-files" ).filter( "dlkey", blobKey ).get()
if fileObj: #Its already marked
logging.info("Stale blob already marked for deletion, %s" % blobKey)
return
fileObj = db.Entity("viur-deleted-files")
fileObj["itercount"] = 0
fileObj["dlkey"] = str(blobKey)
logging.info("Stale blob marked dirty, %s" % blobKey)
db.Put(fileObj)
newCursor = query.getCursor()
if gotAtLeastOne and newCursor and newCursor.urlsafe() != cursor:
doCheckForUnreferencedBlobs(newCursor.urlsafe())
def save(self, req): """ Writes the session to the memcache/datastore. Does nothing, if the session hasn't been changed in the current request. """ if self.changed: serialized = base64.b64encode( pickle.dumps(self.session, protocol=pickle.HIGHEST_PROTOCOL ) ) self.getSessionKey( req ) # Get the current user id userid = None try: if "user" in dir( conf["viur.mainApp"] ): #Check for our custom user-api userid = conf["viur.mainApp"].user.getCurrentUser()["key"] except: pass try: dbSession = db.Entity( self.kindName, name=self.key ) dbSession["data"] = serialized dbSession["sslkey"] = self.sslKey dbSession["skey"] = self.sessionSecurityKey dbSession["lastseen"] = time() dbSession["user"] = str(userid) or "guest" #Store the userid inside the sessionobj, so we can kill specific sessions if needed dbSession.set_unindexed_properties( ["data","sslkey" ] ) db.Put( dbSession ) except (OverQuotaError, CapabilityDisabledError): pass req.response.headers.add_header( "Set-Cookie", bytes( "%s=%s; Max-Age=99999; Path=/; HttpOnly" % ( self.plainCookieName, self.key ) ) ) if req.isSSLConnection: req.response.headers.add_header( "Set-Cookie", bytes( "%s=%s; Max-Age=99999; Path=/; Secure; HttpOnly" % ( self.sslCookieName, self.sslKey ) ) )
def storeEntry2(self, e, key):
if not self._checkKey(key, export=False):
raise errors.Forbidden()
entry = pickle.loads(e.decode("HEX"))
for k in list(entry.keys())[:]:
if isinstance(entry[k], str):
entry[k] = entry[k].decode("UTF-8")
key = db.Key(encoded=utils.normalizeKey(entry["key"]))
logging.info(key.kind())
logging.info(key.id())
logging.info(key.name())
dbEntry = db.Entity(kind=key.kind(),
parent=key.parent(),
id=key.id(),
name=key.name()) #maybe some more fixes here ?
for k in entry.keys():
if k != "key":
val = entry[k]
dbEntry[k] = val
db.Put(dbEntry)
try:
skel = skeletonByKind(key.kind())()
except:
logging.error("Unknown Skeleton - skipping")
skel.fromDB(str(dbEntry.key()))
skel.refresh()
skel.toDB(clearUpdateTag=True)
def upload(self, oldkey, *args, **kwargs):
logging.error("got UPLOADS")
res = []
for upload in self.getUploads():
fileName = self.decodeFileName(upload.filename)
if str(upload.content_type).startswith("image/"):
try:
servingURL = get_serving_url(upload.key())
except:
servingURL = ""
else:
servingURL = ""
res.append({
"name": fileName,
"size": upload.size,
"mimetype": upload.content_type,
"dlkey": str(upload.key()),
"servingurl": servingURL,
"parentdir": "",
"parentrepo": "",
"weak": False
})
oldKeyHash = sha256(oldkey).hexdigest().encode("hex")
e = db.Entity("viur-blobimportmap", name=oldKeyHash)
e["newkey"] = str(upload.key())
e["oldkey"] = oldkey
e["servingurl"] = servingURL
e["available"] = True
db.Put(e)
return (json.dumps({"action": "addSuccess", "values": res}))
def create(duration=None, **kwargs):
"""
Creates a new onetime Securitykey for the current session
If duration is not set, this key is valid only for the current session.
Otherwise, the key and its data is serialized and saved inside the datastore
for up to duration-seconds
:param duration: Make this key valid for a fixed timeframe (and independend of the current session)
:type duration: int or None
:returns: The new onetime key
"""
key = generateRandomString()
if duration is None:
sessionDependend = True
duration = 30 * 60 # 30 Mins from now
else:
sessionDependend = False
duration = int(duration)
dbObj = db.Entity(securityKeyKindName, name=key)
for k, v in kwargs.items():
dbObj[k] = v
dbObj["until"] = datetime.now() + timedelta(seconds=duration)
if sessionDependend:
dbObj["session"] = currentSession.getSessionKey()
else:
dbObj["session"] = None
dbObj.set_unindexed_properties(
[x for x in dbObj.keys() if not x == "until"])
db.Put(dbObj)
return (key)
def index(self, *args, **kwargs):
global _callableTasks, _periodicTasks
logging.debug("Starting maintenance-run")
checkUpdate() #Let the update-module verify the database layout first
logging.debug("Updatecheck complete")
for task, intervall in _periodicTasks.items(
): #Call all periodic tasks
if intervall: #Ensure this task doesn't get called to often
try:
lastCall = db.Get(
db.Key.from_path("viur-task-interval",
task.periodicTaskName))
if lastCall["date"] > datetime.now() - timedelta(
minutes=intervall):
logging.debug(
"Skipping task %s - Has already run recently." %
task.periodicTaskName)
continue
except db.EntityNotFoundError:
pass
res = self.findBoundTask(task)
if res: #Its bound, call it this way :)
res[0]()
else:
task() #It seems it wasnt bound - call it as a static method
logging.debug("Successfully called task %s" %
task.periodicTaskName)
if intervall:
# Update its last-call timestamp
entry = db.Entity("viur-task-interval",
name=task.periodicTaskName)
entry["date"] = datetime.now()
db.Put(entry)
logging.debug("Periodic tasks complete")
for currentTask in db.Query(
"viur-queued-tasks").iter(): #Look for queued tasks
db.Delete(currentTask.key())
if currentTask["taskid"] in _callableTasks:
task = _callableTasks[currentTask["taskid"]]()
tmpDict = {}
for k in currentTask.keys():
if k == "taskid":
continue
tmpDict[k] = json.loads(currentTask[k])
try:
task.execute(**tmpDict)
except Exception as e:
logging.error("Error executing Task")
logging.exception(e)
logging.debug("Scheduled tasks complete")
def wrapF(self, *args, **kwargs):
currentRequest = request.current.get()
if conf["viur.disableCache"] or currentRequest.disableCache:
# Caching disabled
if conf["viur.disableCache"]:
logging.debug("Caching is disabled by config")
return (f(self, *args, **kwargs))
# How many arguments are part of the way to the function called (and how many are just *args)
offset = -len(currentRequest.args) or len(currentRequest.pathlist)
path = "/" + "/".join(currentRequest.pathlist[:offset])
if not path in urls:
# This path (possibly a sub-render) should not be cached
logging.debug("Not caching for %s" % path)
return (f(self, *args, **kwargs))
key = keyFromArgs(f, userSensitive, languageSensitive, evaluatedArgs,
path, args, kwargs)
if not key:
# Someting is wrong (possibly the parameter-count)
# Letz call f, but we knew already that this will clash
return (f(self, *args, **kwargs))
try:
dbRes = db.Get(db.Key.from_path(viurCacheName, key))
except db.EntityNotFoundError:
dbRes = None
if dbRes:
if not maxCacheTime or \
dbRes["creationtime"] > datetime.now()-timedelta( seconds=maxCacheTime ):
# We store it unlimited or the cache is fresh enough
logging.debug("This request was served from cache.")
currentRequest.response.headers['Content-Type'] = dbRes[
"content-type"].encode("UTF-8")
return (dbRes["data"])
# If we made it this far, the request wasnt cached or too old; we need to rebuild it
res = f(self, *args, **kwargs)
dbEntity = db.Entity(viurCacheName, name=key)
dbEntity["data"] = res
dbEntity["creationtime"] = datetime.now()
dbEntity["path"] = path
dbEntity["content-type"] = request.current.get(
).response.headers['Content-Type']
dbEntity.set_unindexed_properties(["data", "content-type"
]) #We can save 2 DB-Writs :)
db.Put(dbEntity)
logging.debug("This request was a cache-miss. Cache has been updated.")
return (res)
def getOrBuildIndex(self, origQuery):
"""
Builds a specific index based on origQuery AND local variables (self.indexPage and self.indexMaxPage)
Returns a list of starting-cursors for each page.
You probably shouldn't call this directly. Use cursorForQuery.
:param origQuery: Query to build the index for
:type origQuery: db.Query
:param key: DB-Key to save the index to
:type key: string
:returns: []
"""
key = self.keyFromQuery(origQuery)
if key in self._cache.keys(): #We have it cached
return (self._cache[key])
#We dont have it cached - try to load it from DB
try:
index = db.Get(db.Key.from_path(self._dbType, key))
res = json.loads(index["data"])
self._cache[key] = res
return (res)
except db.EntityNotFoundError: #Its not in the datastore, too
pass
#We dont have this index yet.. Build it
#Clone the original Query
queryRes = origQuery.clone(keysOnly=True).datastoreQuery.Run(
limit=self.maxPages * self.pageSize)
#Build-Up the index
res = []
i = 0
previousCursor = None #The first page dosnt have any cursor
for discardedKey in queryRes:
if i % self.pageSize == 0:
res.append(previousCursor)
if i % self.pageSize == (self.pageSize - 1) and i > 1:
previousCursor = str(queryRes.cursor().urlsafe())
i += 1
if len(res) == 0: # Ensure that the first page exists
res.append(None)
entry = db.Entity(self._dbType, name=key)
entry["data"] = json.dumps(res)
entry["creationdate"] = datetime.now()
db.Put(entry)
return (res)
def markFileForDeletion(dlkey):
"""
Adds a marker to the data store that the file specified as *dlkey* can be deleted.
Once the mark has been set, the data store is checked four times (default: every 4 hours)
if the file is in use somewhere. If it is still in use, the mark goes away, otherwise
the mark and the file are removed from the datastore. These delayed checks are necessary
due to database inconsistency.
:type dlkey: str
:param dlkey: Unique download-key of the file that shall be marked for deletion.
"""
fileObj = db.Query("viur-deleted-files").filter("dlkey", dlkey).get()
if fileObj: #Its allready marked
return
fileObj = db.Entity("viur-deleted-files")
fileObj["itercount"] = 0
fileObj["dlkey"] = str(dlkey)
db.Put(fileObj)
def postSavedHandler(self, valuesCache, boneName, skel, key, dbfields):
if not valuesCache[boneName]:
values = []
elif isinstance(valuesCache[boneName], dict):
values = [dict((k, v) for k, v in valuesCache[boneName].items())]
else:
values = [
dict((k, v) for k, v in x.items())
for x in valuesCache[boneName]
]
parentValues = {}
for parentKey in dbfields.keys():
if parentKey in self.parentKeys or any(
[parentKey.startswith(x + ".") for x in self.parentKeys]):
parentValues[parentKey] = dbfields[parentKey]
dbVals = db.Query("viur-relations").ancestor(
db.Key(key)) #skel.kindName+"_"+self.kind+"_"+key
dbVals.filter("viur_src_kind =", skel.kindName)
dbVals.filter("viur_dest_kind =", self.kind)
dbVals.filter("viur_src_property =", boneName)
for dbObj in dbVals.iter():
try:
if not dbObj["dest.key"] in [x["dest"]["key"] for x in values
]: #Relation has been removed
db.Delete(dbObj.key())
continue
except: #This entry is corrupt
db.Delete(dbObj.key())
else: # Relation: Updated
data = [
x for x in values if x["dest"]["key"] == dbObj["dest.key"]
][0]
if self.indexed: #We dont store more than key and kinds, and these dont change
#Write our (updated) values in
refSkel = self._refSkelCache
refSkel.setValuesCache(data["dest"])
for k, v in refSkel.serialize().items():
dbObj["dest." + k] = v
for k, v in parentValues.items():
dbObj["src." + k] = v
if self.using is not None:
usingSkel = self._usingSkelCache
usingSkel.setValuesCache(data["rel"])
for k, v in usingSkel.serialize().items():
dbObj["rel." + k] = v
dbObj["viur_delayed_update_tag"] = time()
db.Put(dbObj)
values.remove(data)
# Add any new Relation
for val in values:
dbObj = db.Entity(
"viur-relations",
parent=db.Key(key)) #skel.kindName+"_"+self.kind+"_"+key
if not self.indexed: #Dont store more than key and kinds, as they aren't used anyway
dbObj["dest.key"] = val["dest"]["key"]
dbObj["src.key"] = key
else:
refSkel = self._refSkelCache
refSkel.setValuesCache(val["dest"])
for k, v in refSkel.serialize().items():
dbObj["dest." + k] = v
for k, v in parentValues.items():
dbObj["src." + k] = v
if self.using is not None:
usingSkel = self._usingSkelCache
usingSkel.setValuesCache(val["rel"])
for k, v in usingSkel.serialize().items():
dbObj["rel." + k] = v
dbObj["viur_delayed_update_tag"] = time()
dbObj[
"viur_src_kind"] = skel.kindName #The kind of the entry referencing
#dbObj[ "viur_src_key" ] = str( key ) #The key of the entry referencing
dbObj[
"viur_src_property"] = boneName #The key of the bone referencing
#dbObj[ "viur_dest_key" ] = val["key"]
dbObj["viur_dest_kind"] = self.kind
db.Put(dbObj)
def checkout(self, step=None, key=None, skey=None, *args, **kwargs):
"""
Performs the checkout process trough the state machine provided by self.steps.
:param step: The current step index, None for beginning a new checkout
:param key: Key of the current checkout
:param skey: Server security key
:return: Returns the rendered template or throws redirection exceptions.
"""
myKindName = self.viewSkel().kindName
if step is None:
logging.info("Starting new checkout process")
billObj = db.Entity(myKindName)
billObj["idx"] = "0000000"
for state in self.states:
billObj["state_%s" % state] = "0"
db.Put(billObj)
key = str(billObj.key())
#Copy the Cart
if "amountSkel" in dir(self):
cart = session.current.get("cart_products") or {}
s = self.amountSkel()
products = []
for prod, atts in cart.items():
for i in range(0, atts["amount"]):
products.append(str(prod))
s.fromClient({"product": products})
s.toDB()
session.current["order_" + myKindName] = {
"key": str(key),
"completedSteps": []
}
session.current.markChanged()
raise errors.Redirect("?step=0&key=%s" % str(key))
elif key:
try:
orderKey = db.Key(key)
step = int(step)
assert (step >= 0)
assert (step < len(self.steps))
except:
raise errors.NotAcceptable()
sessionInfo = session.current.get("order_" + myKindName)
if not sessionInfo or not sessionInfo.get("key") == str(orderKey):
raise errors.Unauthorized()
if step in sessionInfo["completedSteps"]:
session.current["order_" + myKindName]["completedSteps"] = [
x for x in sessionInfo["completedSteps"] if x < step
]
session.current.markChanged()
#Make sure that no steps can be skipped
if step != 0 and not step - 1 in sessionInfo["completedSteps"]:
raise errors.Redirect("?step=0&key=%s" % str(str(orderKey)))
currentStep = self.steps[step]
if "preHandler" in currentStep.keys():
try:
if isinstance(currentStep["preHandler"], list):
for handler in currentStep["preHandler"]:
handler(self, step, str(orderKey), *args, **kwargs)
else:
currentStep["preHandler"](self,
step,
str(orderKey),
refkwargs=kwargs,
*args,
**kwargs)
except SkipStepException:
session.current["order_" +
myKindName]["completedSteps"].append(step)
session.current.markChanged()
raise errors.Redirect("?step=%s&key=%s" %
(str(step + 1), str(orderKey)))
except ReturnHtmlException as e:
return (e.html)
if "requiresSecurityKey" in currentStep and currentStep[
"requiresSecurityKey"]:
if not securitykey.validate(skey):
raise errors.PreconditionFailed()
pass
if "mainHandler" in currentStep:
if currentStep["mainHandler"]["action"] == "edit":
skel = self.getSkelByName(
currentStep["mainHandler"]["skeleton"], str(orderKey))
skel.fromDB(str(orderKey))
if not len(kwargs.keys()) or not skel.fromClient(kwargs):
return (self.render.edit(
skel,
tpl=currentStep["mainHandler"]["template"],
step=step))
skel.toDB()
if currentStep["mainHandler"]["action"] == "view":
if not "complete" in kwargs or not kwargs[
"complete"] == u"1":
skel = self.getSkelByName(
currentStep["mainHandler"]["skeleton"],
str(orderKey))
skel.fromDB(str(orderKey))
return (self.render.view(
skel,
tpl=currentStep["mainHandler"]["template"],
step=step))
elif currentStep["mainHandler"]["action"] == "function":
res = currentStep["mainHandler"]["function"](self, step,
str(orderKey),
*args,
**kwargs)
if res:
return res
if "postHandler" in currentStep:
currentStep["postHandler"](self, step, str(orderKey), *args,
**kwargs)
session.current["order_" +
myKindName]["completedSteps"].append(step)
session.current.markChanged()
logging.info("next ?step=%s&key=%s" %
(str(step + 1), str(orderKey)))
raise errors.Redirect("?step=%s&key=%s" %
(str(step + 1), str(orderKey)))
def iterImport(module, target, exportKey, cursor=None, amount=0):
"""
Processes 100 Entries and calls the next batch
"""
urlfetch.set_default_fetch_deadline(20)
payload = {"module": module, "key": exportKey}
if cursor:
payload.update({"cursor": cursor})
result = urlfetch.fetch(
url=target,
payload=urllib.urlencode(payload),
method=urlfetch.POST,
headers={'Content-Type': 'application/x-www-form-urlencoded'})
if result.status_code == 200:
res = pickle.loads(result.content.decode("HEX"))
skel = skeletonByKind(module)()
logging.info("%s: %d new entries fetched, total %d entries fetched" %
(module, len(res["values"]), amount))
if len(res["values"]) == 0:
try:
utils.sendEMailToAdmins(
"Import of kind %s finished with %d entities" %
(module, amount), "ViUR finished to import %d entities of "
"kind %s from %s.\n" % (amount, module, target))
except: #OverQuota, whatever
logging.error("Unable to send Email")
return
for entry in res["values"]:
for k in list(entry.keys())[:]:
if isinstance(entry[k], str):
entry[k] = entry[k].decode("UTF-8")
if not "key" in entry.keys():
entry["key"] = entry["id"]
key = db.Key(encoded=utils.normalizeKey(entry["key"]))
# Special case: Convert old module root nodes!!!
if module.endswith(
"_rootNode") and key.name() and "_modul_" in key.name():
name = key.name().replace("_modul_", "_module_")
else:
name = key.name()
dbEntry = db.Entity(kind=key.kind(),
parent=key.parent(),
id=key.id(),
name=name)
for k in entry.keys():
if k == "key":
continue
dbEntry[k] = entry[k]
# Special case: Convert old module root nodes!!!
if (isinstance(skel, (HierarchySkel, TreeLeafSkel))
and k in ["parentdir", "parententry", "parentrepo"]
and entry[k]):
key = db.Key(encoded=str(entry[k]))
if key.parent():
parent = db.Key(
encoded=utils.normalizeKey(key.parent()))
else:
parent = None
if key.id_or_name() and "_modul_" in str(key.id_or_name()):
name = key.id_or_name().replace("_modul_", "_module_")
else:
name = key.id_or_name()
dbEntry[k] = str(
db.Key.from_path(key.kind(), name, parent=parent))
db.Put(dbEntry)
skel.fromDB(str(dbEntry.key()))
skel.refresh()
skel.toDB(clearUpdateTag=True)
amount += 1
iterImport(module, target, exportKey, res["cursor"], amount)
def txnUpdate(key, mergeFrom, clearUpdateTag):
blobList = set()
skel = type(mergeFrom)()
# Load the current values from Datastore or create a new, empty db.Entity
if not key:
dbObj = db.Entity(skel.kindName)
oldBlobLockObj = None
else:
k = db.Key(key)
assert k.kind() == skel.kindName, "Cannot write to invalid kind!"
try:
dbObj = db.Get(k)
except db.EntityNotFoundError:
dbObj = db.Entity(k.kind(), id=k.id(), name=k.name(), parent=k.parent())
else:
skel.setValues(dbObj)
try:
oldBlobLockObj = db.Get(db.Key.from_path("viur-blob-locks", str(k)))
except:
oldBlobLockObj = None
# Remember old hashes for bones that must have an unique value
oldUniqeValues = {}
for boneName, boneInstance in skel.items():
if boneInstance.unique:
if "%s.uniqueIndexValue" % boneName in dbObj:
oldUniqeValues[boneName] = dbObj["%s.uniqueIndexValue" % boneName]
## Merge the values from mergeFrom in
for key, bone in skel.items():
if key in mergeFrom:
bone.mergeFrom(skel.valuesCache, key, mergeFrom)
for key, _bone in skel.items():
dbObj = _bone.serialize(skel.valuesCache, key, dbObj)
blobList.update(_bone.getReferencedBlobs(self.valuesCache, key))
if clearUpdateTag:
dbObj["viur_delayed_update_tag"] = 0 # Mark this entity as Up-to-date.
else:
dbObj[
"viur_delayed_update_tag"] = time() # Mark this entity as dirty, so the background-task will catch it up and update its references.
dbObj = skel.preProcessSerializedData(dbObj)
try:
ourKey = str(dbObj.key())
except: # Its not an update but an insert, no key yet
ourKey = None
# Lock hashes from bones that must have unique values
newUniqeValues = {}
for boneName, boneInstance in skel.items():
if boneInstance.unique:
# Check if the property is really unique
newUniqeValues[boneName] = boneInstance.getUniquePropertyIndexValue(
self.valuesCache, boneName)
if newUniqeValues[boneName] is not None:
try:
lockObj = db.Get(db.Key.from_path(
"%s_%s_uniquePropertyIndex" % (skel.kindName, boneName),
newUniqeValues[boneName]))
if lockObj["references"] != ourKey:
# This value has been claimed, and that not by us
raise ValueError(
"The unique value '%s' of bone '%s' has been recently claimed!" %
(self.valuesCache[boneName], boneName))
except db.EntityNotFoundError: # No lockObj found for that value, we can use that
pass
dbObj["%s.uniqueIndexValue" % boneName] = newUniqeValues[boneName]
else:
if "%s.uniqueIndexValue" % boneName in dbObj:
del dbObj["%s.uniqueIndexValue" % boneName]
if not skel.searchIndex:
# We generate the searchindex using the full skel, not this (maybe incomplete one)
tags = []
for key, _bone in skel.items():
if _bone.searchable:
tags += [tag for tag in _bone.getSearchTags(self.valuesCache, key) if
(tag not in tags and len(tag) < 400)]
dbObj["viur_tags"] = tags
db.Put(dbObj) # Write the core entry back
# Now write the blob-lock object
blobList = skel.preProcessBlobLocks(blobList)
if blobList is None:
raise ValueError(
"Did you forget to return the bloblist somewhere inside getReferencedBlobs()?")
if None in blobList:
raise ValueError("None is not a valid blobKey.")
if oldBlobLockObj is not None:
oldBlobs = set(oldBlobLockObj["active_blob_references"] if oldBlobLockObj[
"active_blob_references"] is not None else [])
removedBlobs = oldBlobs - blobList
oldBlobLockObj["active_blob_references"] = list(blobList)
if oldBlobLockObj["old_blob_references"] is None:
oldBlobLockObj["old_blob_references"] = [x for x in removedBlobs]
else:
tmp = set(oldBlobLockObj["old_blob_references"] + [x for x in removedBlobs])
oldBlobLockObj["old_blob_references"] = [x for x in (tmp - blobList)]
oldBlobLockObj["has_old_blob_references"] = oldBlobLockObj[
"old_blob_references"] is not None and len(
oldBlobLockObj["old_blob_references"]) > 0
oldBlobLockObj["is_stale"] = False
db.Put(oldBlobLockObj)
else: # We need to create a new blob-lock-object
blobLockObj = db.Entity("viur-blob-locks", name=str(dbObj.key()))
blobLockObj["active_blob_references"] = list(blobList)
blobLockObj["old_blob_references"] = []
blobLockObj["has_old_blob_references"] = False
blobLockObj["is_stale"] = False
db.Put(blobLockObj)
for boneName, boneInstance in skel.items():
if boneInstance.unique:
# Update/create/delete missing lock-objects
if boneName in oldUniqeValues and oldUniqeValues[boneName] != \
newUniqeValues[boneName]:
# We had an old lock and its value changed
try:
# Try to delete the old lock
oldLockObj = db.Get(db.Key.from_path(
"%s_%s_uniquePropertyIndex" % (skel.kindName, boneName),
oldUniqeValues[boneName]))
if oldLockObj["references"] != ourKey:
# We've been supposed to have that lock - but we don't.
# Don't remove that lock as it now belongs to a different entry
logging.critical(
"Detected Database corruption! A Value-Lock had been reassigned!")
else:
# It's our lock which we don't need anymore
db.Delete(db.Key.from_path(
"%s_%s_uniquePropertyIndex" % (
skel.kindName, boneName),
oldUniqeValues[boneName]))
except db.EntityNotFoundError as e:
logging.critical(
"Detected Database corruption! Could not delete stale lock-object!")
if newUniqeValues[boneName] is not None:
# Lock the new value
newLockObj = db.Entity(
"%s_%s_uniquePropertyIndex" % (skel.kindName, boneName),
name=newUniqeValues[boneName])
newLockObj["references"] = str(dbObj.key())
db.Put(newLockObj)
return (str(dbObj.key()), dbObj, skel)
