def save(self, req): """ Writes the session to the memcache/datastore. Does nothing, if the session hasn't been changed in the current request. """ if self.changed: serialized = base64.b64encode( pickle.dumps(self.session, protocol=pickle.HIGHEST_PROTOCOL ) ) self.getSessionKey( req ) # Get the current user id userid = None try: if "user" in dir( conf["viur.mainApp"] ): #Check for our custom user-api userid = conf["viur.mainApp"].user.getCurrentUser()["key"] except: pass try: dbSession = db.Entity( self.kindName, name=self.key ) dbSession["data"] = serialized dbSession["sslkey"] = self.sslKey dbSession["skey"] = self.sessionSecurityKey dbSession["lastseen"] = time() dbSession["user"] = str(userid) or "guest" #Store the userid inside the sessionobj, so we can kill specific sessions if needed dbSession.set_unindexed_properties( ["data","sslkey" ] ) db.Put( dbSession ) except (OverQuotaError, CapabilityDisabledError): pass req.response.headers.add_header( "Set-Cookie", bytes( "%s=%s; Max-Age=99999; Path=/; HttpOnly" % ( self.plainCookieName, self.key ) ) ) if req.isSSLConnection: req.response.headers.add_header( "Set-Cookie", bytes( "%s=%s; Max-Age=99999; Path=/; Secure; HttpOnly" % ( self.sslCookieName, self.sslKey ) ) )
def getKeyTxn( kindName, orderKey ): """Generates and returns a new, unique Key""" seqObj = db.GetOrInsert(kindName, "viur_bill_sequences", count=1000) idx = seqObj["count"] seqObj["count"] += 1 db.Put( seqObj ) return str(idx)
def storeEntry2(self, e, key):
if not self._checkKey(key, export=False):
raise errors.Forbidden()
entry = pickle.loads(e.decode("HEX"))
for k in list(entry.keys())[:]:
if isinstance(entry[k], str):
entry[k] = entry[k].decode("UTF-8")
key = db.Key(encoded=utils.normalizeKey(entry["key"]))
logging.info(key.kind())
logging.info(key.id())
logging.info(key.name())
dbEntry = db.Entity(kind=key.kind(),
parent=key.parent(),
id=key.id(),
name=key.name()) #maybe some more fixes here ?
for k in entry.keys():
if k != "key":
val = entry[k]
dbEntry[k] = val
db.Put(dbEntry)
try:
skel = skeletonByKind(key.kind())()
except:
logging.error("Unknown Skeleton - skipping")
skel.fromDB(str(dbEntry.key()))
skel.refresh()
skel.toDB(clearUpdateTag=True)
def upload(self, oldkey, *args, **kwargs):
logging.error("got UPLOADS")
res = []
for upload in self.getUploads():
fileName = self.decodeFileName(upload.filename)
if str(upload.content_type).startswith("image/"):
try:
servingURL = get_serving_url(upload.key())
except:
servingURL = ""
else:
servingURL = ""
res.append({
"name": fileName,
"size": upload.size,
"mimetype": upload.content_type,
"dlkey": str(upload.key()),
"servingurl": servingURL,
"parentdir": "",
"parentrepo": "",
"weak": False
})
oldKeyHash = sha256(oldkey).hexdigest().encode("hex")
e = db.Entity("viur-blobimportmap", name=oldKeyHash)
e["newkey"] = str(upload.key())
e["oldkey"] = oldkey
e["servingurl"] = servingURL
e["available"] = True
db.Put(e)
return (json.dumps({"action": "addSuccess", "values": res}))
def storeEntry(self, e, key):
if not self._checkKey(key, export=False):
raise errors.Forbidden()
entry = pickle.loads(e)
for k in list(entry.keys())[:]:
if isinstance(entry[k], str):
entry[k] = entry[k].decode("UTF-8")
if "key" in entry.keys():
key = db.Key(encoded=entry["key"])
elif "id" in entry.keys():
key = db.Key(encoded=entry["id"])
else:
raise AttributeError()
logging.error(key.kind())
logging.error(key.id())
logging.error(key.name())
dbEntry = db.Entity(kind=key.kind(),
parent=key.parent(),
id=key.id(),
_app=key.app(),
name=key.name()) #maybe some more fixes here ?
for k in entry.keys():
if k != "key":
val = entry[k]
#if isinstance(val, dict) or isinstance(val, list):
# val = pickle.dumps( val )
dbEntry[k] = val
if dbEntry.key().id():
# Ensure the Datastore knows that it's id is in use
datastore._GetConnection()._reserve_keys([dbEntry.key()])
db.Put(dbEntry)
def doCheckForUnreferencedBlobs(cursor = None):
def getOldBlobKeysTxn(dbKey):
obj = db.Get(dbKey)
res = obj["old_blob_references"] or []
if obj["is_stale"]:
db.Delete(dbKey)
else:
obj["has_old_blob_references"] = False
obj["old_blob_references"] = []
db.Put(obj)
return res
gotAtLeastOne = False
query = db.Query("viur-blob-locks").filter("has_old_blob_references", True).cursor(cursor)
for lockKey in query.run(100, keysOnly=True):
gotAtLeastOne = True
oldBlobKeys = db.RunInTransaction( getOldBlobKeysTxn, lockKey )
for blobKey in oldBlobKeys:
if db.Query("viur-blob-locks").filter("active_blob_references =", blobKey).get():
#This blob is referenced elsewhere
logging.info("Stale blob is still referenced, %s" % blobKey)
continue
# Add a marker and schedule it for deletion
fileObj = db.Query( "viur-deleted-files" ).filter( "dlkey", blobKey ).get()
if fileObj: #Its already marked
logging.info("Stale blob already marked for deletion, %s" % blobKey)
return
fileObj = db.Entity("viur-deleted-files")
fileObj["itercount"] = 0
fileObj["dlkey"] = str(blobKey)
logging.info("Stale blob marked dirty, %s" % blobKey)
db.Put(fileObj)
newCursor = query.getCursor()
if gotAtLeastOne and newCursor and newCursor.urlsafe() != cursor:
doCheckForUnreferencedBlobs(newCursor.urlsafe())
def setIndex(self, item, index, skey, *args, **kwargs):
"""
Changes the order of the elements in the current level by changing the index of *item*.
.. seealso:: :func:`canSetIndex`
:param item: URL-safe key of the item which index should be changed.
:type item: str
:param index: New index for this item. This value must be cast-able to float.
:type index: str
:returns: A rendered success result generated by the default renderer.
:raises: :exc:`server.errors.NotFound`, when no entry with the given *key* was found.
:raises: :exc:`server.errors.Unauthorized`, if the current user does not have the required permissions.
:raises: :exc:`server.errors.PreconditionFailed`, if the *skey* could not be verified.
"""
if not securitykey.validate(skey, acceptSessionKey=True):
raise errors.PreconditionFailed()
if not self.canSetIndex(item, index):
raise errors.Unauthorized()
fromItem = db.Get(item)
fromItem["sortindex"] = float(index)
db.Put(fromItem)
skel = self.editSkel()
assert skel.fromDB(item)
self.onItemSetIndex(skel)
self.onItemChanged(skel)
return self.render.setIndexSuccess(obj=fromItem)
def doCleanupDeletedFiles(cursor = None):
maxIterCount = 2 # How often a file will be checked for deletion
gotAtLeastOne = False
query = db.Query("viur-deleted-files").cursor( cursor )
for file in query.run(100):
gotAtLeastOne = True
if not "dlkey" in file.keys():
db.Delete(file.key())
elif db.Query("viur-blob-locks").filter("active_blob_references =", file["dlkey"]).get():
logging.info("is referenced, %s" % file["dlkey"])
db.Delete(file.key())
else:
if file["itercount"] > maxIterCount:
logging.info("Finally deleting, %s" % file["dlkey"])
blobstore.delete(file["dlkey"])
db.Delete(file.key())
# There should be exactly 1 or 0 of these
for f in db.Query("file").filter("dlkey =", file["dlkey"]).iter(keysOnly=True):
db.Delete(f)
else:
logging.error("Increasing count, %s" % file["dlkey"])
file["itercount"] += 1
db.Put(file)
newCursor = query.getCursor()
if gotAtLeastOne and newCursor and newCursor.urlsafe() != cursor:
doCleanupDeletedFiles(newCursor.urlsafe())
def login(self, name=None, password=None, skey="", *args, **kwargs):
if self.userModule.getCurrentUser(): #Were already logged in
return self.userModule.render.loginSucceeded()
if not name or not password or not securitykey.validate(skey):
return self.userModule.render.login(self.loginSkel())
query = db.Query(self.userModule.viewSkel().kindName)
res = query.filter("name.idx >=", name.lower()).get()
if res is None:
res = {"password": "", "status": 0, "name": "", "name.idx": ""}
if "password_salt" in res.keys(): # Its the new, more secure passwd
passwd = pbkdf2(password[:conf["viur.maxPasswordLength"]],
res["password_salt"])
else:
passwd = sha512(password.encode("UTF-8") +
conf["viur.salt"]).hexdigest()
isOkay = True
# We do this exactly that way to avoid timing attacks
# Check if the username matches
storedUserName = res.get("name.idx", "")
if len(storedUserName) != len(name.lower()):
isOkay = False
else:
for x, y in izip(storedUserName, name.lower()):
if x != y:
isOkay = False
# Check if the password matches
storedPasswordHash = res.get("password", "")
if len(storedPasswordHash) != len(passwd):
isOkay = False
else:
for x, y in izip(storedPasswordHash, passwd):
if x != y:
isOkay = False
# Verify that this account isn't blocked
if res["status"] < 10:
isOkay = False
if not isOkay:
skel = self.loginSkel()
skel.fromClient({"name": name, "nomissing": "1"})
return self.userModule.render.login(skel, loginFailed=True)
else:
if not "password_salt" in res.keys(
): #Update the password to the new, more secure format
res["password_salt"] = utils.generateRandomString(13)
res["password"] = pbkdf2(
password[:conf["viur.maxPasswordLength"]],
res["password_salt"])
db.Put(res)
return self.userModule.continueAuthenticationFlow(self, res.key())
def create(duration=None, **kwargs):
"""
Creates a new onetime Securitykey for the current session
If duration is not set, this key is valid only for the current session.
Otherwise, the key and its data is serialized and saved inside the datastore
for up to duration-seconds
:param duration: Make this key valid for a fixed timeframe (and independend of the current session)
:type duration: int or None
:returns: The new onetime key
"""
key = generateRandomString()
if duration is None:
sessionDependend = True
duration = 30 * 60 # 30 Mins from now
else:
sessionDependend = False
duration = int(duration)
dbObj = db.Entity(securityKeyKindName, name=key)
for k, v in kwargs.items():
dbObj[k] = v
dbObj["until"] = datetime.now() + timedelta(seconds=duration)
if sessionDependend:
dbObj["session"] = currentSession.getSessionKey()
else:
dbObj["session"] = None
dbObj.set_unindexed_properties(
[x for x in dbObj.keys() if not x == "until"])
db.Put(dbObj)
return (key)
def updateTransaction(userKey, idx):
user = db.Get(userKey)
if not "otptimedrift" in user.keys() or not isinstance(
user["otptimedrift"], float):
user["otptimedrift"] = 0.0
user["otptimedrift"] += min(max(0.1 * idx, -0.3), 0.3)
db.Put(user)
def setKeyTxn(orderKey, idx):
"""Assigns the new order to the given order"""
dbObj = db.Get(db.Key(orderKey))
if not dbObj:
return
dbObj["idx"] = idx
db.Put(dbObj)
def txn(orderKey, state, removeState):
dbObj = db.Get(db.Key(orderKey))
if not dbObj:
return
dbObj["state_%s" % state] = "1" if not removeState else "0"
dbObj["changedate"] = datetime.now()
db.Put(dbObj)
def reparent(self, item, dest, skey, *args, **kwargs):
"""
Moves an entry *item* (and everything beneath it) to another parent-node *dest*.
.. seealso:: :func:`canReparent`
:param item: URL-safe key of the item which will be moved.
:type item: str
:param dest: URL-safe key of the new parent for this item.
:type dest: str
:returns: A rendered success result generated by the default renderer.
:raises: :exc:`server.errors.NotFound`, when no entry with the given *id* was found.
:raises: :exc:`server.errors.Unauthorized`, if the current user does not have the required permissions.
:raises: :exc:`server.errors.PreconditionFailed`, if the *skey* could not be verified.
"""
if not securitykey.validate(skey, acceptSessionKey=True):
raise errors.PreconditionFailed()
if not self.canReparent(item, dest):
raise errors.Unauthorized()
if not self.isValidParent(dest) or item == dest:
raise errors.NotAcceptable()
## Test for recursion
isValid = False
currLevel = db.Get(dest)
for x in range(0, 99):
if str(currLevel.key()) == item:
break
if currLevel.key().kind(
) == self.viewSkel().kindName + "_rootNode":
# We reached a rootNode
isValid = True
break
currLevel = db.Get(currLevel["parententry"])
if not isValid:
raise errors.NotAcceptable()
## Update entry
fromItem = db.Get(item)
fromItem["parententry"] = dest
fromItem["parentrepo"] = str(self.getRootNode(dest).key())
db.Put(fromItem)
skel = self.editSkel()
assert skel.fromDB(item)
self.onItemReparent(skel)
self.onItemChanged(skel)
return self.render.reparentSuccess(obj=fromItem)
def getOldBlobKeysTxn(dbKey): obj = db.Get(dbKey) res = obj["old_blob_references"] or [] if obj["is_stale"]: db.Delete(dbKey) else: obj["has_old_blob_references"] = False obj["old_blob_references"] = [] db.Put(obj) return res
def index(self, *args, **kwargs):
global _callableTasks, _periodicTasks
logging.debug("Starting maintenance-run")
checkUpdate() #Let the update-module verify the database layout first
logging.debug("Updatecheck complete")
for task, intervall in _periodicTasks.items(
): #Call all periodic tasks
if intervall: #Ensure this task doesn't get called to often
try:
lastCall = db.Get(
db.Key.from_path("viur-task-interval",
task.periodicTaskName))
if lastCall["date"] > datetime.now() - timedelta(
minutes=intervall):
logging.debug(
"Skipping task %s - Has already run recently." %
task.periodicTaskName)
continue
except db.EntityNotFoundError:
pass
res = self.findBoundTask(task)
if res: #Its bound, call it this way :)
res[0]()
else:
task() #It seems it wasnt bound - call it as a static method
logging.debug("Successfully called task %s" %
task.periodicTaskName)
if intervall:
# Update its last-call timestamp
entry = db.Entity("viur-task-interval",
name=task.periodicTaskName)
entry["date"] = datetime.now()
db.Put(entry)
logging.debug("Periodic tasks complete")
for currentTask in db.Query(
"viur-queued-tasks").iter(): #Look for queued tasks
db.Delete(currentTask.key())
if currentTask["taskid"] in _callableTasks:
task = _callableTasks[currentTask["taskid"]]()
tmpDict = {}
for k in currentTask.keys():
if k == "taskid":
continue
tmpDict[k] = json.loads(currentTask[k])
try:
task.execute(**tmpDict)
except Exception as e:
logging.error("Error executing Task")
logging.exception(e)
logging.debug("Scheduled tasks complete")
def pwrecover(self, authtoken=None, skey=None, *args, **kwargs):
if authtoken:
data = securitykey.validate(authtoken)
if data and isinstance(data, dict) and "userKey" in data.keys(
) and "password" in data.keys():
skel = self.userModule.editSkel()
assert skel.fromDB(data["userKey"])
skel["password"] = data["password"]
skel.toDB()
return self.userModule.render.view(
skel, self.passwordRecoverySuccessTemplate)
else:
return self.userModule.render.view(
None, self.passwordRecoveryInvalidTokenTemplate)
else:
skel = self.lostPasswordSkel()
if len(kwargs) == 0 or not skel.fromClient(
kwargs) or not securitykey.validate(skey):
return self.userModule.render.passwdRecover(
skel, tpl=self.passwordRecoveryTemplate)
user = self.userModule.viewSkel().all().filter(
"name.idx =", skel["name"].lower()).get()
if not user or user[
"status"] < 10: # Unknown user or locked account
skel.errors["name"] = _("Unknown user")
return self.userModule.render.passwdRecover(
skel, tpl=self.passwordRecoveryTemplate)
try:
if user["changedate"] > (datetime.datetime.now() -
datetime.timedelta(days=1)):
# This user probably has already requested a password reset
# within the last 24 hrss
return self.userModule.render.view(
skel, self.passwordRecoveryAlreadySendTemplate)
except AttributeError: #Some newly generated user-objects dont have such a changedate yet
pass
user["changedate"] = datetime.datetime.now()
db.Put(user)
userSkel = self.userModule.viewSkel().ensureIsCloned()
assert userSkel.fromDB(user.key())
userSkel.skey = baseBone(descr="Skey")
userSkel["skey"] = securitykey.create(60 * 60 * 24,
userKey=str(user.key()),
password=skel["password"])
utils.sendEMail([userSkel["name"]],
self.userModule.passwordRecoveryMail, userSkel)
return self.userModule.render.view(
{}, self.passwordRecoveryInstuctionsSendTemplate)
def calculateOrderSum(self, step, orderKey, *args, **kwargs):
"""
Calculates the final price for this order.
This function *must* be called before any attempt is made to start a payment process.
:param step: Current step within the ordering process
:type step: int
:param orderKey: order to calculate the price for
:type orderKey: str
"""
price = sum([x[3] for x in self.getBillItems(orderKey)])
orderObj = db.Get(db.Key(str(orderKey)))
orderObj["price"] = price
db.Put(orderObj)
def wrapF(self, *args, **kwargs):
currentRequest = request.current.get()
if conf["viur.disableCache"] or currentRequest.disableCache:
# Caching disabled
if conf["viur.disableCache"]:
logging.debug("Caching is disabled by config")
return (f(self, *args, **kwargs))
# How many arguments are part of the way to the function called (and how many are just *args)
offset = -len(currentRequest.args) or len(currentRequest.pathlist)
path = "/" + "/".join(currentRequest.pathlist[:offset])
if not path in urls:
# This path (possibly a sub-render) should not be cached
logging.debug("Not caching for %s" % path)
return (f(self, *args, **kwargs))
key = keyFromArgs(f, userSensitive, languageSensitive, evaluatedArgs,
path, args, kwargs)
if not key:
# Someting is wrong (possibly the parameter-count)
# Letz call f, but we knew already that this will clash
return (f(self, *args, **kwargs))
try:
dbRes = db.Get(db.Key.from_path(viurCacheName, key))
except db.EntityNotFoundError:
dbRes = None
if dbRes:
if not maxCacheTime or \
dbRes["creationtime"] > datetime.now()-timedelta( seconds=maxCacheTime ):
# We store it unlimited or the cache is fresh enough
logging.debug("This request was served from cache.")
currentRequest.response.headers['Content-Type'] = dbRes[
"content-type"].encode("UTF-8")
return (dbRes["data"])
# If we made it this far, the request wasnt cached or too old; we need to rebuild it
res = f(self, *args, **kwargs)
dbEntity = db.Entity(viurCacheName, name=key)
dbEntity["data"] = res
dbEntity["creationtime"] = datetime.now()
dbEntity["path"] = path
dbEntity["content-type"] = request.current.get(
).response.headers['Content-Type']
dbEntity.set_unindexed_properties(["data", "content-type"
]) #We can save 2 DB-Writs :)
db.Put(dbEntity)
logging.debug("This request was a cache-miss. Cache has been updated.")
return (res)
def getOrBuildIndex(self, origQuery):
"""
Builds a specific index based on origQuery AND local variables (self.indexPage and self.indexMaxPage)
Returns a list of starting-cursors for each page.
You probably shouldn't call this directly. Use cursorForQuery.
:param origQuery: Query to build the index for
:type origQuery: db.Query
:param key: DB-Key to save the index to
:type key: string
:returns: []
"""
key = self.keyFromQuery(origQuery)
if key in self._cache.keys(): #We have it cached
return (self._cache[key])
#We dont have it cached - try to load it from DB
try:
index = db.Get(db.Key.from_path(self._dbType, key))
res = json.loads(index["data"])
self._cache[key] = res
return (res)
except db.EntityNotFoundError: #Its not in the datastore, too
pass
#We dont have this index yet.. Build it
#Clone the original Query
queryRes = origQuery.clone(keysOnly=True).datastoreQuery.Run(
limit=self.maxPages * self.pageSize)
#Build-Up the index
res = []
i = 0
previousCursor = None #The first page dosnt have any cursor
for discardedKey in queryRes:
if i % self.pageSize == 0:
res.append(previousCursor)
if i % self.pageSize == (self.pageSize - 1) and i > 1:
previousCursor = str(queryRes.cursor().urlsafe())
i += 1
if len(res) == 0: # Ensure that the first page exists
res.append(None)
entry = db.Entity(self._dbType, name=key)
entry["data"] = json.dumps(res)
entry["creationdate"] = datetime.now()
db.Put(entry)
return (res)
def txnDelete(key, skel):
dbObj = db.Get(db.Key(
key)) # Fetch the raw object as we might have to clear locks
for boneName, bone in skel.items():
# Ensure that we delete any value-lock objects remaining for this entry
if bone.unique:
try:
logging.error("x1")
logging.error(dbObj.keys())
if "%s.uniqueIndexValue" % boneName in dbObj.keys():
logging.error("x2")
db.Delete(
db.Key.from_path(
"%s_%s_uniquePropertyIndex" %
(skel.kindName, boneName),
dbObj["%s.uniqueIndexValue" % boneName]))
except db.EntityNotFoundError:
raise
pass
# Delete the blob-key lock object
try:
lockObj = db.Get(db.Key.from_path("viur-blob-locks", str(key)))
except:
lockObj = None
if lockObj is not None:
if lockObj["old_blob_references"] is None and lockObj[
"active_blob_references"] is None:
db.Delete(lockObj) # Nothing to do here
else:
if lockObj["old_blob_references"] is None:
# No old stale entries, move active_blob_references -> old_blob_references
lockObj["old_blob_references"] = lockObj[
"active_blob_references"]
elif lockObj["active_blob_references"] is not None:
# Append the current references to the list of old & stale references
lockObj["old_blob_references"] += lockObj[
"active_blob_references"]
lockObj["active_blob_references"] = [
] # There are no active ones left
lockObj["is_stale"] = True
lockObj["has_old_blob_references"] = True
db.Put(lockObj)
db.Delete(db.Key(key))
def transaction():
obj = db.Get(key)
if check:
assert isinstance(check,
dict), "'check' has to be a dict, you diggi!"
assert all([obj[bone] == value for bone, value in check.items()])
if values:
for bone, value in values.items():
if bone[0] == "+":
obj[bone[1:]] += value
elif bone[0] == "-":
obj[bone[1:]] -= value
else:
obj[bone] = value
if func and callable(func):
func(obj)
db.Put(obj)
def markFileForDeletion(dlkey):
"""
Adds a marker to the data store that the file specified as *dlkey* can be deleted.
Once the mark has been set, the data store is checked four times (default: every 4 hours)
if the file is in use somewhere. If it is still in use, the mark goes away, otherwise
the mark and the file are removed from the datastore. These delayed checks are necessary
due to database inconsistency.
:type dlkey: str
:param dlkey: Unique download-key of the file that shall be marked for deletion.
"""
fileObj = db.Query("viur-deleted-files").filter("dlkey", dlkey).get()
if fileObj: #Its allready marked
return
fileObj = db.Entity("viur-deleted-files")
fileObj["itercount"] = 0
fileObj["dlkey"] = str(dlkey)
db.Put(fileObj)
def postSavedHandler(self, valuesCache, boneName, skel, key, dbfields):
if not valuesCache[boneName]:
values = []
elif isinstance(valuesCache[boneName], dict):
values = [dict((k, v) for k, v in valuesCache[boneName].items())]
else:
values = [
dict((k, v) for k, v in x.items())
for x in valuesCache[boneName]
]
parentValues = {}
for parentKey in dbfields.keys():
if parentKey in self.parentKeys or any(
[parentKey.startswith(x + ".") for x in self.parentKeys]):
parentValues[parentKey] = dbfields[parentKey]
dbVals = db.Query("viur-relations").ancestor(
db.Key(key)) #skel.kindName+"_"+self.kind+"_"+key
dbVals.filter("viur_src_kind =", skel.kindName)
dbVals.filter("viur_dest_kind =", self.kind)
dbVals.filter("viur_src_property =", boneName)
for dbObj in dbVals.iter():
try:
if not dbObj["dest.key"] in [x["dest"]["key"] for x in values
]: #Relation has been removed
db.Delete(dbObj.key())
continue
except: #This entry is corrupt
db.Delete(dbObj.key())
else: # Relation: Updated
data = [
x for x in values if x["dest"]["key"] == dbObj["dest.key"]
][0]
if self.indexed: #We dont store more than key and kinds, and these dont change
#Write our (updated) values in
refSkel = self._refSkelCache
refSkel.setValuesCache(data["dest"])
for k, v in refSkel.serialize().items():
dbObj["dest." + k] = v
for k, v in parentValues.items():
dbObj["src." + k] = v
if self.using is not None:
usingSkel = self._usingSkelCache
usingSkel.setValuesCache(data["rel"])
for k, v in usingSkel.serialize().items():
dbObj["rel." + k] = v
dbObj["viur_delayed_update_tag"] = time()
db.Put(dbObj)
values.remove(data)
# Add any new Relation
for val in values:
dbObj = db.Entity(
"viur-relations",
parent=db.Key(key)) #skel.kindName+"_"+self.kind+"_"+key
if not self.indexed: #Dont store more than key and kinds, as they aren't used anyway
dbObj["dest.key"] = val["dest"]["key"]
dbObj["src.key"] = key
else:
refSkel = self._refSkelCache
refSkel.setValuesCache(val["dest"])
for k, v in refSkel.serialize().items():
dbObj["dest." + k] = v
for k, v in parentValues.items():
dbObj["src." + k] = v
if self.using is not None:
usingSkel = self._usingSkelCache
usingSkel.setValuesCache(val["rel"])
for k, v in usingSkel.serialize().items():
dbObj["rel." + k] = v
dbObj["viur_delayed_update_tag"] = time()
dbObj[
"viur_src_kind"] = skel.kindName #The kind of the entry referencing
#dbObj[ "viur_src_key" ] = str( key ) #The key of the entry referencing
dbObj[
"viur_src_property"] = boneName #The key of the bone referencing
#dbObj[ "viur_dest_key" ] = val["key"]
dbObj["viur_dest_kind"] = self.kind
db.Put(dbObj)
def markFileRefsStrong():
"""Adds the weak=False marker to all Filerefs without such an marker"""
for file in db.Query("file").iter():
if not "weak" in file.keys():
file["weak"] = False
db.Put(file)
def setTokenTxn(key, token):
order = db.Get(key)
if not order:
return
order["paypal_token"] = urllib.unquote(token)
db.Put(order)
def txnUpdate(key, mergeFrom, clearUpdateTag):
blobList = set()
skel = type(mergeFrom)()
# Load the current values from Datastore or create a new, empty db.Entity
if not key:
dbObj = db.Entity(skel.kindName)
oldBlobLockObj = None
else:
k = db.Key(key)
assert k.kind() == skel.kindName, "Cannot write to invalid kind!"
try:
dbObj = db.Get(k)
except db.EntityNotFoundError:
dbObj = db.Entity(k.kind(), id=k.id(), name=k.name(), parent=k.parent())
else:
skel.setValues(dbObj)
try:
oldBlobLockObj = db.Get(db.Key.from_path("viur-blob-locks", str(k)))
except:
oldBlobLockObj = None
# Remember old hashes for bones that must have an unique value
oldUniqeValues = {}
for boneName, boneInstance in skel.items():
if boneInstance.unique:
if "%s.uniqueIndexValue" % boneName in dbObj:
oldUniqeValues[boneName] = dbObj["%s.uniqueIndexValue" % boneName]
## Merge the values from mergeFrom in
for key, bone in skel.items():
if key in mergeFrom:
bone.mergeFrom(skel.valuesCache, key, mergeFrom)
for key, _bone in skel.items():
dbObj = _bone.serialize(skel.valuesCache, key, dbObj)
blobList.update(_bone.getReferencedBlobs(self.valuesCache, key))
if clearUpdateTag:
dbObj["viur_delayed_update_tag"] = 0 # Mark this entity as Up-to-date.
else:
dbObj[
"viur_delayed_update_tag"] = time() # Mark this entity as dirty, so the background-task will catch it up and update its references.
dbObj = skel.preProcessSerializedData(dbObj)
try:
ourKey = str(dbObj.key())
except: # Its not an update but an insert, no key yet
ourKey = None
# Lock hashes from bones that must have unique values
newUniqeValues = {}
for boneName, boneInstance in skel.items():
if boneInstance.unique:
# Check if the property is really unique
newUniqeValues[boneName] = boneInstance.getUniquePropertyIndexValue(
self.valuesCache, boneName)
if newUniqeValues[boneName] is not None:
try:
lockObj = db.Get(db.Key.from_path(
"%s_%s_uniquePropertyIndex" % (skel.kindName, boneName),
newUniqeValues[boneName]))
if lockObj["references"] != ourKey:
# This value has been claimed, and that not by us
raise ValueError(
"The unique value '%s' of bone '%s' has been recently claimed!" %
(self.valuesCache[boneName], boneName))
except db.EntityNotFoundError: # No lockObj found for that value, we can use that
pass
dbObj["%s.uniqueIndexValue" % boneName] = newUniqeValues[boneName]
else:
if "%s.uniqueIndexValue" % boneName in dbObj:
del dbObj["%s.uniqueIndexValue" % boneName]
if not skel.searchIndex:
# We generate the searchindex using the full skel, not this (maybe incomplete one)
tags = []
for key, _bone in skel.items():
if _bone.searchable:
tags += [tag for tag in _bone.getSearchTags(self.valuesCache, key) if
(tag not in tags and len(tag) < 400)]
dbObj["viur_tags"] = tags
db.Put(dbObj) # Write the core entry back
# Now write the blob-lock object
blobList = skel.preProcessBlobLocks(blobList)
if blobList is None:
raise ValueError(
"Did you forget to return the bloblist somewhere inside getReferencedBlobs()?")
if None in blobList:
raise ValueError("None is not a valid blobKey.")
if oldBlobLockObj is not None:
oldBlobs = set(oldBlobLockObj["active_blob_references"] if oldBlobLockObj[
"active_blob_references"] is not None else [])
removedBlobs = oldBlobs - blobList
oldBlobLockObj["active_blob_references"] = list(blobList)
if oldBlobLockObj["old_blob_references"] is None:
oldBlobLockObj["old_blob_references"] = [x for x in removedBlobs]
else:
tmp = set(oldBlobLockObj["old_blob_references"] + [x for x in removedBlobs])
oldBlobLockObj["old_blob_references"] = [x for x in (tmp - blobList)]
oldBlobLockObj["has_old_blob_references"] = oldBlobLockObj[
"old_blob_references"] is not None and len(
oldBlobLockObj["old_blob_references"]) > 0
oldBlobLockObj["is_stale"] = False
db.Put(oldBlobLockObj)
else: # We need to create a new blob-lock-object
blobLockObj = db.Entity("viur-blob-locks", name=str(dbObj.key()))
blobLockObj["active_blob_references"] = list(blobList)
blobLockObj["old_blob_references"] = []
blobLockObj["has_old_blob_references"] = False
blobLockObj["is_stale"] = False
db.Put(blobLockObj)
for boneName, boneInstance in skel.items():
if boneInstance.unique:
# Update/create/delete missing lock-objects
if boneName in oldUniqeValues and oldUniqeValues[boneName] != \
newUniqeValues[boneName]:
# We had an old lock and its value changed
try:
# Try to delete the old lock
oldLockObj = db.Get(db.Key.from_path(
"%s_%s_uniquePropertyIndex" % (skel.kindName, boneName),
oldUniqeValues[boneName]))
if oldLockObj["references"] != ourKey:
# We've been supposed to have that lock - but we don't.
# Don't remove that lock as it now belongs to a different entry
logging.critical(
"Detected Database corruption! A Value-Lock had been reassigned!")
else:
# It's our lock which we don't need anymore
db.Delete(db.Key.from_path(
"%s_%s_uniquePropertyIndex" % (
skel.kindName, boneName),
oldUniqeValues[boneName]))
except db.EntityNotFoundError as e:
logging.critical(
"Detected Database corruption! Could not delete stale lock-object!")
if newUniqeValues[boneName] is not None:
# Lock the new value
newLockObj = db.Entity(
"%s_%s_uniquePropertyIndex" % (skel.kindName, boneName),
name=newUniqeValues[boneName])
newLockObj["references"] = str(dbObj.key())
db.Put(newLockObj)
return (str(dbObj.key()), dbObj, skel)
def iterImport(module, target, exportKey, cursor=None, amount=0):
"""
Processes 100 Entries and calls the next batch
"""
urlfetch.set_default_fetch_deadline(20)
payload = {"module": module, "key": exportKey}
if cursor:
payload.update({"cursor": cursor})
result = urlfetch.fetch(
url=target,
payload=urllib.urlencode(payload),
method=urlfetch.POST,
headers={'Content-Type': 'application/x-www-form-urlencoded'})
if result.status_code == 200:
res = pickle.loads(result.content.decode("HEX"))
skel = skeletonByKind(module)()
logging.info("%s: %d new entries fetched, total %d entries fetched" %
(module, len(res["values"]), amount))
if len(res["values"]) == 0:
try:
utils.sendEMailToAdmins(
"Import of kind %s finished with %d entities" %
(module, amount), "ViUR finished to import %d entities of "
"kind %s from %s.\n" % (amount, module, target))
except: #OverQuota, whatever
logging.error("Unable to send Email")
return
for entry in res["values"]:
for k in list(entry.keys())[:]:
if isinstance(entry[k], str):
entry[k] = entry[k].decode("UTF-8")
if not "key" in entry.keys():
entry["key"] = entry["id"]
key = db.Key(encoded=utils.normalizeKey(entry["key"]))
# Special case: Convert old module root nodes!!!
if module.endswith(
"_rootNode") and key.name() and "_modul_" in key.name():
name = key.name().replace("_modul_", "_module_")
else:
name = key.name()
dbEntry = db.Entity(kind=key.kind(),
parent=key.parent(),
id=key.id(),
name=name)
for k in entry.keys():
if k == "key":
continue
dbEntry[k] = entry[k]
# Special case: Convert old module root nodes!!!
if (isinstance(skel, (HierarchySkel, TreeLeafSkel))
and k in ["parentdir", "parententry", "parentrepo"]
and entry[k]):
key = db.Key(encoded=str(entry[k]))
if key.parent():
parent = db.Key(
encoded=utils.normalizeKey(key.parent()))
else:
parent = None
if key.id_or_name() and "_modul_" in str(key.id_or_name()):
name = key.id_or_name().replace("_modul_", "_module_")
else:
name = key.id_or_name()
dbEntry[k] = str(
db.Key.from_path(key.kind(), name, parent=parent))
db.Put(dbEntry)
skel.fromDB(str(dbEntry.key()))
skel.refresh()
skel.toDB(clearUpdateTag=True)
amount += 1
iterImport(module, target, exportKey, res["cursor"], amount)
def checkout(self, step=None, key=None, skey=None, *args, **kwargs):
"""
Performs the checkout process trough the state machine provided by self.steps.
:param step: The current step index, None for beginning a new checkout
:param key: Key of the current checkout
:param skey: Server security key
:return: Returns the rendered template or throws redirection exceptions.
"""
myKindName = self.viewSkel().kindName
if step is None:
logging.info("Starting new checkout process")
billObj = db.Entity(myKindName)
billObj["idx"] = "0000000"
for state in self.states:
billObj["state_%s" % state] = "0"
db.Put(billObj)
key = str(billObj.key())
#Copy the Cart
if "amountSkel" in dir(self):
cart = session.current.get("cart_products") or {}
s = self.amountSkel()
products = []
for prod, atts in cart.items():
for i in range(0, atts["amount"]):
products.append(str(prod))
s.fromClient({"product": products})
s.toDB()
session.current["order_" + myKindName] = {
"key": str(key),
"completedSteps": []
}
session.current.markChanged()
raise errors.Redirect("?step=0&key=%s" % str(key))
elif key:
try:
orderKey = db.Key(key)
step = int(step)
assert (step >= 0)
assert (step < len(self.steps))
except:
raise errors.NotAcceptable()
sessionInfo = session.current.get("order_" + myKindName)
if not sessionInfo or not sessionInfo.get("key") == str(orderKey):
raise errors.Unauthorized()
if step in sessionInfo["completedSteps"]:
session.current["order_" + myKindName]["completedSteps"] = [
x for x in sessionInfo["completedSteps"] if x < step
]
session.current.markChanged()
#Make sure that no steps can be skipped
if step != 0 and not step - 1 in sessionInfo["completedSteps"]:
raise errors.Redirect("?step=0&key=%s" % str(str(orderKey)))
currentStep = self.steps[step]
if "preHandler" in currentStep.keys():
try:
if isinstance(currentStep["preHandler"], list):
for handler in currentStep["preHandler"]:
handler(self, step, str(orderKey), *args, **kwargs)
else:
currentStep["preHandler"](self,
step,
str(orderKey),
refkwargs=kwargs,
*args,
**kwargs)
except SkipStepException:
session.current["order_" +
myKindName]["completedSteps"].append(step)
session.current.markChanged()
raise errors.Redirect("?step=%s&key=%s" %
(str(step + 1), str(orderKey)))
except ReturnHtmlException as e:
return (e.html)
if "requiresSecurityKey" in currentStep and currentStep[
"requiresSecurityKey"]:
if not securitykey.validate(skey):
raise errors.PreconditionFailed()
pass
if "mainHandler" in currentStep:
if currentStep["mainHandler"]["action"] == "edit":
skel = self.getSkelByName(
currentStep["mainHandler"]["skeleton"], str(orderKey))
skel.fromDB(str(orderKey))
if not len(kwargs.keys()) or not skel.fromClient(kwargs):
return (self.render.edit(
skel,
tpl=currentStep["mainHandler"]["template"],
step=step))
skel.toDB()
if currentStep["mainHandler"]["action"] == "view":
if not "complete" in kwargs or not kwargs[
"complete"] == u"1":
skel = self.getSkelByName(
currentStep["mainHandler"]["skeleton"],
str(orderKey))
skel.fromDB(str(orderKey))
return (self.render.view(
skel,
tpl=currentStep["mainHandler"]["template"],
step=step))
elif currentStep["mainHandler"]["action"] == "function":
res = currentStep["mainHandler"]["function"](self, step,
str(orderKey),
*args,
**kwargs)
if res:
return res
if "postHandler" in currentStep:
currentStep["postHandler"](self, step, str(orderKey), *args,
**kwargs)
session.current["order_" +
myKindName]["completedSteps"].append(step)
session.current.markChanged()
logging.info("next ?step=%s&key=%s" %
(str(step + 1), str(orderKey)))
raise errors.Redirect("?step=%s&key=%s" %
(str(step + 1), str(orderKey)))
def fixTxn(nodeKey, newRepoKey):
node = db.Get(nodeKey)
node["parentrepo"] = newRepoKey
db.Put(node)
