def reparent(self, item, dest, skey, *args, **kwargs):
"""
Moves an entry *item* (and everything beneath it) to another parent-node *dest*.
.. seealso:: :func:`canReparent`
:param item: URL-safe key of the item which will be moved.
:type item: str
:param dest: URL-safe key of the new parent for this item.
:type dest: str
:returns: A rendered success result generated by the default renderer.
:raises: :exc:`server.errors.NotFound`, when no entry with the given *id* was found.
:raises: :exc:`server.errors.Unauthorized`, if the current user does not have the required permissions.
:raises: :exc:`server.errors.PreconditionFailed`, if the *skey* could not be verified.
"""
if not securitykey.validate(skey, useSessionKey=True):
raise errors.PreconditionFailed()
if not self.canReparent(item, dest):
raise errors.Unauthorized()
if not self.isValidParent(dest) or item == dest:
raise errors.NotAcceptable()
## Test for recursion
isValid = False
currLevel = db.Get(dest)
for x in range(0, 99):
if str(currLevel.key()) == item:
break
if currLevel.key().kind(
) == self.viewSkel().kindName + "_rootNode":
# We reached a rootNode
isValid = True
break
currLevel = db.Get(currLevel["parententry"])
if not isValid:
raise errors.NotAcceptable()
## Update entry
fromItem = db.Get(item)
fromItem["parententry"] = dest
fromItem["parentrepo"] = str(self.getRootNode(dest).key())
db.Put(fromItem)
skel = self.editSkel()
assert skel.fromDB(item)
self.onItemReparent(skel)
self.onItemChanged(skel)
return self.render.reparentSuccess(obj=fromItem)
def authenticateUser(self, userKey, **kwargs):
"""
Performs Log-In for the current session and the given userKey.
This resets the current session: All fields not explicitly marked as persistent
by conf["viur.session.persistentFieldsOnLogin"] are gone afterwards.
:param authProvider: Which authentication-provider issued the authenticateUser request
:type authProvider: object
:param userKey: The (DB-)Key of the user we shall authenticate
:type userKey: db.Key
"""
currSess = currentSession.get()
res = db.Get(userKey)
assert res, "Unable to authenticate unknown user %s" % userKey
oldSession = {k: v
for k, v in currSess.items()
} # Store all items in the current session
currSess.reset()
# Copy the persistent fields over
for k in conf["viur.session.persistentFieldsOnLogin"]:
if k in oldSession:
currSess[k] = oldSession[k]
del oldSession
currSess["user"] = res
currSess.markChanged()
currentRequest.get().response.headers[
"Sec-X-ViUR-StaticSKey"] = currSess.staticSecurityKey
self.onLogin()
return self.render.loginSucceeded(**kwargs)
def validate(key: str, useSessionKey: bool) -> Union[bool, db.Entity]:
"""
Validates a security key. If useSessionKey is true, the key is expected to be the sessions current security key
(or it's static security key). Otherwise it must be a key created with a duration (so it's not session
dependent)
:param key: The key to validate
:param useSessionKey: If True, we validate against the session's skey, otherwise we'll lookup an unbound key
:returns: False if the key was not valid for whatever reasons, the data (given during createSecurityKey) as
dictionary or True if the dict is empty (or :param:useSessionKey was true).
"""
if useSessionKey:
if key == "staticSessionKey":
skeyHeaderValue = currentRequest.get().request.headers.get(
"Sec-X-ViUR-StaticSKey")
if skeyHeaderValue and currentSession.get(
).validateStaticSecurityKey(skeyHeaderValue):
return True
elif currentSession.get().validateSecurityKey(key):
return True
return False
if not key:
return False
dbKey = db.Key(securityKeyKindName, key)
dbObj = db.Get(dbKey)
if dbObj:
db.Delete(dbKey)
until = dbObj["until"]
if until < utcNow(): # This key has expired
return False
del dbObj["until"]
if not dbObj:
return True
return dbObj
return False
def getEntry(self, module, id, key=None):
if not self._checkKey(key, export=True):
raise errors.Forbidden()
res = db.Get(id)
return pickle.dumps(self.genDict(res))
def sofortStatus(self, *args, **kwargs):
sortOrder = [
"transaction", "user_id", "project_id", "sender_holder",
"sender_account_number", "sender_bank_code", "sender_bank_name",
"sender_bank_bic", "sender_iban", "sender_country_id",
"recipient_holder", "recipient_account_number",
"recipient_bank_code", "recipient_bank_name", "recipient_bank_bic",
"recipient_iban", "recipient_country_id",
"international_transaction", "amount", "currency_id", "reason_1",
"reason_2", "security_criteria", "user_variable_0",
"user_variable_1", "user_variable_2", "user_variable_3",
"user_variable_4", "user_variable_5", "created"
]
hashstr = "|".join([kwargs[key] for key in sortOrder] +
[conf["sofort"]["notificationpassword"]])
if hashlib.sha512(
hashstr.encode("utf-8")).hexdigest() != kwargs["hash"]:
logging.error(
"RECEIVED INVALID HASH FOR sofort (%s!=%s)" % (hashlib.sha512(
hashstr.encode("utf-8")).hexdigest(), kwargs["hash"]))
return ("INVALID HASH")
order = db.Get(db.Key(kwargs["user_variable_0"]))
if not order:
logging.error("RECEIVED UNKNOWN ORDER by sofort (%s)" %
(kwargs["user_variable_0"]))
return ("UNKNOWN ORDER")
if ("%.2f" % order["price"]) != kwargs["amount"]:
logging.error("RECEIVED INVALID AMOUNT PAYED sofort (%s!=%s)" %
(order["price"], kwargs["amount"]))
return ("INVALID AMOUNT")
self.orderHandler.setPayed(kwargs["user_variable_0"])
return ("OKAY")
def updateTransaction(userKey, idx):
user = db.Get(userKey)
if not "otptimedrift" in user or not isinstance(
user["otptimedrift"], float):
user["otptimedrift"] = 0.0
user["otptimedrift"] += min(max(0.1 * idx, -0.3), 0.3)
db.Put(user)
def assignBillSequence(self, orderKey):
"""
Assigns an unique order-order to the given order.
"""
def getKeyTxn(kindName, orderKey):
"""Generates and returns a new, unique Key"""
seqObj = db.GetOrInsert(kindName,
"viur_bill_sequences",
count=1000)
idx = seqObj["count"]
seqObj["count"] += 1
db.Put(seqObj)
return str(idx)
def setKeyTxn(orderKey, idx):
"""Assigns the new order to the given order"""
dbObj = db.Get(db.Key(orderKey))
if not dbObj:
return
dbObj["idx"] = idx
db.Put(dbObj)
dbObj = db.Get(db.Key(orderKey))
if not dbObj:
return
idx = db.RunInTransaction(getKeyTxn,
self.viewSkel().kindName, orderKey)
db.RunInTransaction(setKeyTxn, orderKey, idx)
self.billSequenceAvailable(orderKey)
def setKeyTxn(orderKey, idx):
"""Assigns the new order to the given order"""
dbObj = db.Get(db.Key(orderKey))
if not dbObj:
return
dbObj["idx"] = idx
db.Put(dbObj)
def setIndex(self, item, index, skey, *args, **kwargs):
"""
Changes the order of the elements in the current level by changing the index of *item*.
.. seealso:: :func:`canSetIndex`
:param item: URL-safe key of the item which index should be changed.
:type item: str
:param index: New index for this item. This value must be cast-able to float.
:type index: str
:returns: A rendered success result generated by the default renderer.
:raises: :exc:`server.errors.NotFound`, when no entry with the given *key* was found.
:raises: :exc:`server.errors.Unauthorized`, if the current user does not have the required permissions.
:raises: :exc:`server.errors.PreconditionFailed`, if the *skey* could not be verified.
"""
if not securitykey.validate(skey, useSessionKey=True):
raise errors.PreconditionFailed()
if not self.canSetIndex(item, index):
raise errors.Unauthorized()
fromItem = db.Get(item)
fromItem["sortindex"] = float(index)
db.Put(fromItem)
skel = self.editSkel()
assert skel.fromDB(item)
self.onItemSetIndex(skel)
self.onItemChanged(skel)
return self.render.setIndexSuccess(obj=fromItem)
def validate(key: str, useSessionKey: bool) -> Union[bool, db.Entity]:
"""
Validates a onetime securitykey
:type key: str
:param key: The key to validate
:type useSessionKey: Bool
:param useSessionKey: If True, we validate against the session's skey, otherwise we'll lookup an unbound key
:returns: False if the key was not valid for whatever reasons, the data (given during createSecurityKey) as dictionary or True if the dict is empty.
"""
if useSessionKey:
if key == "staticSessionKey":
skeyHeaderValue = request.current.get().request.headers.get(
"Sec-X-ViUR-StaticSKey")
if skeyHeaderValue and currentSession.validateStaticSecurityKey(
skeyHeaderValue):
return True
elif currentSession.validateSecurityKey(key):
return True
return False
if not key or "/" in key:
return False
dbObj = db.Get((securityKeyKindName, key))
if dbObj:
db.Delete((securityKeyKindName, key))
if dbObj["until"] < datetime.now(): # This key has expired
return False
del dbObj["until"]
if not dbObj:
return True
return dbObj
return False
def load(self, req):
"""
Initializes the Session.
If the client supplied a valid Cookie, the session is read
from the memcache/datastore, otherwise a new, empty session
will be initialized.
"""
self.changed = False
self.isInitial = False
self.cookieKey = None
self.sslKey = None
self.staticSecurityKey = None
self.securityKey = None
self.session = {}
if self.cookieName in req.request.cookies:
cookie = str(req.request.cookies[self.cookieName])
data = db.Get(db.Key(self.kindName, cookie))
if data: # Loaded successfully from Memcache
if data["lastseen"] < time() - conf["viur.session.lifeTime"]:
# This session is too old
self.reset()
return False
self.session = data["data"]
self.staticSecurityKey = data["staticSecurityKey"]
self.securityKey = data["securityKey"]
self.cookieKey = cookie
if data["lastseen"] < time() - 5 * 60: # Refresh every 5 Minutes
self.changed = True
else:
# We could not load from firebase; create a new one
self.reset()
else:
self.reset()
def getRootNode(self, entryKey: db.Key) -> Skeleton:
"""
Returns the root-node for a given child.
:param entryKey: URL-Safe key of the child entry
:type entryKey: str
:returns: The entity of the root-node.
:rtype: :class:`server.db.Entity`
"""
rootNodeSkel = self.nodeSkelCls()
entryKey = db.keyHelper(entryKey, rootNodeSkel.kindName)
repo = db.Get(entryKey)
while repo and "parententry" in repo:
repo = db.Get(repo["parententry"])
rootNodeSkel.fromDB(repo.key)
return rootNodeSkel
def txn(orderKey, state, removeState):
dbObj = db.Get(db.Key(orderKey))
if not dbObj:
return
dbObj["state_%s" % state] = "1" if not removeState else "0"
dbObj["changedate"] = datetime.now()
db.Put(dbObj)
def getRootNode(self, entryKey):
"""
Returns the root-node for a given child.
:param entryKey: URL-Safe key of the child entry
:type entryKey: str
:returns: The entity of the root-node.
:rtype: :class:`server.db.Entity`
"""
repo = db.Get(entryKey)
while repo and "parententry" in repo:
repo = db.Get(repo["parententry"])
assert repo and repo.key().kind(
) == self.viewSkel().kindName + "_rootNode"
return repo
def exchangeSecurityKey(): dbSession = db.Get(db.Key(self.kindName, self.cookieKey)) if not dbSession: # Should not happen (except if session.reset has been called in the same request) return False if dbSession["securityKey"] != key: # Race-Condidtion: That skey has been used in another instance return False dbSession["securityKey"] = utils.generateRandomString(13) db.Put(dbSession) return dbSession["securityKey"]
def updateTxn(cacheKey):
key = db.Key(self.rateLimitKind, cacheKey)
obj = db.Get(key)
if obj is None:
obj = db.Entity(key)
obj["value"] = 0
obj["value"] += 1
obj["expires"] = utils.utcNow() + timedelta(minutes=2 *
self.minutes)
db.Put(obj)
def getOldBlobKeysTxn(dbKey):
obj = db.Get(dbKey)
res = obj["old_blob_references"] or []
if obj["is_stale"]:
db.Delete(dbKey)
else:
obj["has_old_blob_references"] = False
obj["old_blob_references"] = []
db.Put(obj)
return res
def getSofortURL(self, orderID):
order = db.Get(db.Key(orderID))
hashstr = "%s|%s|||||%.2f|EUR|%s||%s||||||%s" % (
conf["sofort"]["userid"], conf["sofort"]["projectid"],
float(order["price"]), str(order.key()), str(
order.key()), conf["sofort"]["projectpassword"])
hash = hashlib.sha512(hashstr.encode("UTF-8")).hexdigest()
returnURL = "https://www.sofortueberweisung.de/payment/start?user_id=%s&project_id=%s&amount=%.2f¤cy_id=EUR&reason_1=%s&user_variable_0=%s&hash=%s" % (
conf["sofort"]["userid"], conf["sofort"]["projectid"],
float(order["price"]), str(order.key()), str(order.key()), hash)
return (returnURL)
def updateTxn(key, resDict):
obj = db.Get(key)
if not obj: # File-object got deleted during building of our derives
return
obj["derived"] = obj.get("derived") or {}
obj["derived"]["deriveStatus"] = obj["derived"].get("deriveStatus") or {}
obj["derived"]["files"] = obj["derived"].get("files") or {}
for k, v in resDict.items():
obj["derived"]["deriveStatus"][k] = v["version"]
for fileName, fileDict in v["files"].items():
obj["derived"]["files"][fileName] = fileDict
db.Put(obj)
def importBlobFromViur2(dlKey, fileName):
if not conf.get("viur.viur2import.blobsource"):
return False
existingImport = db.Get(db.Key("viur-viur2-blobimport", dlKey))
if existingImport:
if existingImport["success"]:
return existingImport["dlurl"]
return False
if conf["viur.viur2import.blobsource"]["infoURL"]:
try:
importDataReq = urlopen(
conf["viur.viur2import.blobsource"]["infoURL"] + dlKey)
except:
marker = db.Entity(db.Key("viur-viur2-blobimport", dlKey))
marker["success"] = False
marker["error"] = "Failed URL-FETCH 1"
db.Put(marker)
return False
if importDataReq.status != 200:
marker = db.Entity(db.Key("viur-viur2-blobimport", dlKey))
marker["success"] = False
marker["error"] = "Failed URL-FETCH 2"
db.Put(marker)
return False
importData = json.loads(importDataReq.read())
oldBlobName = conf["viur.viur2import.blobsource"][
"gsdir"] + "/" + importData["key"]
srcBlob = storage.Blob(
bucket=bucket,
name=conf["viur.viur2import.blobsource"]["gsdir"] + "/" +
importData["key"])
else:
oldBlobName = conf["viur.viur2import.blobsource"]["gsdir"] + "/" + dlKey
srcBlob = storage.Blob(
bucket=bucket,
name=conf["viur.viur2import.blobsource"]["gsdir"] + "/" + dlKey)
if not srcBlob.exists():
marker = db.Entity(db.Key("viur-viur2-blobimport", dlKey))
marker["success"] = False
marker["error"] = "Local SRC-Blob missing"
marker["oldBlobName"] = oldBlobName
db.Put(marker)
return False
bucket.rename_blob(srcBlob, "%s/source/%s" % (dlKey, fileName))
marker = db.Entity(db.Key("viur-viur2-blobimport", dlKey))
marker["success"] = True
marker["old_src_key"] = dlKey
marker["old_src_name"] = fileName
marker["dlurl"] = utils.downloadUrlFor(dlKey, fileName, False, None)
db.Put(marker)
return marker["dlurl"]
def startProcessing(self, userKey):
user = db.Get(userKey)
if all([(x in user and user[x]) for x in ["otpid", "otpkey"]]):
logging.info("OTP wanted for user")
currentSession.get()["_otp_user"] = {"uid": str(userKey),
"otpid": user["otpid"],
"otpkey": user["otpkey"],
"otptimedrift": user["otptimedrift"],
"timestamp": time(),
"failures": 0}
currentSession.get().markChanged()
return self.userModule.render.loginSucceeded(msg="X-VIUR-2FACTOR-TimeBasedOTP")
return None
def hasblob(self, blobkey, key):
if not self._checkKey(key, export=False):
raise errors.Forbidden()
try:
oldKeyHash = sha256(blobkey).hexdigest().encode("hex")
res = db.Get(db.Key.from_path("viur-blobimportmap", oldKeyHash))
if res:
if "available" in res:
return json.dumps(res["available"])
else:
return json.dumps(True)
except:
pass
return json.dumps(False)
def updateInplace(relDict):
"""
Fetches the entity referenced by valDict["dest.key"] and updates all dest.* keys
accordingly
"""
if not (isinstance(relDict, dict) and "dest" in relDict):
logging.error("Invalid dictionary in updateInplace: %s" % relDict)
return
newValues = db.Get(db.keyHelper(relDict["dest"]["key"], self.kind))
if newValues is None:
logging.info("The key %s does not exist" % relDict["dest"]["key"])
return
for boneName in self.refKeys:
if boneName != "key" and boneName in newValues:
relDict["dest"].dbEntity[boneName] = newValues[boneName]
def load(self, req):
"""
Initializes the Session.
If the client supplied a valid Cookie, the session is read
from the memcache/datastore, otherwise a new, empty session
will be initialized.
"""
self.changed = False
self.isInitial = False
self.httpKey = None
self.sslKey = None
self.staticSecurityKey = None
self.securityKey = None
self.session = {}
if self.plainCookieName in req.request.cookies:
cookie = str(req.request.cookies[self.plainCookieName])
try:
data = db.Get(db.Key(self.kindName, cookie))
except:
raise # FIXME
return False
if data: # Loaded successfully from Memcache
if data["lastseen"] < time() - conf["viur.session.lifeTime"]:
# This session is too old
self.reset()
return False
self.session = pickle.loads(base64.b64decode(data["data"]))
self.sslKey = data["sslkey"]
self.staticSecurityKey = data["staticSecurityKey"]
self.securityKey = data["securityKey"]
self.httpKey = cookie
if data["lastseen"] < time(
) - 5 * 60: # Refresh every 5 Minutes
self.changed = True
else:
# We could not load from firebase; create a new one
self.reset()
if req.isSSLConnection and self.sslKey and not req.request.cookies.get(
self.sslCookieName) == self.sslKey:
logging.critical(
"Possible session hijack attempt! Session dropped.")
self.reset()
return False
return True
else:
self.reset()
def startProcessing(self, step, orderID):
def setTokenTxn(key, token):
order = db.Get(key)
if not order:
return
order["paypal_token"] = urllib.unquote(token)
db.Put(order)
paypal = PayPal.PayPalHandler()
key = db.Key(orderID)
order = db.Get(key)
if not order:
return
token = paypal.SetExpressCheckout("%.2f" % order["price"])
db.RunInTransaction(setTokenTxn, key, token)
raise (errors.Redirect(paypal.getPayURL(token)))
def calculateOrderSum(self, step, orderKey, *args, **kwargs):
"""
Calculates the final price for this order.
This function *must* be called before any attempt is made to start a payment process.
:param step: Current step within the ordering process
:type step: int
:param orderKey: order to calculate the price for
:type orderKey: str
"""
price = sum([x[3] for x in self.getBillItems(orderKey)])
orderObj = db.Get(db.Key(str(orderKey)))
orderObj["price"] = price
db.Put(orderObj)
def delete(self, skel: 'viur.core.skeleton.SkeletonInstance', name: str):
"""
Ensure any outgoing relational lock is cleared
:param skel:
:param name:
:return:
"""
if skel.dbEntity.get("%s_outgoingRelationalLocks" % name):
for refKey in skel.dbEntity["%s_outgoingRelationalLocks" % name]:
referencedEntry = db.Get(refKey)
if not referencedEntry:
logging.warning("Programming error detected: Entry %s is gone despite lock" % refKey)
continue
incommingLocks = referencedEntry.get("viur_incomming_relational_locks", [])
# We remove any reference to ourself as multiple bones may hold Locks to the same entry
referencedEntry["viur_incomming_relational_locks"] = [x for x in incommingLocks if x != skel["key"]]
db.Put(referencedEntry)
def wrapF(self, *args, **kwargs) -> Union[str, bytes]:
currReq = currentRequest.get()
if conf["viur.disableCache"] or currReq.disableCache:
# Caching disabled
if conf["viur.disableCache"]:
logging.debug("Caching is disabled by config")
return f(self, *args, **kwargs)
# How many arguments are part of the way to the function called (and how many are just *args)
offset = -len(currReq.args) or len(currReq.pathlist)
path = "/" + "/".join(currReq.pathlist[:offset])
if not path in urls:
# This path (possibly a sub-render) should not be cached
logging.debug("Not caching for %s" % path)
return f(self, *args, **kwargs)
key = keyFromArgs(f, userSensitive, languageSensitive, evaluatedArgs,
path, args, kwargs)
if not key:
# Something is wrong (possibly the parameter-count)
# Let's call f, but we knew already that this will clash
return f(self, *args, **kwargs)
dbRes = db.Get(db.Key(viurCacheName, key))
if dbRes is not None:
if not maxCacheTime \
or dbRes["creationtime"] > utils.utcNow() - timedelta(seconds=maxCacheTime):
# We store it unlimited or the cache is fresh enough
logging.debug("This request was served from cache.")
currReq.response.headers['Content-Type'] = dbRes[
"content-type"]
return dbRes["data"]
# If we made it this far, the request wasn't cached or too old; we need to rebuild it
oldAccessLog = db.startDataAccessLog()
try:
res = f(self, *args, **kwargs)
finally:
accessedEntries = db.endDataAccessLog(oldAccessLog)
dbEntity = db.Entity(db.Key(viurCacheName, key))
dbEntity["data"] = res
dbEntity["creationtime"] = utils.utcNow()
dbEntity["path"] = path
dbEntity["content-type"] = currReq.response.headers['Content-Type']
dbEntity["accessedEntries"] = list(accessedEntries)
dbEntity.exclude_from_indexes = ["data", "content-type"
] # We can save 2 DB-Writs :)
db.Put(dbEntity)
logging.debug("This request was a cache-miss. Cache has been updated.")
return res
def wrapF(self, *args, **kwargs):
currentRequest = request.current.get()
if conf["viur.disableCache"] or currentRequest.disableCache:
# Caching disabled
if conf["viur.disableCache"]:
logging.debug("Caching is disabled by config")
return (f(self, *args, **kwargs))
# How many arguments are part of the way to the function called (and how many are just *args)
offset = -len(currentRequest.args) or len(currentRequest.pathlist)
path = "/" + "/".join(currentRequest.pathlist[:offset])
if not path in urls:
# This path (possibly a sub-render) should not be cached
logging.debug("Not caching for %s" % path)
return (f(self, *args, **kwargs))
key = keyFromArgs(f, userSensitive, languageSensitive, evaluatedArgs,
path, args, kwargs)
if not key:
# Someting is wrong (possibly the parameter-count)
# Letz call f, but we knew already that this will clash
return (f(self, *args, **kwargs))
try:
dbRes = db.Get(db.Key.from_path(viurCacheName, key))
except db.EntityNotFoundError:
dbRes = None
if dbRes:
if not maxCacheTime or \
dbRes["creationtime"] > datetime.now() - timedelta(seconds=maxCacheTime):
# We store it unlimited or the cache is fresh enough
logging.debug("This request was served from cache.")
currentRequest.response.headers['Content-Type'] = dbRes[
"content-type"].encode("UTF-8")
return (dbRes["data"])
# If we made it this far, the request wasnt cached or too old; we need to rebuild it
res = f(self, *args, **kwargs)
dbEntity = db.Entity(viurCacheName, name=key)
dbEntity["data"] = res
dbEntity["creationtime"] = datetime.now()
dbEntity["path"] = path
dbEntity["content-type"] = request.current.get(
).response.headers['Content-Type']
dbEntity.set_unindexed_properties(["data", "content-type"
]) # We can save 2 DB-Writs :)
db.Put(dbEntity)
logging.debug("This request was a cache-miss. Cache has been updated.")
return (res)
def getOrBuildIndex(self, origQuery):
"""
Builds a specific index based on origQuery AND local variables (self.indexPage and self.indexMaxPage)
Returns a list of starting-cursors for each page.
You probably shouldn't call this directly. Use cursorForQuery.
:param origQuery: Query to build the index for
:type origQuery: db.Query
:returns: []
"""
key = self.keyFromQuery(origQuery)
if key in self._cache: # We have it cached
return self._cache[key]
# We don't have it cached - try to load it from DB
try:
index = db.Get(db.Key.from_path(self._dbType, key))
res = json.loads(index["data"])
self._cache[key] = res
return res
except db.EntityNotFoundError: # Its not in the datastore, too
pass
# We don't have this index yet.. Build it
# Clone the original Query
queryRes = origQuery.clone(keysOnly=True).datastoreQuery.Run(
limit=self.maxPages * self.pageSize)
# Build-Up the index
res = list()
previousCursor = None # The first page dosnt have any cursor
# enumerate is slightly faster than a manual loop counter
for counter, discardedKey in enumerate(queryRes):
if counter % self.pageSize == 0:
res.append(previousCursor)
if counter % self.pageSize == (self.pageSize - 1):
previousCursor = str(queryRes.cursor().urlsafe())
if not len(res): # Ensure that the first page exists
res.append(None)
entry = db.Entity(self._dbType, name=key)
entry["data"] = json.dumps(res)
entry["creationdate"] = datetime.now()
db.Put(entry)
return res
