def importBlobFromViur2(dlKey, fileName):
if not conf.get("viur.viur2import.blobsource"):
return False
existingImport = db.Get(db.Key("viur-viur2-blobimport", dlKey))
if existingImport:
if existingImport["success"]:
return existingImport["dlurl"]
return False
if conf["viur.viur2import.blobsource"]["infoURL"]:
try:
importDataReq = urlopen(
conf["viur.viur2import.blobsource"]["infoURL"] + dlKey)
except:
marker = db.Entity(db.Key("viur-viur2-blobimport", dlKey))
marker["success"] = False
marker["error"] = "Failed URL-FETCH 1"
db.Put(marker)
return False
if importDataReq.status != 200:
marker = db.Entity(db.Key("viur-viur2-blobimport", dlKey))
marker["success"] = False
marker["error"] = "Failed URL-FETCH 2"
db.Put(marker)
return False
importData = json.loads(importDataReq.read())
oldBlobName = conf["viur.viur2import.blobsource"][
"gsdir"] + "/" + importData["key"]
srcBlob = storage.Blob(
bucket=bucket,
name=conf["viur.viur2import.blobsource"]["gsdir"] + "/" +
importData["key"])
else:
oldBlobName = conf["viur.viur2import.blobsource"]["gsdir"] + "/" + dlKey
srcBlob = storage.Blob(
bucket=bucket,
name=conf["viur.viur2import.blobsource"]["gsdir"] + "/" + dlKey)
if not srcBlob.exists():
marker = db.Entity(db.Key("viur-viur2-blobimport", dlKey))
marker["success"] = False
marker["error"] = "Local SRC-Blob missing"
marker["oldBlobName"] = oldBlobName
db.Put(marker)
return False
bucket.rename_blob(srcBlob, "%s/source/%s" % (dlKey, fileName))
marker = db.Entity(db.Key("viur-viur2-blobimport", dlKey))
marker["success"] = True
marker["old_src_key"] = dlKey
marker["old_src_name"] = fileName
marker["dlurl"] = utils.downloadUrlFor(dlKey, fileName, False, None)
db.Put(marker)
return marker["dlurl"]
def doCleanupDeletedFiles(cursor=None):
maxIterCount = 2 # How often a file will be checked for deletion
query = db.Query("viur-deleted-files")
if cursor:
query.setCursor(cursor)
for file in query.run(100):
if not "dlkey" in file:
db.Delete(file.key)
elif db.Query("viur-blob-locks").filter("active_blob_references =",
file["dlkey"]).getEntry():
logging.info("is referenced, %s" % file["dlkey"])
db.Delete(file.key)
else:
if file["itercount"] > maxIterCount:
logging.info("Finally deleting, %s" % file["dlkey"])
blobs = bucket.list_blobs(prefix="%s/" % file["dlkey"])
for blob in blobs:
blob.delete()
db.Delete(file.key)
# There should be exactly 1 or 0 of these
for f in skeletonByKind("file")().all().filter(
"dlkey =", file["dlkey"]).fetch(99):
f.delete()
else:
logging.debug("Increasing count, %s" % file["dlkey"])
file["itercount"] += 1
db.Put(file)
newCursor = query.getCursor()
if newCursor:
doCleanupDeletedFiles(newCursor)
def updateTransaction(userKey, idx):
user = db.Get(userKey)
if not "otptimedrift" in user or not isinstance(
user["otptimedrift"], float):
user["otptimedrift"] = 0.0
user["otptimedrift"] += min(max(0.1 * idx, -0.3), 0.3)
db.Put(user)
def storeEntry2(self, e, key):
if not self._checkKey(key, export=False):
raise errors.Forbidden()
entry = pickle.loads(e.decode("HEX"))
if not "key" in entry and "id" in entry:
entry["key"] = entry["id"]
for k in list(entry.keys())[:]:
if isinstance(entry[k], str):
entry[k] = entry[k].decode("UTF-8")
key = db.Key(encoded=utils.normalizeKey(entry["key"]))
dbEntry = db.Entity(kind=key.kind(),
parent=key.parent(),
id=key.id(),
name=key.name()) # maybe some more fixes here ?
for k in entry.keys():
if k != "key":
val = entry[k]
dbEntry[k] = val
db.Put(dbEntry)
if dbEntry.key().id():
# Ensure the Datastore knows that it's id is in use
datastore._GetConnection()._reserve_keys([dbEntry.key()])
try:
skel = skeletonByKind(key.kind())()
except:
logging.error("Unknown Skeleton - skipping")
skel.fromDB(str(dbEntry.key()))
skel.refresh()
skel.toDB(clearUpdateTag=True)
def upload(self, oldkey, *args, **kwargs):
res = []
for upload in self.getUploads():
fileName = decodeFileName(upload.filename)
if str(upload.content_type).startswith("image/"):
try:
servingURL = get_serving_url(upload.key())
except:
servingURL = ""
else:
servingURL = ""
res.append({
"name": fileName,
"size": upload.size,
"mimetype": upload.content_type,
"dlkey": str(upload.key()),
"servingurl": servingURL,
"parentdir": "",
"parentrepo": "",
"weak": False
})
oldkey = decodeFileName(oldkey)
oldKeyHash = sha256(oldkey).hexdigest().encode("hex")
e = db.Entity("viur-blobimportmap", name=oldKeyHash)
e["newkey"] = str(upload.key())
e["oldkey"] = oldkey
e["servingurl"] = servingURL
e["available"] = True
db.Put(e)
return (json.dumps({"action": "addSuccess", "values": res}))
def setKeyTxn(orderKey, idx):
"""Assigns the new order to the given order"""
dbObj = db.Get(db.Key(orderKey))
if not dbObj:
return
dbObj["idx"] = idx
db.Put(dbObj)
def doCheckForUnreferencedBlobs(cursor=None):
def getOldBlobKeysTxn(dbKey):
obj = db.Get(dbKey)
res = obj["old_blob_references"] or []
if obj["is_stale"]:
db.Delete(dbKey)
else:
obj["has_old_blob_references"] = False
obj["old_blob_references"] = []
db.Put(obj)
return res
query = db.Query("viur-blob-locks").filter("has_old_blob_references", True).setCursor(cursor)
for lockObj in query.run(100):
oldBlobKeys = db.RunInTransaction(getOldBlobKeysTxn, lockObj.key)
for blobKey in oldBlobKeys:
if db.Query("viur-blob-locks").filter("active_blob_references =", blobKey).getEntry():
# This blob is referenced elsewhere
logging.info("Stale blob is still referenced, %s" % blobKey)
continue
# Add a marker and schedule it for deletion
fileObj = db.Query("viur-deleted-files").filter("dlkey", blobKey).getEntry()
if fileObj: # Its already marked
logging.info("Stale blob already marked for deletion, %s" % blobKey)
return
fileObj = db.Entity(db.Key("viur-deleted-files"))
fileObj["itercount"] = 0
fileObj["dlkey"] = str(blobKey)
logging.info("Stale blob marked dirty, %s" % blobKey)
db.Put(fileObj)
newCursor = query.getCursor()
if newCursor:
doCheckForUnreferencedBlobs(newCursor)
def setIndex(self, item, index, skey, *args, **kwargs):
"""
Changes the order of the elements in the current level by changing the index of *item*.
.. seealso:: :func:`canSetIndex`
:param item: URL-safe key of the item which index should be changed.
:type item: str
:param index: New index for this item. This value must be cast-able to float.
:type index: str
:returns: A rendered success result generated by the default renderer.
:raises: :exc:`server.errors.NotFound`, when no entry with the given *key* was found.
:raises: :exc:`server.errors.Unauthorized`, if the current user does not have the required permissions.
:raises: :exc:`server.errors.PreconditionFailed`, if the *skey* could not be verified.
"""
if not securitykey.validate(skey, useSessionKey=True):
raise errors.PreconditionFailed()
if not self.canSetIndex(item, index):
raise errors.Unauthorized()
fromItem = db.Get(item)
fromItem["sortindex"] = float(index)
db.Put(fromItem)
skel = self.editSkel()
assert skel.fromDB(item)
self.onItemSetIndex(skel)
self.onItemChanged(skel)
return self.render.setIndexSuccess(obj=fromItem)
def txn(orderKey, state, removeState):
dbObj = db.Get(db.Key(orderKey))
if not dbObj:
return
dbObj["state_%s" % state] = "1" if not removeState else "0"
dbObj["changedate"] = datetime.now()
db.Put(dbObj)
def getKeyTxn(kindName, orderKey):
"""Generates and returns a new, unique Key"""
seqObj = db.GetOrInsert(kindName,
"viur_bill_sequences",
count=1000)
idx = seqObj["count"]
seqObj["count"] += 1
db.Put(seqObj)
return str(idx)
def exchangeSecurityKey(): dbSession = db.Get(db.Key(self.kindName, self.cookieKey)) if not dbSession: # Should not happen (except if session.reset has been called in the same request) return False if dbSession["securityKey"] != key: # Race-Condidtion: That skey has been used in another instance return False dbSession["securityKey"] = utils.generateRandomString(13) db.Put(dbSession) return dbSession["securityKey"]
def reparent(self, item, dest, skey, *args, **kwargs):
"""
Moves an entry *item* (and everything beneath it) to another parent-node *dest*.
.. seealso:: :func:`canReparent`
:param item: URL-safe key of the item which will be moved.
:type item: str
:param dest: URL-safe key of the new parent for this item.
:type dest: str
:returns: A rendered success result generated by the default renderer.
:raises: :exc:`server.errors.NotFound`, when no entry with the given *id* was found.
:raises: :exc:`server.errors.Unauthorized`, if the current user does not have the required permissions.
:raises: :exc:`server.errors.PreconditionFailed`, if the *skey* could not be verified.
"""
if not securitykey.validate(skey, useSessionKey=True):
raise errors.PreconditionFailed()
if not self.canReparent(item, dest):
raise errors.Unauthorized()
if not self.isValidParent(dest) or item == dest:
raise errors.NotAcceptable()
## Test for recursion
isValid = False
currLevel = db.Get(dest)
for x in range(0, 99):
if str(currLevel.key()) == item:
break
if currLevel.key().kind(
) == self.viewSkel().kindName + "_rootNode":
# We reached a rootNode
isValid = True
break
currLevel = db.Get(currLevel["parententry"])
if not isValid:
raise errors.NotAcceptable()
## Update entry
fromItem = db.Get(item)
fromItem["parententry"] = dest
fromItem["parentrepo"] = str(self.getRootNode(dest).key())
db.Put(fromItem)
skel = self.editSkel()
assert skel.fromDB(item)
self.onItemReparent(skel)
self.onItemChanged(skel)
return self.render.reparentSuccess(obj=fromItem)
def updateTxn(cacheKey):
key = db.Key(self.rateLimitKind, cacheKey)
obj = db.Get(key)
if obj is None:
obj = db.Entity(key)
obj["value"] = 0
obj["value"] += 1
obj["expires"] = utils.utcNow() + timedelta(minutes=2 *
self.minutes)
db.Put(obj)
def getOldBlobKeysTxn(dbKey):
obj = db.Get(dbKey)
res = obj["old_blob_references"] or []
if obj["is_stale"]:
db.Delete(dbKey)
else:
obj["has_old_blob_references"] = False
obj["old_blob_references"] = []
db.Put(obj)
return res
def sendEmailDeferred(emailKey: db.KeyClass):
"""
Callback from the Taskqueue to send the given Email
:param emailKey: Database-Key of the email we should send
"""
logging.debug("Sending deferred email: %s" % str(emailKey))
queueEntity = db.Get(emailKey)
assert queueEntity, "Email queue object went missing!"
if queueEntity["isSend"]:
return True
elif queueEntity["errorCount"] > 3:
raise ChildProcessError("Error-Count exceeded")
transportClass = conf[
"viur.email.transportClass"] # First, ensure we're able to send email at all
assert issubclass(
transportClass,
EmailTransport), "No or invalid email transportclass specified!"
try:
resultData = transportClass.deliverEmail(
dests=queueEntity["dests"],
sender=queueEntity["sender"],
cc=queueEntity["cc"],
bcc=queueEntity["bcc"],
subject=queueEntity["subject"],
body=queueEntity["body"],
headers=queueEntity["headers"],
attachments=queueEntity["attachments"])
except Exception:
# Increase the errorCount and bail out
queueEntity["errorCount"] += 1
db.Put(queueEntity)
raise
# If that transportFunction did not raise an error that email has been successfully send
queueEntity["isSend"] = True
queueEntity["sendDate"] = utils.utcNow()
queueEntity["transportFuncResult"] = resultData
queueEntity.exclude_from_indexes.add("transportFuncResult")
db.Put(queueEntity)
try:
transportClass.transportSuccessfulCallback(queueEntity)
except Exception as e:
logging.exception(e)
def updateTxn(key, resDict):
obj = db.Get(key)
if not obj: # File-object got deleted during building of our derives
return
obj["derived"] = obj.get("derived") or {}
obj["derived"]["deriveStatus"] = obj["derived"].get("deriveStatus") or {}
obj["derived"]["files"] = obj["derived"].get("files") or {}
for k, v in resDict.items():
obj["derived"]["deriveStatus"][k] = v["version"]
for fileName, fileDict in v["files"].items():
obj["derived"]["files"][fileName] = fileDict
db.Put(obj)
def pwrecover(self, authtoken=None, skey=None, *args, **kwargs):
if authtoken:
data = securitykey.validate(authtoken, useSessionKey=False)
if data and isinstance(
data, dict) and "userKey" in data and "password" in data:
skel = self.userModule.editSkel()
assert skel.fromDB(data["userKey"])
skel["password"] = data["password"]
skel.toDB()
return self.userModule.render.view(
skel, self.passwordRecoverySuccessTemplate)
else:
return self.userModule.render.view(
None, self.passwordRecoveryInvalidTokenTemplate)
else:
skel = self.lostPasswordSkel()
if len(kwargs) == 0 or not skel.fromClient(
kwargs) or not securitykey.validate(skey,
useSessionKey=True):
return self.userModule.render.passwdRecover(
skel, tpl=self.passwordRecoveryTemplate)
user = self.userModule.viewSkel().all().filter(
"name.idx =", skel["name"].lower()).get()
if not user or user[
"status"] < 10: # Unknown user or locked account
skel.errors["name"] = str("Unknown user")
return self.userModule.render.passwdRecover(
skel, tpl=self.passwordRecoveryTemplate)
try:
if user["changedate"] > (datetime.datetime.now() -
datetime.timedelta(days=1)):
# This user probably has already requested a password reset
# within the last 24 hrss
return self.userModule.render.view(
skel, self.passwordRecoveryAlreadySendTemplate)
except AttributeError: # Some newly generated user-objects dont have such a changedate yet
pass
user["changedate"] = datetime.datetime.now()
db.Put(user)
userSkel = self.userModule.viewSkel().ensureIsCloned()
assert userSkel.fromDB(user.key())
userSkel.skey = baseBone(descr="Skey")
userSkel["skey"] = securitykey.create(60 * 60 * 24,
userKey=str(user.key()),
password=skel["password"])
utils.sendEMail([userSkel["name"]],
self.userModule.passwordRecoveryMail, userSkel)
return self.userModule.render.view(
{}, self.passwordRecoveryInstuctionsSendTemplate)
def save(self, req):
"""
Writes the session to the memcache/datastore.
Does nothing, if the session hasn't been changed in the current request.
"""
try:
if self.changed or self.isInitial:
serialized = base64.b64encode(
pickle.dumps(self.session,
protocol=pickle.HIGHEST_PROTOCOL))
# Get the current user id
try:
# Check for our custom user-api
userid = conf["viur.mainApp"].user.getCurrentUser()["key"]
except:
userid = None
if self.isInitial and not req.isSSLConnection:
# Reset the Secure only key to None - we can't set it anyway.
self.sslKey = None
try:
dbSession = db.Entity(db.Key(self.kindName, self.httpKey))
dbSession["data"] = serialized
dbSession["sslkey"] = self.sslKey
dbSession["staticSecurityKey"] = self.staticSecurityKey
dbSession["securityKey"] = self.securityKey
dbSession["lastseen"] = time()
# Store the userid inside the sessionobj, so we can kill specific sessions if needed
dbSession["user"] = str(userid) or "guest"
db.Put(dbSession)
except Exception as e:
logging.exception(e)
raise # FIXME
pass
if self.sameSite:
sameSite = "; SameSite=%s" % self.sameSite
else:
sameSite = ""
req.response.headerlist.append(
("Set-Cookie", "%s=%s; Max-Age=99999; Path=/; HttpOnly%s" %
(self.plainCookieName, self.httpKey, sameSite)))
if req.isSSLConnection:
req.response.headerlist.append(
("Set-Cookie",
"%s=%s; Max-Age=99999; Path=/; Secure; HttpOnly%s" %
(self.sslCookieName, self.sslKey, sameSite)))
except Exception as e:
raise # FIXME
logging.exception(e)
def calculateOrderSum(self, step, orderKey, *args, **kwargs):
"""
Calculates the final price for this order.
This function *must* be called before any attempt is made to start a payment process.
:param step: Current step within the ordering process
:type step: int
:param orderKey: order to calculate the price for
:type orderKey: str
"""
price = sum([x[3] for x in self.getBillItems(orderKey)])
orderObj = db.Get(db.Key(str(orderKey)))
orderObj["price"] = price
db.Put(orderObj)
def delete(self, skel: 'viur.core.skeleton.SkeletonInstance', name: str):
"""
Ensure any outgoing relational lock is cleared
:param skel:
:param name:
:return:
"""
if skel.dbEntity.get("%s_outgoingRelationalLocks" % name):
for refKey in skel.dbEntity["%s_outgoingRelationalLocks" % name]:
referencedEntry = db.Get(refKey)
if not referencedEntry:
logging.warning("Programming error detected: Entry %s is gone despite lock" % refKey)
continue
incommingLocks = referencedEntry.get("viur_incomming_relational_locks", [])
# We remove any reference to ourself as multiple bones may hold Locks to the same entry
referencedEntry["viur_incomming_relational_locks"] = [x for x in incommingLocks if x != skel["key"]]
db.Put(referencedEntry)
def wrapF(self, *args, **kwargs) -> Union[str, bytes]:
currReq = currentRequest.get()
if conf["viur.disableCache"] or currReq.disableCache:
# Caching disabled
if conf["viur.disableCache"]:
logging.debug("Caching is disabled by config")
return f(self, *args, **kwargs)
# How many arguments are part of the way to the function called (and how many are just *args)
offset = -len(currReq.args) or len(currReq.pathlist)
path = "/" + "/".join(currReq.pathlist[:offset])
if not path in urls:
# This path (possibly a sub-render) should not be cached
logging.debug("Not caching for %s" % path)
return f(self, *args, **kwargs)
key = keyFromArgs(f, userSensitive, languageSensitive, evaluatedArgs,
path, args, kwargs)
if not key:
# Something is wrong (possibly the parameter-count)
# Let's call f, but we knew already that this will clash
return f(self, *args, **kwargs)
dbRes = db.Get(db.Key(viurCacheName, key))
if dbRes is not None:
if not maxCacheTime \
or dbRes["creationtime"] > utils.utcNow() - timedelta(seconds=maxCacheTime):
# We store it unlimited or the cache is fresh enough
logging.debug("This request was served from cache.")
currReq.response.headers['Content-Type'] = dbRes[
"content-type"]
return dbRes["data"]
# If we made it this far, the request wasn't cached or too old; we need to rebuild it
oldAccessLog = db.startDataAccessLog()
try:
res = f(self, *args, **kwargs)
finally:
accessedEntries = db.endDataAccessLog(oldAccessLog)
dbEntity = db.Entity(db.Key(viurCacheName, key))
dbEntity["data"] = res
dbEntity["creationtime"] = utils.utcNow()
dbEntity["path"] = path
dbEntity["content-type"] = currReq.response.headers['Content-Type']
dbEntity["accessedEntries"] = list(accessedEntries)
dbEntity.exclude_from_indexes = ["data", "content-type"
] # We can save 2 DB-Writs :)
db.Put(dbEntity)
logging.debug("This request was a cache-miss. Cache has been updated.")
return res
def wrapF(self, *args, **kwargs):
currentRequest = request.current.get()
if conf["viur.disableCache"] or currentRequest.disableCache:
# Caching disabled
if conf["viur.disableCache"]:
logging.debug("Caching is disabled by config")
return (f(self, *args, **kwargs))
# How many arguments are part of the way to the function called (and how many are just *args)
offset = -len(currentRequest.args) or len(currentRequest.pathlist)
path = "/" + "/".join(currentRequest.pathlist[:offset])
if not path in urls:
# This path (possibly a sub-render) should not be cached
logging.debug("Not caching for %s" % path)
return (f(self, *args, **kwargs))
key = keyFromArgs(f, userSensitive, languageSensitive, evaluatedArgs,
path, args, kwargs)
if not key:
# Someting is wrong (possibly the parameter-count)
# Letz call f, but we knew already that this will clash
return (f(self, *args, **kwargs))
try:
dbRes = db.Get(db.Key.from_path(viurCacheName, key))
except db.EntityNotFoundError:
dbRes = None
if dbRes:
if not maxCacheTime or \
dbRes["creationtime"] > datetime.now() - timedelta(seconds=maxCacheTime):
# We store it unlimited or the cache is fresh enough
logging.debug("This request was served from cache.")
currentRequest.response.headers['Content-Type'] = dbRes[
"content-type"].encode("UTF-8")
return (dbRes["data"])
# If we made it this far, the request wasnt cached or too old; we need to rebuild it
res = f(self, *args, **kwargs)
dbEntity = db.Entity(viurCacheName, name=key)
dbEntity["data"] = res
dbEntity["creationtime"] = datetime.now()
dbEntity["path"] = path
dbEntity["content-type"] = request.current.get(
).response.headers['Content-Type']
dbEntity.set_unindexed_properties(["data", "content-type"
]) # We can save 2 DB-Writs :)
db.Put(dbEntity)
logging.debug("This request was a cache-miss. Cache has been updated.")
return (res)
def save(self, req):
"""
Writes the session to the datastore.
Does nothing, if the session hasn't been changed in the current request.
"""
try:
if self.changed or self.isInitial:
if not (req.isSSLConnection or req.isDevServer
): # We will not issue sessions over http anymore
return False
# Get the current user id
try:
# Check for our custom user-api
userid = conf["viur.mainApp"].user.getCurrentUser()["key"]
except:
userid = None
try:
dbSession = db.Entity(db.Key(self.kindName,
self.cookieKey))
dbSession["data"] = db.fixUnindexableProperties(
self.session)
dbSession["staticSecurityKey"] = self.staticSecurityKey
dbSession["securityKey"] = self.securityKey
dbSession["lastseen"] = time()
# Store the userid inside the sessionobj, so we can kill specific sessions if needed
dbSession["user"] = str(userid) or "guest"
dbSession.exclude_from_indexes = ["data"]
db.Put(dbSession)
except Exception as e:
logging.exception(e)
raise # FIXME
pass
sameSite = "; SameSite=%s" % self.sameSite if self.sameSite else ""
secure = "; Secure" if not req.isDevServer else ""
maxAge = "; Max-Age=%s" % conf[
"viur.session.lifeTime"] if not self.sessionCookie else ""
req.response.headerlist.append(
("Set-Cookie", "%s=%s; Path=/; HttpOnly%s%s%s" %
(self.cookieName, self.cookieKey, sameSite, secure,
maxAge)))
except Exception as e:
raise # FIXME
logging.exception(e)
def getOrBuildIndex(self, origQuery):
"""
Builds a specific index based on origQuery AND local variables (self.indexPage and self.indexMaxPage)
Returns a list of starting-cursors for each page.
You probably shouldn't call this directly. Use cursorForQuery.
:param origQuery: Query to build the index for
:type origQuery: db.Query
:returns: []
"""
key = self.keyFromQuery(origQuery)
if key in self._cache: # We have it cached
return self._cache[key]
# We don't have it cached - try to load it from DB
try:
index = db.Get(db.Key.from_path(self._dbType, key))
res = json.loads(index["data"])
self._cache[key] = res
return res
except db.EntityNotFoundError: # Its not in the datastore, too
pass
# We don't have this index yet.. Build it
# Clone the original Query
queryRes = origQuery.clone(keysOnly=True).datastoreQuery.Run(
limit=self.maxPages * self.pageSize)
# Build-Up the index
res = list()
previousCursor = None # The first page dosnt have any cursor
# enumerate is slightly faster than a manual loop counter
for counter, discardedKey in enumerate(queryRes):
if counter % self.pageSize == 0:
res.append(previousCursor)
if counter % self.pageSize == (self.pageSize - 1):
previousCursor = str(queryRes.cursor().urlsafe())
if not len(res): # Ensure that the first page exists
res.append(None)
entry = db.Entity(self._dbType, name=key)
entry["data"] = json.dumps(res)
entry["creationdate"] = datetime.now()
db.Put(entry)
return res
def create(duration: Union[None, int] = None, **kwargs) -> str:
"""
Creates a new onetime Securitykey or returns the current sessions csrf-token.
The custom data (given as keyword arguments) that can be stored with the key if :param:duration is set must
be serializable by the datastore.
:param duration: Make this key valid for a fixed timeframe (and independent of the current session)
:returns: The new onetime key
"""
if not duration:
return currentSession.get().getSecurityKey()
key = generateRandomString()
duration = int(duration)
dbObj = db.Entity(db.Key(securityKeyKindName, key))
for k, v in kwargs.items():
dbObj[k] = v
dbObj["until"] = utcNow() + timedelta(seconds=duration)
db.Put(dbObj)
return key
def storeEntry(self, e, key):
if not self._checkKey(key, export=False):
raise errors.Forbidden()
entry = pickle.loads(bytes.fromhex(e))
for k in list(entry.keys()):
if isinstance(entry[k], bytes):
entry[k] = entry[k].decode("UTF-8")
for k in list(entry.keys()):
if k in entry and "." in k:
base = k.split(".")[0]
tmpDict = {}
for subkey in list(entry.keys()):
if subkey.startswith("%s." % base):
postfix = subkey.split(".")[1]
tmpDict[postfix] = entry[subkey]
del entry[subkey]
entry[base] = tmpDict
for k in list(entry.keys()):
if "-" in k:
entry[k.replace("-", "_")] = entry[k]
del entry[k]
if "key" in entry:
key = db.Key(*entry["key"].split("/"))
else:
raise AttributeError()
dbEntry = db.Entity(key) # maybe some more fixes here ?
for k in entry.keys():
if k != "key":
val = entry[k]
# if isinstance(val, dict) or isinstance(val, list):
# val = pickle.dumps( val )
dbEntry[k] = val
#dbEntry = fixUnindexed(dbEntry)
try:
db.Put(dbEntry)
except:
from pprint import pprint
pprint(dbEntry)
raise
def create(duration: Union[None, int] = None, **kwargs):
"""
Creates a new onetime Securitykey for the current session
If duration is not set, this key is valid only for the current session.
Otherwise, the key and its data is serialized and saved inside the datastore
for up to duration-seconds
:param duration: Make this key valid for a fixed timeframe (and independend of the current session)
:type duration: int or None
:returns: The new onetime key
"""
if not duration:
return currentSession.getSecurityKey()
key = generateRandomString()
duration = int(duration)
dbObj = db.Entity(securityKeyKindName, name=key)
for k, v in kwargs.items():
dbObj[k] = v
dbObj["until"] = datetime.now() + timedelta(seconds=duration)
db.Put(dbObj)
return key
def markFileForDeletion(dlkey):
"""
Adds a marker to the data store that the file specified as *dlkey* can be deleted.
Once the mark has been set, the data store is checked four times (default: every 4 hours)
if the file is in use somewhere. If it is still in use, the mark goes away, otherwise
the mark and the file are removed from the datastore. These delayed checks are necessary
due to database inconsistency.
:type dlkey: str
:param dlkey: Unique download-key of the file that shall be marked for deletion.
"""
fileObj = db.Query("viur-deleted-files").filter("dlkey", dlkey).getEntry()
if fileObj: # Its allready marked
return
fileObj = db.Entity(db.Key("viur-deleted-files"))
fileObj["itercount"] = 0
fileObj["dlkey"] = str(dlkey)
db.Put(fileObj)
def sendEMail(*,
tpl: str = None,
stringTemplate: str = None,
skel: Union[None, Dict, "SkeletonInstance",
List["SkeletonInstance"]] = None,
sender: str = None,
dests: Union[str, List[str]] = None,
cc: Union[str, List[str]] = None,
bcc: Union[str, List[str]] = None,
headers: Dict[str, str] = None,
attachments: List[Dict[str, Any]] = None,
context: Union[db.DATASTORE_BASE_TYPES,
List[db.DATASTORE_BASE_TYPES], db.Entity] = None,
**kwargs) -> Any:
"""
General purpose function for sending e-mail.
This function allows for sending e-mails, also with generated content using the Jinja2 template engine.
Your have to implement a method which should be called to send the prepared email finally. For this you have
to allocate *viur.email.transportClass* in conf.
:param tpl: The name of a template from the deploy/emails directory.
:param stringTemplate: This string is interpreted as the template contents. Alternative to load from template file.
:param skel: The data made available to the template. In case of a Skeleton or SkelList, its parsed the usual way;\
Dictionaries are passed unchanged.
:param sender: The address sending this mail.
:param dests: A list of addresses to send this mail to. A bare string will be treated as a list with 1 address.
:param cc: Carbon-copy recipients. A bare string will be treated as a list with 1 address.
:param bcc: Blind carbon-copy recipients. A bare string will be treated as a list with 1 address.
:param headers: Specify headers for this email.
:param attachments: List of files to be sent within the mail as attachments. Each attachment must be a dictionary
with these keys:
filename (string): Name of the file that's attached. Always required
content (bytes): Content of the attachment as bytes. Required for the send in blue API.
mimetype (string): Mimetype of the file. Suggested parameter for other implementations (not used by SIB)
gcsfile (string): Link to a GCS-File to include instead of content. Not supported by the current
SIB implementation
:param context: Arbitrary data that can be stored along the queue entry to be evaluated in
transportSuccessfulCallback (useful for tracking delivery / opening events etc).
.. Warning: As emails will be queued (and not send directly) you cannot exceed 1MB in total
(for all text and attachments combined)!
"""
# First, ensure we're able to send email at all
transportClass = conf[
"viur.email.transportClass"] # First, ensure we're able to send email at all
assert issubclass(
transportClass,
EmailTransport), "No or invalid email transportclass specified!"
# Ensure that all recipient parameters (dest, cc, bcc) are a list
dests = normalizeToList(dests)
cc = normalizeToList(cc)
bcc = normalizeToList(bcc)
assert dests or cc or bcc, "No destination address given"
attachments = normalizeToList(attachments)
if not (bool(stringTemplate) ^ bool(tpl)):
raise ValueError(
"You have to set the params 'tpl' xor a 'stringTemplate'.")
if attachments:
# Ensure each attachment has the filename key and rewrite each dict to db.Entity so we can exclude
# it from being indexed
for _ in range(0, len(attachments)):
attachment = attachments.pop(0)
assert "filename" in attachment
entity = db.Entity()
for k, v in attachment.items():
entity[k] = v
entity.exclude_from_indexes.add(k)
attachments.append(entity)
assert all(["filename" in x for x in attachments
]), "Attachment is missing the filename key"
# If conf["viur.email.recipientOverride"] is set we'll redirect any email to these address(es)
if conf["viur.email.recipientOverride"]:
logging.warning("Overriding destination %s with %s", dests,
conf["viur.email.recipientOverride"])
oldDests = dests
newDests = normalizeToList(conf["viur.email.recipientOverride"])
dests = []
for newDest in newDests:
if newDest.startswith("@"):
for oldDest in oldDests:
dests.append(
oldDest.replace(".", "_dot_").replace("@", "_at_") +
newDest)
else:
dests.append(newDest)
cc = bcc = []
elif conf["viur.email.recipientOverride"] is False:
logging.warning(
"Sending emails disabled by config[viur.email.recipientOverride]")
return False
if conf["viur.email.senderOverride"]:
sender = conf["viur.email.senderOverride"]
elif sender is None:
sender = f"viur@{projectID}.appspotmail.com"
subject, body = conf["viur.emailRenderer"](dests, tpl, stringTemplate,
skel, **kwargs)
# Push that email to the outgoing queue
queueEntity = db.Entity(db.Key("viur-emails"))
queueEntity["isSend"] = False
queueEntity["errorCount"] = 0
queueEntity["creationDate"] = utils.utcNow()
queueEntity["sender"] = sender
queueEntity["dests"] = dests
queueEntity["cc"] = cc
queueEntity["bcc"] = bcc
queueEntity["subject"] = subject
queueEntity["body"] = body
queueEntity["headers"] = headers
queueEntity["attachments"] = attachments
queueEntity["context"] = context
queueEntity.exclude_from_indexes = ["body", "attachments", "context"]
transportClass.validateQueueEntity(
queueEntity) # Will raise an exception if the entity is not valid
if utils.isLocalDevelopmentServer and not conf[
"viur.email.sendFromLocalDevelopmentServer"]:
logging.info("Not sending email from local development server")
logging.info("Subject: %s", queueEntity["subject"])
logging.info("Body: %s", queueEntity["body"])
logging.info("Recipients: %s", queueEntity["dests"])
return False
db.Put(queueEntity)
sendEmailDeferred(queueEntity.key, _queue="viur-emails")
return True
def checkout(self, step=None, key=None, skey=None, *args, **kwargs):
"""
Performs the checkout process trough the state machine provided by self.steps.
:param step: The current step index, None for beginning a new checkout
:param key: Key of the current checkout
:param skey: Server security key
:return: Returns the rendered template or throws redirection exceptions.
"""
myKindName = self.viewSkel().kindName
if step is None:
logging.info("Starting new checkout process")
billObj = db.Entity(myKindName)
billObj["idx"] = "0000000"
for state in self.states:
billObj["state_%s" % state] = "0"
db.Put(billObj)
key = str(billObj.key())
# Copy the Cart
if "amountSkel" in dir(self):
cart = session.current.get("cart_products") or {}
s = self.amountSkel()
products = []
for prod, atts in cart.items():
for i in range(0, atts["amount"]):
products.append(str(prod))
s.fromClient({"product": products})
s.toDB()
session.current["order_" + myKindName] = {
"key": str(key),
"completedSteps": []
}
session.current.markChanged()
raise errors.Redirect("?step=0&key=%s" % str(key))
elif key:
try:
orderKey = db.Key(key)
step = int(step)
assert (step >= 0)
assert (step < len(self.steps))
except:
raise errors.NotAcceptable()
sessionInfo = session.current.get("order_" + myKindName)
if not sessionInfo or not sessionInfo.get("key") == str(orderKey):
raise errors.Unauthorized()
if step in sessionInfo["completedSteps"]:
session.current["order_" + myKindName]["completedSteps"] = [
x for x in sessionInfo["completedSteps"] if x < step
]
session.current.markChanged()
# Make sure that no steps can be skipped
if step != 0 and not step - 1 in sessionInfo["completedSteps"]:
raise errors.Redirect("?step=0&key=%s" % str(str(orderKey)))
currentStep = self.steps[step]
if "preHandler" in currentStep.keys():
try:
if isinstance(currentStep["preHandler"], list):
for handler in currentStep["preHandler"]:
handler(self, step, str(orderKey), *args, **kwargs)
else:
currentStep["preHandler"](self,
step,
str(orderKey),
refkwargs=kwargs,
*args,
**kwargs)
except SkipStepException:
session.current["order_" +
myKindName]["completedSteps"].append(step)
session.current.markChanged()
raise errors.Redirect("?step=%s&key=%s" %
(str(step + 1), str(orderKey)))
except ReturnHtmlException as e:
return (e.html)
if "requiresSecurityKey" in currentStep and currentStep[
"requiresSecurityKey"]:
if not securitykey.validate(skey):
raise errors.PreconditionFailed()
pass
if "mainHandler" in currentStep:
if currentStep["mainHandler"]["action"] == "edit":
skel = self.getSkelByName(
currentStep["mainHandler"]["skeleton"], str(orderKey))
skel.fromDB(str(orderKey))
if not len(kwargs.keys()) or not skel.fromClient(kwargs):
return (self.render.edit(
skel,
tpl=currentStep["mainHandler"]["template"],
step=step))
skel.toDB()
if currentStep["mainHandler"]["action"] == "view":
if not "complete" in kwargs or not kwargs[
"complete"] == u"1":
skel = self.getSkelByName(
currentStep["mainHandler"]["skeleton"],
str(orderKey))
skel.fromDB(str(orderKey))
return (self.render.view(
skel,
tpl=currentStep["mainHandler"]["template"],
step=step))
elif currentStep["mainHandler"]["action"] == "function":
res = currentStep["mainHandler"]["function"](self, step,
str(orderKey),
*args,
**kwargs)
if res:
return res
if "postHandler" in currentStep:
currentStep["postHandler"](self, step, str(orderKey), *args,
**kwargs)
session.current["order_" +
myKindName]["completedSteps"].append(step)
session.current.markChanged()
logging.info("next ?step=%s&key=%s" %
(str(step + 1), str(orderKey)))
raise errors.Redirect("?step=%s&key=%s" %
(str(step + 1), str(orderKey)))
