def exportItems(module, target, importKey, startCursor, endCursor):
Skel = skeletonByKind(module)
query = Skel().all().cursor(startCursor, endCursor)
for item in query.run(250):
flatItem = DbTransfer.genDict(item)
formFields = {
"e": pickle.dumps(flatItem).encode("HEX"),
"key": importKey
}
result = urlfetch.fetch(
url=target,
payload=urllib.urlencode(formFields),
method=urlfetch.POST,
headers={'Content-Type': 'application/x-www-form-urlencoded'})
if startCursor == endCursor:
try:
email.sendEMailToAdmins(
"Export of kind %s finished" % module,
"ViUR finished to export kind %s to %s.\n" % (module, target))
except: # OverQuota, whatever
pass
def renderBoneStructure(self, bone):
"""
Renders the structure of a bone.
This function is used by `renderSkelStructure`.
can be overridden and super-called from a custom renderer.
:param bone: The bone which structure should be rendered.
:type bone: Any bone that inherits from :class:`server.bones.base.baseBone`.
:return: A dict containing the rendered attributes.
:rtype: dict
"""
# Base bone contents.
ret = {
"descr": str(bone.descr),
"type": bone.type,
"required": bone.required,
"params": bone.params,
"visible": bone.visible,
"readonly": bone.readOnly,
"unique": bone.unique.method.value if bone.unique else False
}
if bone.type == "relational" or bone.type.startswith("relational."):
if isinstance(bone, bones.hierarchyBone):
boneType = "hierarchy"
elif isinstance(bone, bones.treeItemBone):
boneType = "treeitem"
elif isinstance(bone, bones.treeDirBone):
boneType = "treedir"
else:
boneType = "relational"
ret.update({
"type":
"%s.%s" % (boneType, bone.kind),
"module":
bone.module,
"multiple":
bone.multiple,
"format":
bone.format,
"using":
self.renderSkelStructure(bone.using()) if bone.using else None,
"relskel":
self.renderSkelStructure(
RefSkel.fromSkel(skeletonByKind(bone.kind), *bone.refKeys))
})
elif bone.type == "record" or bone.type.startswith("record."):
ret.update({
"multiple": bone.multiple,
"format": bone.format,
"using": self.renderSkelStructure(bone.using())
})
elif bone.type == "select" or bone.type.startswith("select."):
ret.update({
"values": [(k, str(v)) for k, v in bone.values.items()],
"multiple": bone.multiple,
})
elif bone.type == "date" or bone.type.startswith("date."):
ret.update({"date": bone.date, "time": bone.time})
elif bone.type == "numeric" or bone.type.startswith("numeric."):
ret.update({
"precision": bone.precision,
"min": bone.min,
"max": bone.max
})
elif bone.type == "text" or bone.type.startswith("text."):
ret.update({
"validHtml": bone.validHtml,
"languages": bone.languages
})
elif bone.type == "str" or bone.type.startswith("str."):
ret.update({
"multiple": bone.multiple,
"languages": bone.languages
})
return ret
def login(self, skey="", token="", *args, **kwargs):
# FIXME: Check if already logged in
if not conf.get("viur.user.google.clientID"):
raise errors.PreconditionFailed(
"Please configure 'viur.user.google.clientID' in your conf!")
if not skey or not token:
currentRequest.get().response.headers["Content-Type"] = "text/html"
if currentRequest.get().response.headers.get(
"cross-origin-opener-policy") == "same-origin":
# We have to allow popups here
currentRequest.get().response.headers[
"cross-origin-opener-policy"] = "same-origin-allow-popups"
# Fixme: Render with Jinja2?
tplStr = open(
os.path.join(utils.coreBasePath,
"viur/core/template/vi_user_google_login.html"),
"r").read()
tplStr = tplStr.replace("{{ clientID }}",
conf["viur.user.google.clientID"])
extendCsp({
"script-src":
["sha256-JpzaUIxV/gVOQhKoDLerccwqDDIVsdn1JclA6kRNkLw="],
"style-src":
["sha256-FQpGSicYMVC5jxKGS5sIEzrRjSJmkxKPaetUc7eamqc="]
})
return tplStr
if not securitykey.validate(skey, useSessionKey=True):
raise errors.PreconditionFailed()
userInfo = id_token.verify_oauth2_token(
token, requests.Request(), conf["viur.user.google.clientID"])
if userInfo['iss'] not in {
'accounts.google.com', 'https://accounts.google.com'
}:
raise ValueError('Wrong issuer.')
# Token looks valid :)
uid = userInfo['sub']
email = userInfo['email']
addSkel = skeletonByKind(self.userModule.addSkel().kindName
) # Ensure that we have the full skeleton
userSkel = addSkel().all().filter("uid =", uid).getSkel()
if not userSkel:
# We'll try again - checking if there's already an user with that email
userSkel = addSkel().all().filter("name.idx =",
email.lower()).getSkel()
if not userSkel: # Still no luck - it's a completely new user
if not self.registrationEnabled:
if userInfo.get("hd") and userInfo["hd"] in conf[
"viur.user.google.gsuiteDomains"]:
print("User is from domain - adding account")
else:
logging.warning("Denying registration of %s", email)
raise errors.Forbidden(
"Registration for new users is disabled")
userSkel = addSkel() # We'll add a new user
userSkel["uid"] = uid
userSkel["name"] = email
isAdd = True
else:
isAdd = False
now = utils.utcNow()
if isAdd or (now -
userSkel["lastlogin"]) > datetime.timedelta(minutes=30):
# Conserve DB-Writes: Update the user max once in 30 Minutes
userSkel["lastlogin"] = now
# if users.is_current_user_admin():
# if not userSkel["access"]:
# userSkel["access"] = []
# if not "root" in userSkel["access"]:
# userSkel["access"].append("root")
# userSkel["gaeadmin"] = True
# else:
# userSkel["gaeadmin"] = False
assert userSkel.toDB()
return self.userModule.continueAuthenticationFlow(
self, userSkel["key"])
def ensureDerived(key: db.KeyClass, srcKey, deriveMap: Dict[str, Any]):
"""
Ensure that pending thumbnails or other derived Files are build
:param key: DB-Key of the file-object on which we should update the derivemap
:param srcKey: Prefix for a (hopefully) stable key to prevent rebuilding derives over and over again
:param deriveMap: List of DeriveDicts we should build/update
"""
from viur.core.skeleton import skeletonByKind, updateRelations
deriveFuncMap = conf["viur.file.derivers"]
skel = skeletonByKind("file")()
if not skel.fromDB(key):
logging.info("File-Entry went missing in ensureDerived")
return
if not skel["derived"]:
logging.info("No Derives for this file")
skel["derived"] = {}
skel["derived"]["deriveStatus"] = skel["derived"].get("deriveStatus") or {}
skel["derived"]["files"] = skel["derived"].get("files") or {}
resDict = {
} # Will contain new or updated resultDicts that will be merged into our file
for calleeKey, params in deriveMap.items():
fullSrcKey = "%s_%s" % (srcKey, calleeKey)
paramsHash = sha256(str(params).encode(
"UTF-8")).hexdigest() # Hash over given params (dict?)
if skel["derived"]["deriveStatus"].get(fullSrcKey) != paramsHash:
if calleeKey not in deriveFuncMap:
logging.warning("File-Deriver %s not found - skipping!" %
calleeKey)
continue
callee = deriveFuncMap[calleeKey]
callRes = callee(skel, skel["derived"]["files"], params)
if callRes:
assert isinstance(
callRes,
list), "Old (non-list) return value from deriveFunc"
resDict[fullSrcKey] = {"version": paramsHash, "files": {}}
for fileName, size, mimetype, customData in callRes:
resDict[fullSrcKey]["files"][fileName] = {
"size": size,
"mimetype": mimetype,
"customData": customData
}
def updateTxn(key, resDict):
obj = db.Get(key)
if not obj: # File-object got deleted during building of our derives
return
obj["derived"] = obj.get("derived") or {}
obj["derived"]["deriveStatus"] = obj["derived"].get(
"deriveStatus") or {}
obj["derived"]["files"] = obj["derived"].get("files") or {}
for k, v in resDict.items():
obj["derived"]["deriveStatus"][k] = v["version"]
for fileName, fileDict in v["files"].items():
obj["derived"]["files"][fileName] = fileDict
db.Put(obj)
if resDict: # Write updated results back and queue updateRelationsTask
db.RunInTransaction(updateTxn, key, resDict)
# Queue that updateRelations call at least 30 seconds into the future, so that other ensureDerived calls from
# the same fileBone have the chance to finish, otherwise that updateRelations Task will call postSavedHandler
# on that fileBone again - re-queueing any ensureDerivedCalls that have not finished yet.
updateRelations(key, time() + 1, "derived", _countdown=30)
def iterImport(module, target, exportKey, cursor=None, amount=0):
"""
Processes 100 Entries and calls the next batch
"""
urlfetch.set_default_fetch_deadline(20)
payload = {"module": module, "key": exportKey}
if cursor:
payload.update({"cursor": cursor})
result = urlfetch.fetch(
url=target,
payload=urllib.urlencode(payload),
method=urlfetch.POST,
headers={'Content-Type': 'application/x-www-form-urlencoded'})
if result.status_code == 200:
res = pickle.loads(result.content.decode("HEX"))
skel = skeletonByKind(module)()
logging.info("%s: %d new entries fetched, total %d entries fetched" %
(module, len(res["values"]), amount))
if len(res["values"]) == 0:
try:
utils.sendEMailToAdmins(
"Import of kind %s finished with %d entities" %
(module, amount), "ViUR finished to import %d entities of "
"kind %s from %s.\n" % (amount, module, target))
except: # OverQuota, whatever
logging.error("Unable to send Email")
return
for entry in res["values"]:
for k in list(entry.keys())[:]:
if isinstance(entry[k], str):
entry[k] = entry[k].decode("UTF-8")
if not "key" in entry:
entry["key"] = entry["id"]
key = db.Key(encoded=utils.normalizeKey(entry["key"]))
# Special case: Convert old module root nodes!!!
if module.endswith(
"_rootNode") and key.name() and "_modul_" in key.name():
name = key.name().replace("_modul_", "_module_")
else:
name = key.name()
dbEntry = db.Entity(kind=key.kind(),
parent=key.parent(),
id=key.id(),
name=name)
for k in entry.keys():
if k == "key":
continue
dbEntry[k] = entry[k]
# Special case: Convert old module root nodes!!!
if (isinstance(skel, (HierarchySkel, TreeLeafSkel))
and k in ["parentdir", "parententry", "parentrepo"]
and entry[k]):
key = db.Key(encoded=str(entry[k]))
if key.parent():
parent = db.Key(
encoded=utils.normalizeKey(key.parent()))
else:
parent = None
if key.id_or_name() and "_modul_" in str(key.id_or_name()):
name = key.id_or_name().replace("_modul_", "_module_")
else:
name = key.id_or_name()
dbEntry[k] = str(
db.Key.from_path(key.kind(), name, parent=parent))
db.Put(dbEntry)
skel.fromDB(str(dbEntry.key()))
skel.refresh()
skel.toDB(clearUpdateTag=True)
amount += 1
iterImport(module, target, exportKey, res["cursor"], amount)
def login(self, skey="", token="", *args, **kwargs):
# FIXME: Check if already logged in
if not conf.get("viur.user.google.clientID"):
raise errors.PreconditionFailed()
if not skey or not token:
request.current.get(
).response.headers["Content-Type"] = "text/html"
# Fixme: Render with Jinja2?
tplStr = open("server/template/vi_user_google_login.html",
"r").read()
tplStr = tplStr.replace("{{ clientID }}",
conf["viur.user.google.clientID"])
return tplStr
if not securitykey.validate(skey, useSessionKey=True):
raise errors.PreconditionFailed()
userInfo = id_token.verify_oauth2_token(
token, requests.Request(), conf["viur.user.google.clientID"])
if userInfo['iss'] not in {
'accounts.google.com', 'https://accounts.google.com'
}:
raise ValueError('Wrong issuer.')
# Token looks valid :)
uid = userInfo['sub']
email = userInfo['email']
addSkel = skeletonByKind(self.userModule.addSkel().kindName
) # Ensure that we have the full skeleton
userSkel = addSkel().all().filter("uid =", uid).getSkel()
if not userSkel:
# We'll try again - checking if there's already an user with that email
userSkel = addSkel().all().filter("name.idx =",
email.lower()).getSkel()
if not userSkel: # Still no luck - it's a completely new user
if not self.registrationEnabled:
if userInfo.get("hd") and userInfo["hd"] in conf[
"viur.user.google.gsuiteDomains"]:
print("User is from domain - adding account")
else:
logging.warning("Denying registration of %s", email)
raise errors.Forbidden(
"Registration for new users is disabled")
userSkel = addSkel() # We'll add a new user
userSkel["uid"] = uid
userSkel["name"] = email
isAdd = True
else:
isAdd = False
now = datetime.datetime.now()
if isAdd or (now -
userSkel["lastlogin"]) > datetime.timedelta(minutes=30):
# Conserve DB-Writes: Update the user max once in 30 Minutes
userSkel["lastlogin"] = now
#if users.is_current_user_admin():
# if not userSkel["access"]:
# userSkel["access"] = []
# if not "root" in userSkel["access"]:
# userSkel["access"].append("root")
# userSkel["gaeadmin"] = True
#else:
# userSkel["gaeadmin"] = False
assert userSkel.toDB()
return self.userModule.continueAuthenticationFlow(
self, (userSkel.kindName, userSkel["key"]))
if users.get_current_user():
currentUser = users.get_current_user()
uid = currentUser.user_id()
raise errors.Redirect(
users.create_login_url(self.modulePath + "/login"))
def refreshTxn(): skel = skeletonByKind(refreshKey.kind)() if not skel.fromDB(refreshKey): return skel.refresh() skel.toDB(clearUpdateTag=True)
