def createNewUserIfNotExists():
"""
Create a new Admin user, if the userDB is empty
"""
userMod = getattr(conf["viur.mainApp"], "user", None)
if (userMod # We have a user module
and isinstance(userMod, User)
and "addSkel" in dir(userMod)
and "validAuthenticationMethods" in dir(userMod) #Its our user module :)
and any([x[0] is UserPassword for x in userMod.validAuthenticationMethods])): #It uses UserPassword login
if not db.Query(userMod.addSkel().kindName).get(): #There's currently no user in the database
addSkel = skeletonByKind(userMod.addSkel().kindName)() # Ensure we have the full skeleton
uname = "admin@%s.appspot.com" % app_identity.get_application_id()
pw = utils.generateRandomString(13)
addSkel["name"] = uname
addSkel["status"] = 10 # Ensure its enabled right away
addSkel["access"] = ["root"]
addSkel["password"] = pw
try:
addSkel.toDB()
except Exception as e:
logging.error("Something went wrong when trying to add admin user %s with Password %s", (uname, pw))
logging.exception(e)
return
logging.warning("ViUR created a new admin-user for you! Username: %s, Password: %s" % (uname, pw))
utils.sendEMailToAdmins("Your new ViUR password",
"ViUR created a new admin-user for you! Username: %s, Password: %s" % (uname, pw))
def storeEntry2(self, e, key):
if not self._checkKey(key, export=False):
raise errors.Forbidden()
entry = pickle.loads(e.decode("HEX"))
for k in list(entry.keys())[:]:
if isinstance(entry[k], str):
entry[k] = entry[k].decode("UTF-8")
key = db.Key(encoded=utils.normalizeKey(entry["key"]))
logging.info(key.kind())
logging.info(key.id())
logging.info(key.name())
dbEntry = db.Entity(kind=key.kind(),
parent=key.parent(),
id=key.id(),
name=key.name()) #maybe some more fixes here ?
for k in entry.keys():
if k != "key":
val = entry[k]
dbEntry[k] = val
db.Put(dbEntry)
try:
skel = skeletonByKind(key.kind())()
except:
logging.error("Unknown Skeleton - skipping")
skel.fromDB(str(dbEntry.key()))
skel.refresh()
skel.toDB(clearUpdateTag=True)
def login(self, skey="", *args, **kwargs):
if users.get_current_user():
addSkel = skeletonByKind(self.userModule.addSkel().kindName) # Ensure that we have the full skeleton
currentUser = users.get_current_user()
uid = currentUser.user_id()
userSkel = addSkel().all().filter("uid =", uid).getSkel()
if not userSkel:
# We'll try again - checking if there's already an user with that email
userSkel = addSkel().all().filter("name.idx =", currentUser.email().lower()).getSkel()
if not userSkel: # Still no luck - it's a completely new user
if not self.registrationEnabled and not users.is_current_user_admin():
# Registration is disabled, it's a new user and that user is not admin
logging.warning("Denying registration of %s", currentUser.email())
raise errors.Forbidden("Registration for new users is disabled")
userSkel = addSkel() # We'll add a new user
userSkel["uid"] = uid
userSkel["name"] = currentUser.email()
isAdd = True
else:
isAdd = False
now = datetime.datetime.now()
if isAdd or (now-userSkel["lastlogin"]) > datetime.timedelta(minutes=30):
# Conserve DB-Writes: Update the user max once in 30 Minutes
userSkel["lastlogin"] = now
if users.is_current_user_admin():
if not userSkel["access"]:
userSkel["access"] = []
if not "root" in userSkel["access"]:
userSkel["access"].append("root")
userSkel["gaeadmin"] = True
else:
userSkel["gaeadmin"] = False
assert userSkel.toDB()
return self.userModule.continueAuthenticationFlow(self, userSkel["key"])
raise errors.Redirect( users.create_login_url( self.modulePath+"/login") )
def getCfg(self, module, key):
if not self._checkKey(key, export=False):
raise errors.Forbidden()
skel = skeletonByKind(module)
assert skel is not None
res = skel()
r = DefaultRender()
return (pickle.dumps(r.renderSkelStructure(res)))
def doDeleteWeakReferences(timeStamp, cursor):
skelCls = skeletonByKind("file")
gotAtLeastOne = False
query = skelCls().all().filter("weak =", True).filter("creationdate <", datetime.strptime(timeStamp,"%d.%m.%Y %H:%M:%S")).cursor(cursor)
for skel in query.fetch(99):
# FIXME: Is that still needed? See hotfix/weakfile
anyRel = any(db.Query("viur-relations").filter("dest.key =", skel["key"]).run(1, keysOnly=True))
if anyRel:
logging.debug("doDeleteWeakReferences: found relations with that file - don't delete!")
continue
gotAtLeastOne = True
skel.delete()
newCursor = query.getCursor()
if gotAtLeastOne and newCursor and newCursor.urlsafe() != cursor:
doDeleteWeakReferences(timeStamp, newCursor.urlsafe())
def relSkelFromKey(key):
if not isinstance(key, db.Key):
key = db.Key(encoded=key)
if not key.kind() == self.kind:
logging.error(
"I got a key, which kind doesn't match my type! (Got: %s, my type %s)"
% (key.kind(), self.kind))
return None
entity = db.Get(key)
if not entity:
logging.error("Key %s not found" % str(key))
return None
relSkel = RefSkel.fromSkel(skeletonByKind(self.kind),
*self.refKeys)
relSkel.unserialize(entity)
return relSkel
def _resolveSkelCls(self, *args, **kwargs):
"""
Retrieve the generally associated :class:`server.skeleton.Skeleton` that is used by
the application.
This is either be defined by the member variable *kindName* or by a Skeleton named like the
application class in lower-case order.
If this behavior is not wanted, it can be definitely overridden by defining module-specific
:func:`viewSkel`,:func:`addSkel`, or :func:`editSkel` functions, or by overriding this
function in general.
:return: Returns a Skeleton instance that matches the application.
:rtype: server.skeleton.Skeleton
"""
return skeletonByKind(self.kindName if self.kindName else unicode(
type(self).__name__).lower())
def iterExport(module, target, importKey, cursor=None):
"""
Processes 100 Entries and calls the next batch
"""
urlfetch.set_default_fetch_deadline(20)
Skel = skeletonByKind(module)
if not Skel:
logging.error("TaskExportKind: Invalid module")
return
query = Skel().all().cursor(cursor)
startCursor = cursor
query.run(100, keysOnly=True)
endCursor = query.getCursor().urlsafe()
exportItems(module, target, importKey, startCursor, endCursor)
if startCursor is None or startCursor != endCursor:
iterExport(module, target, importKey, endCursor)
def getSkelForRequest(skelName, key=None, attr=None):
"""
Retrieves a skeleton with a key and caches it in the current request.
If no key is given, the function tries to retrieve the key for the requested object
from the request parameters, by preceding an "@" sign to the skelName as parameter.
So adding "@project=abc" will try to load a "project"-Skeleton with the key "abc".
"""
#print("getSkelForRequest", skelName, key, attr)
if key is None:
key = request.current.get().kwargs.get("@%s" % skelName)
if key is None or not key:
return None
reqData = request.current.requestData()
if "%s.%s" % (skelName, key) in reqData.keys():
skel = reqData["%s.%s" % (skelName, key)]
if skel and attr is not None:
return skel[attr]
return skel
skel = skeleton.skeletonByKind(skelName)
if skel:
skel = skel()
if skel.fromDB(key):
skel = skel.clone()
else:
skel = None
reqData["%s.%s" % (skelName, key)] = skel
if skel and attr is not None:
return skel[attr]
return skel
def exportItems(module, target, importKey, startCursor, endCursor):
Skel = skeletonByKind(module)
query = Skel().all().cursor(startCursor, endCursor)
for item in query.run(250):
flatItem = DbTransfer.genDict(item)
formFields = {
"e": pickle.dumps(flatItem).encode("HEX"),
"key": importKey
}
result = urlfetch.fetch(
url=target,
payload=urllib.urlencode(formFields),
method=urlfetch.POST,
headers={'Content-Type': 'application/x-www-form-urlencoded'})
if startCursor == endCursor:
try:
utils.sendEMailToAdmins(
"Export of kind %s finished" % module,
"ViUR finished to export kind %s to %s.\n" % (module, target))
except: #OverQuota, whatever
pass
def _restoreValueFromDatastore(self, val):
"""
Restores one of our values (including the Rel- and Using-Skel) from the serialized data read from the datastore
:param value: Json-Encoded datastore property
:return: Our Value (with restored RelSkel and using-Skel)
"""
value = extjson.loads(val)
from server.skeleton import RefSkel, skeletonByKind
assert isinstance(
value, dict
), "Read something from the datastore thats not a dict: %s" % str(
type(value))
relSkel = RefSkel.fromSkel(skeletonByKind(self.kind), *self.refKeys)
# !!!ViUR re-design compatibility!!!
if not "dest" in value.keys():
nvalue = dict()
nvalue["dest"] = value
value = nvalue
if "id" in value["dest"].keys() and not ("key" in value["dest"].keys()
and value["dest"]["key"]):
value["dest"]["key"] = value["dest"]["id"]
del value["dest"]["id"]
# UNTIL HERE!
relSkel.unserialize(value["dest"])
if self.using is not None:
usingSkel = self.using()
if value["rel"] is not None:
usingSkel.unserialize(value["rel"])
else:
usingSkel = None
return {"dest": relSkel, "rel": usingSkel}
def buildDBFilter(self, name, skel, dbFilter, rawFilter, prefix=None):
from server.skeleton import RefSkel, skeletonByKind
origFilter = dbFilter.datastoreQuery
if origFilter is None: #This query is unsatisfiable
return (dbFilter)
myKeys = [x for x in rawFilter.keys() if x.startswith("%s." % name)]
if len(myKeys) > 0 and not self.indexed:
logging.warning("Invalid searchfilter! %s is not indexed!" % name)
raise RuntimeError()
if len(myKeys) > 0: #We filter by some properties
if dbFilter.getKind() != "viur-relations" and self.multiple:
name, skel, dbFilter, rawFilter = self._rewriteQuery(
name, skel, dbFilter, rawFilter)
relSkel = RefSkel.fromSkel(skeletonByKind(self.kind),
*self.refKeys)
# Merge the relational filters in
for myKey in myKeys:
value = rawFilter[myKey]
try:
unused, _type, key = myKey.split(".", 2)
assert _type in ["dest", "rel"]
except:
if self.using is None:
# This will be a "dest" query
_type = "dest"
try:
unused, key = myKey.split(".", 1)
except:
continue
else:
continue
# just use the first part of "key" to check against our refSkel / relSkel (strip any leading .something and $something)
checkKey = key
if "." in checkKey:
checkKey = checkKey.split(".")[0]
if "$" in checkKey:
checkKey = checkKey.split("$")[0]
if _type == "dest":
#Ensure that the relational-filter is in refKeys
if checkKey not in self.refKeys:
logging.warning(
"Invalid filtering! %s is not in refKeys of RelationalBone %s!"
% (key, name))
raise RuntimeError()
# Iterate our relSkel and let these bones write their filters in
for bname, bone in relSkel.items():
if checkKey == bname:
newFilter = {key: value}
if self.multiple:
bone.buildDBFilter(bname,
relSkel,
dbFilter,
newFilter,
prefix=(prefix or "") +
"dest.")
else:
bone.buildDBFilter(bname,
relSkel,
dbFilter,
newFilter,
prefix=(prefix or "") +
name + ".dest.")
elif _type == "rel":
#Ensure that the relational-filter is in refKeys
if self.using is None or checkKey not in self.using():
logging.warning(
"Invalid filtering! %s is not a bone in 'using' of %s"
% (key, name))
raise RuntimeError()
# Iterate our usingSkel and let these bones write their filters in
for bname, bone in self.using().items():
if key.startswith(bname):
newFilter = {key: value}
if self.multiple:
bone.buildDBFilter(bname,
relSkel,
dbFilter,
newFilter,
prefix=(prefix or "") +
"rel.")
else:
bone.buildDBFilter(bname,
relSkel,
dbFilter,
newFilter,
prefix=(prefix or "") +
name + ".rel.")
if self.multiple:
dbFilter.setFilterHook(lambda s, filter, value: self.
filterHook(name, s, filter, value))
dbFilter.setOrderHook(
lambda s, orderings: self.orderHook(name, s, orderings))
elif name in rawFilter and rawFilter[name].lower() == "none":
dbFilter = dbFilter.filter("%s =" % name, None)
return dbFilter
def __init__(self,
kind=None,
module=None,
refKeys=None,
parentKeys=None,
multiple=False,
format="$(dest.name)",
using=None,
*args,
**kwargs):
"""
Initialize a new relationalBone.
:param kind: KindName of the referenced property.
:type kind: String
:param module: Name of the modul which should be used to select entities of kind "type". If not set,
the value of "type" will be used (the kindName must match the moduleName)
:type type: String
:param refKeys: A list of properties to include from the referenced property. These properties will be
avaiable in the template without having to fetch the referenced property. Filtering is also only possible
by properties named here!
:type refKeys: List of Strings
:param parentKeys: A list of properties from the current skeleton to include. If mixing filtering by
relational properties and properties of the class itself, these must be named here.
:type parentKeys: List of Strings
:param multiple: If True, allow referencing multiple Elements of the given class. (Eg. n:n-relation.
otherwise its n:1 )
:type multiple: False
:param format: Hint for the admin how to display such an relation. See admin/utils.py:formatString for
more information
:type format: String
"""
baseBone.__init__(self, *args, **kwargs)
self.multiple = multiple
self.format = format
#self._dbValue = None #Store the original result fetched from the db here so we have that information in case a referenced entity has been deleted
if kind:
self.kind = kind
if module:
self.module = module
elif self.kind:
self.module = self.kind
if self.kind is None or self.module is None:
raise NotImplementedError(
"Type and Module of relationalbone's must not be None")
if refKeys:
if not "key" in refKeys:
raise AttributeError("'key' must be included in refKeys!")
self.refKeys = refKeys
if parentKeys:
if not "key" in parentKeys:
raise AttributeError("'key' must be included in parentKeys!")
self.parentKeys = parentKeys
self.using = using
if getSystemInitialized():
from server.skeleton import RefSkel, skeletonByKind
self._refSkelCache = RefSkel.fromSkel(skeletonByKind(self.kind),
*self.refKeys)
self._usingSkelCache = using() if using else None
else:
self._refSkelCache = None
self._usingSkelCache = None
def setSystemInitialized(self):
super(relationalBone, self).setSystemInitialized()
from server.skeleton import RefSkel, skeletonByKind
self._refSkelCache = RefSkel.fromSkel(skeletonByKind(self.kind),
*self.refKeys)
self._usingSkelCache = self.using() if self.using else None
def renderBoneStructure(self, bone):
"""
Renders the structure of a bone.
This function is used by :func:`renderSkelStructure`.
can be overridden and super-called from a custom renderer.
:param bone: The bone which structure should be rendered.
:type bone: Any bone that inherits from :class:`server.bones.base.baseBone`.
:return: A dict containing the rendered attributes.
:rtype: dict
"""
# Base bone contents.
ret = {
"descr": _(bone.descr),
"type": bone.type,
"required": bone.required,
"params": bone.params,
"visible": bone.visible,
"readOnly": bone.readOnly
}
if bone.type == "relational" or bone.type.startswith("relational."):
if isinstance(bone, hierarchyBone):
boneType = "hierarchy"
elif isinstance(bone, treeItemBone):
boneType = "treeitem"
else:
boneType = "relational"
ret.update({
"type":
bone.type,
"module":
bone.module,
"multiple":
bone.multiple,
"format":
bone.format,
"using":
self.renderSkelStructure(bone.using()) if bone.using else None,
"relskel":
self.renderSkelStructure(
RefSkel.fromSkel(skeletonByKind(bone.kind), *bone.refKeys))
})
elif bone.type == "selectone" or bone.type.startswith(
"selectone."
) or bone.type == "selectmulti" or bone.type.startswith(
"selectmulti."):
ret.update({"values": {k: _(v) for k, v in bone.values.items()}})
elif bone.type == "date" or bone.type.startswith("date."):
ret.update({"date": bone.date, "time": bone.time})
elif bone.type == "numeric" or bone.type.startswith("numeric."):
ret.update({
"precision": bone.precision,
"min": bone.min,
"max": bone.max
})
elif bone.type == "text" or bone.type.startswith("text."):
ret.update({
"validHtml": bone.validHtml,
"languages": bone.languages
})
elif bone.type == "str" or bone.type.startswith("str."):
ret.update({
"languages": bone.languages,
"multiple": bone.multiple
})
return ret
def fromClient(self, valuesCache, name, data):
"""
Reads a value from the client.
If this value is valid for this bone,
store this value and return None.
Otherwise our previous value is
left unchanged and an error-message
is returned.
:param name: Our name in the skeleton
:type name: String
:param data: *User-supplied* request-data
:type data: Dict
:returns: None or String
"""
from server.skeleton import RefSkel, skeletonByKind
oldValues = valuesCache.get(name, None)
valuesCache[name] = []
tmpRes = {}
clientPrefix = "%s." % name
for k, v in data.items():
if k.startswith(clientPrefix) or k == name:
if k == name:
k = k.replace(name, "", 1)
else:
k = k.replace(clientPrefix, "", 1)
if "." in k:
try:
idx, bname = k.split(".", 1)
idx = int(idx)
except ValueError:
# We got some garbarge as input; don't try to parse it
continue
elif k.isdigit() and self.using is None:
idx = int(k)
bname = "key"
elif self.using is None and not self.multiple:
idx = 0
bname = "key"
else:
continue
if not idx in tmpRes.keys():
tmpRes[idx] = {}
if bname in tmpRes[idx].keys():
if isinstance(tmpRes[idx][bname], list):
tmpRes[idx][bname].append(v)
else:
tmpRes[idx][bname] = [tmpRes[idx][bname], v]
else:
tmpRes[idx][bname] = v
tmpList = [(k, v) for k, v in tmpRes.items() if "key" in v.keys()]
tmpList.sort(key=lambda k: k[0])
tmpList = [{
"reltmp": v,
"dest": {
"key": v["key"]
}
} for k, v in tmpList]
errorDict = {}
forceFail = False
if not tmpList and self.required:
return "No value selected!"
for r in tmpList[:]:
# Rebuild the referenced entity data
isEntryFromBackup = False #If the referenced entry has been deleted, restore information from
entry = None
try:
entry = db.Get(db.Key(r["dest"]["key"]))
except: #Invalid key or something like thatmnn
logging.info(
"Invalid reference key >%s< detected on bone '%s'",
r["dest"]["key"], name)
if isinstance(oldValues, dict):
if oldValues["dest"]["key"] == str(r["dest"]["key"]):
entry = oldValues["dest"].serialize()
isEntryFromBackup = True
elif isinstance(oldValues, list):
for dbVal in oldValues:
if dbVal["dest"]["key"] == str(r["dest"]["key"]):
entry = dbVal["dest"].serialize()
isEntryFromBackup = True
if not isEntryFromBackup:
if not self.multiple: #We can stop here :/
return ("Invalid entry selected")
else:
tmpList.remove(r)
continue
if not entry or (
not isEntryFromBackup
and not entry.key().kind() == self.kind
): #Entry does not exist or has wrong type (is from another module)
if entry:
logging.error(
"I got a key, which kind doesn't match my type! (Got: %s, my type %s)"
% (entry.key().kind(), self.kind))
tmpList.remove(r)
continue
tmp = {
k: entry[k]
for k in entry.keys()
if (k in self.refKeys
or any([k.startswith("%s." % x) for x in self.refKeys]))
}
tmp["key"] = r["dest"]["key"]
relSkel = RefSkel.fromSkel(skeletonByKind(self.kind),
*self.refKeys)
relSkel.unserialize(tmp)
r["dest"] = relSkel
# Rebuild the refSkel data
if self.using is not None:
refSkel = self.using()
if not refSkel.fromClient(r["reltmp"]):
for k, v in refSkel.errors.items():
errorDict["%s.%s.%s" % (name, tmpList.index(r), k)] = v
forceFail = True
r["rel"] = refSkel
else:
r["rel"] = None
del r["reltmp"]
if self.multiple:
cleanList = []
for item in tmpList:
err = self.isInvalid(item)
if err:
errorDict["%s.%s" % (name, tmpList.index(item))] = err
else:
cleanList.append(item)
if not cleanList:
errorDict[name] = "No value selected"
valuesCache[name] = tmpList
else:
if tmpList:
val = tmpList[0]
else:
val = None
err = self.isInvalid(val)
if not err:
valuesCache[name] = val
if val is None:
errorDict[name] = "No value selected"
if len(errorDict.keys()):
return ReadFromClientError(errorDict, forceFail)
def iterImport(module, target, exportKey, cursor=None, amount=0):
"""
Processes 100 Entries and calls the next batch
"""
urlfetch.set_default_fetch_deadline(20)
payload = {"module": module, "key": exportKey}
if cursor:
payload.update({"cursor": cursor})
result = urlfetch.fetch(
url=target,
payload=urllib.urlencode(payload),
method=urlfetch.POST,
headers={'Content-Type': 'application/x-www-form-urlencoded'})
if result.status_code == 200:
res = pickle.loads(result.content.decode("HEX"))
skel = skeletonByKind(module)()
logging.info("%s: %d new entries fetched, total %d entries fetched" %
(module, len(res["values"]), amount))
if len(res["values"]) == 0:
try:
utils.sendEMailToAdmins(
"Import of kind %s finished with %d entities" %
(module, amount), "ViUR finished to import %d entities of "
"kind %s from %s.\n" % (amount, module, target))
except: #OverQuota, whatever
logging.error("Unable to send Email")
return
for entry in res["values"]:
for k in list(entry.keys())[:]:
if isinstance(entry[k], str):
entry[k] = entry[k].decode("UTF-8")
if not "key" in entry.keys():
entry["key"] = entry["id"]
key = db.Key(encoded=utils.normalizeKey(entry["key"]))
# Special case: Convert old module root nodes!!!
if module.endswith(
"_rootNode") and key.name() and "_modul_" in key.name():
name = key.name().replace("_modul_", "_module_")
else:
name = key.name()
dbEntry = db.Entity(kind=key.kind(),
parent=key.parent(),
id=key.id(),
name=name)
for k in entry.keys():
if k == "key":
continue
dbEntry[k] = entry[k]
# Special case: Convert old module root nodes!!!
if (isinstance(skel, (HierarchySkel, TreeLeafSkel))
and k in ["parentdir", "parententry", "parentrepo"]
and entry[k]):
key = db.Key(encoded=str(entry[k]))
if key.parent():
parent = db.Key(
encoded=utils.normalizeKey(key.parent()))
else:
parent = None
if key.id_or_name() and "_modul_" in str(key.id_or_name()):
name = key.id_or_name().replace("_modul_", "_module_")
else:
name = key.id_or_name()
dbEntry[k] = str(
db.Key.from_path(key.kind(), name, parent=parent))
db.Put(dbEntry)
skel.fromDB(str(dbEntry.key()))
skel.refresh()
skel.toDB(clearUpdateTag=True)
amount += 1
iterImport(module, target, exportKey, res["cursor"], amount)
