def createNewUserIfNotExists(): """ Create a new Admin user, if the userDB is empty """ userMod = getattr(conf["viur.mainApp"], "user", None) if (userMod # We have a user module and isinstance(userMod, User) and "addSkel" in dir(userMod) and "validAuthenticationMethods" in dir(userMod) #Its our user module :) and any([x[0] is UserPassword for x in userMod.validAuthenticationMethods])): #It uses UserPassword login if not db.Query(userMod.addSkel().kindName).get(): #There's currently no user in the database addSkel = skeletonByKind(userMod.addSkel().kindName)() # Ensure we have the full skeleton uname = "admin@%s.appspot.com" % app_identity.get_application_id() pw = utils.generateRandomString(13) addSkel["name"] = uname addSkel["status"] = 10 # Ensure its enabled right away addSkel["access"] = ["root"] addSkel["password"] = pw try: addSkel.toDB() except Exception as e: logging.error("Something went wrong when trying to add admin user %s with Password %s", (uname, pw)) logging.exception(e) return logging.warning("ViUR created a new admin-user for you! Username: %s, Password: %s" % (uname, pw)) utils.sendEMailToAdmins("Your new ViUR password", "ViUR created a new admin-user for you! Username: %s, Password: %s" % (uname, pw))
def storeEntry2(self, e, key): if not self._checkKey(key, export=False): raise errors.Forbidden() entry = pickle.loads(e.decode("HEX")) for k in list(entry.keys())[:]: if isinstance(entry[k], str): entry[k] = entry[k].decode("UTF-8") key = db.Key(encoded=utils.normalizeKey(entry["key"])) logging.info(key.kind()) logging.info(key.id()) logging.info(key.name()) dbEntry = db.Entity(kind=key.kind(), parent=key.parent(), id=key.id(), name=key.name()) #maybe some more fixes here ? for k in entry.keys(): if k != "key": val = entry[k] dbEntry[k] = val db.Put(dbEntry) try: skel = skeletonByKind(key.kind())() except: logging.error("Unknown Skeleton - skipping") skel.fromDB(str(dbEntry.key())) skel.refresh() skel.toDB(clearUpdateTag=True)
def login(self, skey="", *args, **kwargs): if users.get_current_user(): addSkel = skeletonByKind(self.userModule.addSkel().kindName) # Ensure that we have the full skeleton currentUser = users.get_current_user() uid = currentUser.user_id() userSkel = addSkel().all().filter("uid =", uid).getSkel() if not userSkel: # We'll try again - checking if there's already an user with that email userSkel = addSkel().all().filter("name.idx =", currentUser.email().lower()).getSkel() if not userSkel: # Still no luck - it's a completely new user if not self.registrationEnabled and not users.is_current_user_admin(): # Registration is disabled, it's a new user and that user is not admin logging.warning("Denying registration of %s", currentUser.email()) raise errors.Forbidden("Registration for new users is disabled") userSkel = addSkel() # We'll add a new user userSkel["uid"] = uid userSkel["name"] = currentUser.email() isAdd = True else: isAdd = False now = datetime.datetime.now() if isAdd or (now-userSkel["lastlogin"]) > datetime.timedelta(minutes=30): # Conserve DB-Writes: Update the user max once in 30 Minutes userSkel["lastlogin"] = now if users.is_current_user_admin(): if not userSkel["access"]: userSkel["access"] = [] if not "root" in userSkel["access"]: userSkel["access"].append("root") userSkel["gaeadmin"] = True else: userSkel["gaeadmin"] = False assert userSkel.toDB() return self.userModule.continueAuthenticationFlow(self, userSkel["key"]) raise errors.Redirect( users.create_login_url( self.modulePath+"/login") )
def getCfg(self, module, key): if not self._checkKey(key, export=False): raise errors.Forbidden() skel = skeletonByKind(module) assert skel is not None res = skel() r = DefaultRender() return (pickle.dumps(r.renderSkelStructure(res)))
def doDeleteWeakReferences(timeStamp, cursor): skelCls = skeletonByKind("file") gotAtLeastOne = False query = skelCls().all().filter("weak =", True).filter("creationdate <", datetime.strptime(timeStamp,"%d.%m.%Y %H:%M:%S")).cursor(cursor) for skel in query.fetch(99): # FIXME: Is that still needed? See hotfix/weakfile anyRel = any(db.Query("viur-relations").filter("dest.key =", skel["key"]).run(1, keysOnly=True)) if anyRel: logging.debug("doDeleteWeakReferences: found relations with that file - don't delete!") continue gotAtLeastOne = True skel.delete() newCursor = query.getCursor() if gotAtLeastOne and newCursor and newCursor.urlsafe() != cursor: doDeleteWeakReferences(timeStamp, newCursor.urlsafe())
def relSkelFromKey(key): if not isinstance(key, db.Key): key = db.Key(encoded=key) if not key.kind() == self.kind: logging.error( "I got a key, which kind doesn't match my type! (Got: %s, my type %s)" % (key.kind(), self.kind)) return None entity = db.Get(key) if not entity: logging.error("Key %s not found" % str(key)) return None relSkel = RefSkel.fromSkel(skeletonByKind(self.kind), *self.refKeys) relSkel.unserialize(entity) return relSkel
def _resolveSkelCls(self, *args, **kwargs): """ Retrieve the generally associated :class:`server.skeleton.Skeleton` that is used by the application. This is either be defined by the member variable *kindName* or by a Skeleton named like the application class in lower-case order. If this behavior is not wanted, it can be definitely overridden by defining module-specific :func:`viewSkel`,:func:`addSkel`, or :func:`editSkel` functions, or by overriding this function in general. :return: Returns a Skeleton instance that matches the application. :rtype: server.skeleton.Skeleton """ return skeletonByKind(self.kindName if self.kindName else unicode( type(self).__name__).lower())
def iterExport(module, target, importKey, cursor=None): """ Processes 100 Entries and calls the next batch """ urlfetch.set_default_fetch_deadline(20) Skel = skeletonByKind(module) if not Skel: logging.error("TaskExportKind: Invalid module") return query = Skel().all().cursor(cursor) startCursor = cursor query.run(100, keysOnly=True) endCursor = query.getCursor().urlsafe() exportItems(module, target, importKey, startCursor, endCursor) if startCursor is None or startCursor != endCursor: iterExport(module, target, importKey, endCursor)
def getSkelForRequest(skelName, key=None, attr=None): """ Retrieves a skeleton with a key and caches it in the current request. If no key is given, the function tries to retrieve the key for the requested object from the request parameters, by preceding an "@" sign to the skelName as parameter. So adding "@project=abc" will try to load a "project"-Skeleton with the key "abc". """ #print("getSkelForRequest", skelName, key, attr) if key is None: key = request.current.get().kwargs.get("@%s" % skelName) if key is None or not key: return None reqData = request.current.requestData() if "%s.%s" % (skelName, key) in reqData.keys(): skel = reqData["%s.%s" % (skelName, key)] if skel and attr is not None: return skel[attr] return skel skel = skeleton.skeletonByKind(skelName) if skel: skel = skel() if skel.fromDB(key): skel = skel.clone() else: skel = None reqData["%s.%s" % (skelName, key)] = skel if skel and attr is not None: return skel[attr] return skel
def exportItems(module, target, importKey, startCursor, endCursor): Skel = skeletonByKind(module) query = Skel().all().cursor(startCursor, endCursor) for item in query.run(250): flatItem = DbTransfer.genDict(item) formFields = { "e": pickle.dumps(flatItem).encode("HEX"), "key": importKey } result = urlfetch.fetch( url=target, payload=urllib.urlencode(formFields), method=urlfetch.POST, headers={'Content-Type': 'application/x-www-form-urlencoded'}) if startCursor == endCursor: try: utils.sendEMailToAdmins( "Export of kind %s finished" % module, "ViUR finished to export kind %s to %s.\n" % (module, target)) except: #OverQuota, whatever pass
def _restoreValueFromDatastore(self, val): """ Restores one of our values (including the Rel- and Using-Skel) from the serialized data read from the datastore :param value: Json-Encoded datastore property :return: Our Value (with restored RelSkel and using-Skel) """ value = extjson.loads(val) from server.skeleton import RefSkel, skeletonByKind assert isinstance( value, dict ), "Read something from the datastore thats not a dict: %s" % str( type(value)) relSkel = RefSkel.fromSkel(skeletonByKind(self.kind), *self.refKeys) # !!!ViUR re-design compatibility!!! if not "dest" in value.keys(): nvalue = dict() nvalue["dest"] = value value = nvalue if "id" in value["dest"].keys() and not ("key" in value["dest"].keys() and value["dest"]["key"]): value["dest"]["key"] = value["dest"]["id"] del value["dest"]["id"] # UNTIL HERE! relSkel.unserialize(value["dest"]) if self.using is not None: usingSkel = self.using() if value["rel"] is not None: usingSkel.unserialize(value["rel"]) else: usingSkel = None return {"dest": relSkel, "rel": usingSkel}
def buildDBFilter(self, name, skel, dbFilter, rawFilter, prefix=None): from server.skeleton import RefSkel, skeletonByKind origFilter = dbFilter.datastoreQuery if origFilter is None: #This query is unsatisfiable return (dbFilter) myKeys = [x for x in rawFilter.keys() if x.startswith("%s." % name)] if len(myKeys) > 0 and not self.indexed: logging.warning("Invalid searchfilter! %s is not indexed!" % name) raise RuntimeError() if len(myKeys) > 0: #We filter by some properties if dbFilter.getKind() != "viur-relations" and self.multiple: name, skel, dbFilter, rawFilter = self._rewriteQuery( name, skel, dbFilter, rawFilter) relSkel = RefSkel.fromSkel(skeletonByKind(self.kind), *self.refKeys) # Merge the relational filters in for myKey in myKeys: value = rawFilter[myKey] try: unused, _type, key = myKey.split(".", 2) assert _type in ["dest", "rel"] except: if self.using is None: # This will be a "dest" query _type = "dest" try: unused, key = myKey.split(".", 1) except: continue else: continue # just use the first part of "key" to check against our refSkel / relSkel (strip any leading .something and $something) checkKey = key if "." in checkKey: checkKey = checkKey.split(".")[0] if "$" in checkKey: checkKey = checkKey.split("$")[0] if _type == "dest": #Ensure that the relational-filter is in refKeys if checkKey not in self.refKeys: logging.warning( "Invalid filtering! %s is not in refKeys of RelationalBone %s!" % (key, name)) raise RuntimeError() # Iterate our relSkel and let these bones write their filters in for bname, bone in relSkel.items(): if checkKey == bname: newFilter = {key: value} if self.multiple: bone.buildDBFilter(bname, relSkel, dbFilter, newFilter, prefix=(prefix or "") + "dest.") else: bone.buildDBFilter(bname, relSkel, dbFilter, newFilter, prefix=(prefix or "") + name + ".dest.") elif _type == "rel": #Ensure that the relational-filter is in refKeys if self.using is None or checkKey not in self.using(): logging.warning( "Invalid filtering! %s is not a bone in 'using' of %s" % (key, name)) raise RuntimeError() # Iterate our usingSkel and let these bones write their filters in for bname, bone in self.using().items(): if key.startswith(bname): newFilter = {key: value} if self.multiple: bone.buildDBFilter(bname, relSkel, dbFilter, newFilter, prefix=(prefix or "") + "rel.") else: bone.buildDBFilter(bname, relSkel, dbFilter, newFilter, prefix=(prefix or "") + name + ".rel.") if self.multiple: dbFilter.setFilterHook(lambda s, filter, value: self. filterHook(name, s, filter, value)) dbFilter.setOrderHook( lambda s, orderings: self.orderHook(name, s, orderings)) elif name in rawFilter and rawFilter[name].lower() == "none": dbFilter = dbFilter.filter("%s =" % name, None) return dbFilter
def __init__(self, kind=None, module=None, refKeys=None, parentKeys=None, multiple=False, format="$(dest.name)", using=None, *args, **kwargs): """ Initialize a new relationalBone. :param kind: KindName of the referenced property. :type kind: String :param module: Name of the modul which should be used to select entities of kind "type". If not set, the value of "type" will be used (the kindName must match the moduleName) :type type: String :param refKeys: A list of properties to include from the referenced property. These properties will be avaiable in the template without having to fetch the referenced property. Filtering is also only possible by properties named here! :type refKeys: List of Strings :param parentKeys: A list of properties from the current skeleton to include. If mixing filtering by relational properties and properties of the class itself, these must be named here. :type parentKeys: List of Strings :param multiple: If True, allow referencing multiple Elements of the given class. (Eg. n:n-relation. otherwise its n:1 ) :type multiple: False :param format: Hint for the admin how to display such an relation. See admin/utils.py:formatString for more information :type format: String """ baseBone.__init__(self, *args, **kwargs) self.multiple = multiple self.format = format #self._dbValue = None #Store the original result fetched from the db here so we have that information in case a referenced entity has been deleted if kind: self.kind = kind if module: self.module = module elif self.kind: self.module = self.kind if self.kind is None or self.module is None: raise NotImplementedError( "Type and Module of relationalbone's must not be None") if refKeys: if not "key" in refKeys: raise AttributeError("'key' must be included in refKeys!") self.refKeys = refKeys if parentKeys: if not "key" in parentKeys: raise AttributeError("'key' must be included in parentKeys!") self.parentKeys = parentKeys self.using = using if getSystemInitialized(): from server.skeleton import RefSkel, skeletonByKind self._refSkelCache = RefSkel.fromSkel(skeletonByKind(self.kind), *self.refKeys) self._usingSkelCache = using() if using else None else: self._refSkelCache = None self._usingSkelCache = None
def setSystemInitialized(self): super(relationalBone, self).setSystemInitialized() from server.skeleton import RefSkel, skeletonByKind self._refSkelCache = RefSkel.fromSkel(skeletonByKind(self.kind), *self.refKeys) self._usingSkelCache = self.using() if self.using else None
def renderBoneStructure(self, bone): """ Renders the structure of a bone. This function is used by :func:`renderSkelStructure`. can be overridden and super-called from a custom renderer. :param bone: The bone which structure should be rendered. :type bone: Any bone that inherits from :class:`server.bones.base.baseBone`. :return: A dict containing the rendered attributes. :rtype: dict """ # Base bone contents. ret = { "descr": _(bone.descr), "type": bone.type, "required": bone.required, "params": bone.params, "visible": bone.visible, "readOnly": bone.readOnly } if bone.type == "relational" or bone.type.startswith("relational."): if isinstance(bone, hierarchyBone): boneType = "hierarchy" elif isinstance(bone, treeItemBone): boneType = "treeitem" else: boneType = "relational" ret.update({ "type": bone.type, "module": bone.module, "multiple": bone.multiple, "format": bone.format, "using": self.renderSkelStructure(bone.using()) if bone.using else None, "relskel": self.renderSkelStructure( RefSkel.fromSkel(skeletonByKind(bone.kind), *bone.refKeys)) }) elif bone.type == "selectone" or bone.type.startswith( "selectone." ) or bone.type == "selectmulti" or bone.type.startswith( "selectmulti."): ret.update({"values": {k: _(v) for k, v in bone.values.items()}}) elif bone.type == "date" or bone.type.startswith("date."): ret.update({"date": bone.date, "time": bone.time}) elif bone.type == "numeric" or bone.type.startswith("numeric."): ret.update({ "precision": bone.precision, "min": bone.min, "max": bone.max }) elif bone.type == "text" or bone.type.startswith("text."): ret.update({ "validHtml": bone.validHtml, "languages": bone.languages }) elif bone.type == "str" or bone.type.startswith("str."): ret.update({ "languages": bone.languages, "multiple": bone.multiple }) return ret
def fromClient(self, valuesCache, name, data): """ Reads a value from the client. If this value is valid for this bone, store this value and return None. Otherwise our previous value is left unchanged and an error-message is returned. :param name: Our name in the skeleton :type name: String :param data: *User-supplied* request-data :type data: Dict :returns: None or String """ from server.skeleton import RefSkel, skeletonByKind oldValues = valuesCache.get(name, None) valuesCache[name] = [] tmpRes = {} clientPrefix = "%s." % name for k, v in data.items(): if k.startswith(clientPrefix) or k == name: if k == name: k = k.replace(name, "", 1) else: k = k.replace(clientPrefix, "", 1) if "." in k: try: idx, bname = k.split(".", 1) idx = int(idx) except ValueError: # We got some garbarge as input; don't try to parse it continue elif k.isdigit() and self.using is None: idx = int(k) bname = "key" elif self.using is None and not self.multiple: idx = 0 bname = "key" else: continue if not idx in tmpRes.keys(): tmpRes[idx] = {} if bname in tmpRes[idx].keys(): if isinstance(tmpRes[idx][bname], list): tmpRes[idx][bname].append(v) else: tmpRes[idx][bname] = [tmpRes[idx][bname], v] else: tmpRes[idx][bname] = v tmpList = [(k, v) for k, v in tmpRes.items() if "key" in v.keys()] tmpList.sort(key=lambda k: k[0]) tmpList = [{ "reltmp": v, "dest": { "key": v["key"] } } for k, v in tmpList] errorDict = {} forceFail = False if not tmpList and self.required: return "No value selected!" for r in tmpList[:]: # Rebuild the referenced entity data isEntryFromBackup = False #If the referenced entry has been deleted, restore information from entry = None try: entry = db.Get(db.Key(r["dest"]["key"])) except: #Invalid key or something like thatmnn logging.info( "Invalid reference key >%s< detected on bone '%s'", r["dest"]["key"], name) if isinstance(oldValues, dict): if oldValues["dest"]["key"] == str(r["dest"]["key"]): entry = oldValues["dest"].serialize() isEntryFromBackup = True elif isinstance(oldValues, list): for dbVal in oldValues: if dbVal["dest"]["key"] == str(r["dest"]["key"]): entry = dbVal["dest"].serialize() isEntryFromBackup = True if not isEntryFromBackup: if not self.multiple: #We can stop here :/ return ("Invalid entry selected") else: tmpList.remove(r) continue if not entry or ( not isEntryFromBackup and not entry.key().kind() == self.kind ): #Entry does not exist or has wrong type (is from another module) if entry: logging.error( "I got a key, which kind doesn't match my type! (Got: %s, my type %s)" % (entry.key().kind(), self.kind)) tmpList.remove(r) continue tmp = { k: entry[k] for k in entry.keys() if (k in self.refKeys or any([k.startswith("%s." % x) for x in self.refKeys])) } tmp["key"] = r["dest"]["key"] relSkel = RefSkel.fromSkel(skeletonByKind(self.kind), *self.refKeys) relSkel.unserialize(tmp) r["dest"] = relSkel # Rebuild the refSkel data if self.using is not None: refSkel = self.using() if not refSkel.fromClient(r["reltmp"]): for k, v in refSkel.errors.items(): errorDict["%s.%s.%s" % (name, tmpList.index(r), k)] = v forceFail = True r["rel"] = refSkel else: r["rel"] = None del r["reltmp"] if self.multiple: cleanList = [] for item in tmpList: err = self.isInvalid(item) if err: errorDict["%s.%s" % (name, tmpList.index(item))] = err else: cleanList.append(item) if not cleanList: errorDict[name] = "No value selected" valuesCache[name] = tmpList else: if tmpList: val = tmpList[0] else: val = None err = self.isInvalid(val) if not err: valuesCache[name] = val if val is None: errorDict[name] = "No value selected" if len(errorDict.keys()): return ReadFromClientError(errorDict, forceFail)
def iterImport(module, target, exportKey, cursor=None, amount=0): """ Processes 100 Entries and calls the next batch """ urlfetch.set_default_fetch_deadline(20) payload = {"module": module, "key": exportKey} if cursor: payload.update({"cursor": cursor}) result = urlfetch.fetch( url=target, payload=urllib.urlencode(payload), method=urlfetch.POST, headers={'Content-Type': 'application/x-www-form-urlencoded'}) if result.status_code == 200: res = pickle.loads(result.content.decode("HEX")) skel = skeletonByKind(module)() logging.info("%s: %d new entries fetched, total %d entries fetched" % (module, len(res["values"]), amount)) if len(res["values"]) == 0: try: utils.sendEMailToAdmins( "Import of kind %s finished with %d entities" % (module, amount), "ViUR finished to import %d entities of " "kind %s from %s.\n" % (amount, module, target)) except: #OverQuota, whatever logging.error("Unable to send Email") return for entry in res["values"]: for k in list(entry.keys())[:]: if isinstance(entry[k], str): entry[k] = entry[k].decode("UTF-8") if not "key" in entry.keys(): entry["key"] = entry["id"] key = db.Key(encoded=utils.normalizeKey(entry["key"])) # Special case: Convert old module root nodes!!! if module.endswith( "_rootNode") and key.name() and "_modul_" in key.name(): name = key.name().replace("_modul_", "_module_") else: name = key.name() dbEntry = db.Entity(kind=key.kind(), parent=key.parent(), id=key.id(), name=name) for k in entry.keys(): if k == "key": continue dbEntry[k] = entry[k] # Special case: Convert old module root nodes!!! if (isinstance(skel, (HierarchySkel, TreeLeafSkel)) and k in ["parentdir", "parententry", "parentrepo"] and entry[k]): key = db.Key(encoded=str(entry[k])) if key.parent(): parent = db.Key( encoded=utils.normalizeKey(key.parent())) else: parent = None if key.id_or_name() and "_modul_" in str(key.id_or_name()): name = key.id_or_name().replace("_modul_", "_module_") else: name = key.id_or_name() dbEntry[k] = str( db.Key.from_path(key.kind(), name, parent=parent)) db.Put(dbEntry) skel.fromDB(str(dbEntry.key())) skel.refresh() skel.toDB(clearUpdateTag=True) amount += 1 iterImport(module, target, exportKey, res["cursor"], amount)