def doCheckForUnreferencedBlobs(cursor = None): def getOldBlobKeysTxn(dbKey): obj = db.Get(dbKey) res = obj["old_blob_references"] or [] if obj["is_stale"]: db.Delete(dbKey) else: obj["has_old_blob_references"] = False obj["old_blob_references"] = [] db.Put(obj) return res gotAtLeastOne = False query = db.Query("viur-blob-locks").filter("has_old_blob_references", True).cursor(cursor) for lockKey in query.run(100, keysOnly=True): gotAtLeastOne = True oldBlobKeys = db.RunInTransaction( getOldBlobKeysTxn, lockKey ) for blobKey in oldBlobKeys: if db.Query("viur-blob-locks").filter("active_blob_references =", blobKey).get(): #This blob is referenced elsewhere logging.info("Stale blob is still referenced, %s" % blobKey) continue # Add a marker and schedule it for deletion fileObj = db.Query( "viur-deleted-files" ).filter( "dlkey", blobKey ).get() if fileObj: #Its already marked logging.info("Stale blob already marked for deletion, %s" % blobKey) return fileObj = db.Entity("viur-deleted-files") fileObj["itercount"] = 0 fileObj["dlkey"] = str(blobKey) logging.info("Stale blob marked dirty, %s" % blobKey) db.Put(fileObj) newCursor = query.getCursor() if gotAtLeastOne and newCursor and newCursor.urlsafe() != cursor: doCheckForUnreferencedBlobs(newCursor.urlsafe())
def checkStatus(self): from pprint import pformat userList = list() for tmp in userSkel().all().filter("nick >", "user").order( ("nick", db.ASCENDING)).run(limit=6): tmp = u"<li><pre>%s</pre></li>" % pformat( tmp, indent=4, width=80, depth=None) userList.append(tmp) updateList = db.Query("viur-relations").filter( "viur_src_kind =", "report").order( ("viur_src_property", db.ASCENDING)).run(limit=1000) tpl = u"<html><body><h3>Users</h3><ul>%s</ul><h3>Report Relations</h3><ul>%s</ul><h3>Report</h3><ul>%s</ul></body></html>" relationsList = list() for srcRel in sorted(updateList, key=lambda x: (x["viur_src_property"], x["dest.name"])): tmp = srcRel tmp = u"<li><pre>%s</pre></li>" % pformat( tmp, indent=4, width=80, depth=None) relationsList.append(tmp) reportList = list() for entry in db.Query("report").run(limit=100): reportList.append(u"<li><pre>%s</pre></li>" % pformat(entry, indent=4, width=80, depth=None)) return tpl % (u"".join(userList), u"".join(relationsList), u"".join(reportList))
def doCleanupDeletedFiles(cursor = None): maxIterCount = 2 # How often a file will be checked for deletion gotAtLeastOne = False query = db.Query("viur-deleted-files").cursor( cursor ) for file in query.run(100): gotAtLeastOne = True if not "dlkey" in file.keys(): db.Delete(file.key()) elif db.Query("viur-blob-locks").filter("active_blob_references =", file["dlkey"]).get(): logging.info("is referenced, %s" % file["dlkey"]) db.Delete(file.key()) else: if file["itercount"] > maxIterCount: logging.info("Finally deleting, %s" % file["dlkey"]) blobstore.delete(file["dlkey"]) db.Delete(file.key()) # There should be exactly 1 or 0 of these for f in db.Query("file").filter("dlkey =", file["dlkey"]).iter(keysOnly=True): db.Delete(f) else: logging.error("Increasing count, %s" % file["dlkey"]) file["itercount"] += 1 db.Put(file) newCursor = query.getCursor() if gotAtLeastOne and newCursor and newCursor.urlsafe() != cursor: doCleanupDeletedFiles(newCursor.urlsafe())
def deleteRecursive(self, nodeKey): """ Recursively processes a delete request. This will delete all entries which are children of *nodeKey*, except *key* nodeKey. :param key: URL-safe key of the node which children should be deleted. :type key: str :returns: The number of deleted objects. :rtype: int """ count = 0 for f in db.Query(self.viewLeafSkel().kindName).filter( "parentdir", str(nodeKey)).iter(keysOnly=True): s = self.viewLeafSkel() if not s.fromDB(f): continue s.delete() count += 1 for d in db.Query(self.viewNodeSkel().kindName).filter( "parentdir", str(nodeKey)).iter(keysOnly=True): count += self.deleteRecursive(str(d)) s = self.viewNodeSkel() if not s.fromDB(d): continue s.delete() count += 1 return count
def flushCache(prefix="/*"): """ Flushes the cache. Its possible the flush only a part of the cache by specifying the path-prefix. :param prefix: Path or prefix that should be flushed. :type prefix: str Examples: - "/" would flush the main page (and only that), - "/*" everything from the cache, "/page/*" everything from the page-module (default render), - and "/page/view/*" only that specific subset of the page-module. """ items = db.Query(viurCacheName).filter( "path =", prefix.rstrip("*")).iter(keysOnly=True) for item in items: db.Delete(item) if prefix.endswith("*"): items = db.Query(viurCacheName).filter( "path >", prefix.rstrip("*")).filter( "path <", prefix.rstrip("*") + u"\ufffd").iter(keysOnly=True) for item in items: db.Delete(item) logging.debug( "Flushing cache succeeded. Everything matching \"%s\" is gone." % prefix)
def deleteRecursive(self, parentKey): files = db.Query( self.editLeafSkel().kindName ).filter("parentdir =", parentKey).iter() for fileEntry in files: utils.markFileForDeletion(fileEntry["dlkey"]) skel = self.editLeafSkel() if skel.fromDB(str(fileEntry.key())): skel.delete() dirs = db.Query(self.editNodeSkel().kindName).filter("parentdir", parentKey).iter(keysOnly=True) for d in dirs: self.deleteRecursive(str(d)) skel = self.editNodeSkel() if skel.fromDB(str(d)): skel.delete()
def createNewUserIfNotExists(): """ Create a new Admin user, if the userDB is empty """ userMod = getattr(conf["viur.mainApp"], "user", None) if (userMod # We have a user module and isinstance(userMod, User) and "addSkel" in dir(userMod) and "validAuthenticationMethods" in dir(userMod) #Its our user module :) and any([x[0] is UserPassword for x in userMod.validAuthenticationMethods])): #It uses UserPassword login if not db.Query(userMod.addSkel().kindName).get(): #There's currently no user in the database addSkel = skeletonByKind(userMod.addSkel().kindName)() # Ensure we have the full skeleton uname = "admin@%s.appspot.com" % app_identity.get_application_id() pw = utils.generateRandomString(13) addSkel["name"] = uname addSkel["status"] = 10 # Ensure its enabled right away addSkel["access"] = ["root"] addSkel["password"] = pw try: addSkel.toDB() except Exception as e: logging.error("Something went wrong when trying to add admin user %s with Password %s", (uname, pw)) logging.exception(e) return logging.warning("ViUR created a new admin-user for you! Username: %s, Password: %s" % (uname, pw)) utils.sendEMailToAdmins("Your new ViUR password", "ViUR created a new admin-user for you! Username: %s, Password: %s" % (uname, pw))
def login(self, name=None, password=None, skey="", *args, **kwargs): if self.userModule.getCurrentUser(): #Were already logged in return self.userModule.render.loginSucceeded() if not name or not password or not securitykey.validate(skey): return self.userModule.render.login(self.loginSkel()) query = db.Query(self.userModule.viewSkel().kindName) res = query.filter("name.idx >=", name.lower()).get() if res is None: res = {"password": "", "status": 0, "name": "", "name.idx": ""} if "password_salt" in res.keys(): # Its the new, more secure passwd passwd = pbkdf2(password[:conf["viur.maxPasswordLength"]], res["password_salt"]) else: passwd = sha512(password.encode("UTF-8") + conf["viur.salt"]).hexdigest() isOkay = True # We do this exactly that way to avoid timing attacks # Check if the username matches storedUserName = res.get("name.idx", "") if len(storedUserName) != len(name.lower()): isOkay = False else: for x, y in izip(storedUserName, name.lower()): if x != y: isOkay = False # Check if the password matches storedPasswordHash = res.get("password", "") if len(storedPasswordHash) != len(passwd): isOkay = False else: for x, y in izip(storedPasswordHash, passwd): if x != y: isOkay = False # Verify that this account isn't blocked if res["status"] < 10: isOkay = False if not isOkay: skel = self.loginSkel() skel.fromClient({"name": name, "nomissing": "1"}) return self.userModule.render.login(skel, loginFailed=True) else: if not "password_salt" in res.keys( ): #Update the password to the new, more secure format res["password_salt"] = utils.generateRandomString(13) res["password"] = pbkdf2( password[:conf["viur.maxPasswordLength"]], res["password_salt"]) db.Put(res) return self.userModule.continueAuthenticationFlow(self, res.key())
def updateRelations(destID, minChangeTime, cursor=None): logging.debug("Starting updateRelations for %s ; minChangeTime %s", destID, minChangeTime) updateListQuery = db.Query("viur-relations").filter( "dest.key =", destID).filter("viur_delayed_update_tag <", minChangeTime) if cursor: updateListQuery.cursor(cursor) updateList = updateListQuery.run(limit=5) for srcRel in updateList: try: skel = skeletonByKind(srcRel["viur_src_kind"])() except AssertionError: logging.info("Deleting %s which refers to unknown kind %s" % (str(srcRel.key()), srcRel["viur_src_kind"])) continue if not skel.fromDB(str(srcRel.key().parent())): logging.warning( "Cannot update stale reference to %s (referenced from %s)" % (str(srcRel.key().parent()), str(srcRel.key()))) continue for key, _bone in skel.items(): _bone.refresh(skel.valuesCache, key, skel) skel.toDB(clearUpdateTag=True) if len(updateList) == 5: updateRelations(destID, minChangeTime, updateListQuery.getCursor().urlsafe())
def all(self): """ Create a query with the current Skeletons kindName. :returns: A db.Query object which allows for entity filtering and sorting. :rtype: :class:`server.db.Query` """ return (db.Query(self.kindName, srcSkelClass=self))
def doClearSessions(timeStamp, cursor): gotAtLeastOne = False query = db.Query(GaeSession.kindName).filter("lastseen <", timeStamp) for oldKey in query.run(100, keysOnly=True): gotAtLeastOne = True db.Delete(oldKey) newCursor = query.getCursor() if gotAtLeastOne and newCursor and newCursor.urlsafe() != cursor: doClearSessions(timeStamp, newCursor.urlsafe())
def getAvailableRootNodes(self, name, *args, **kwargs): thisuser = utils.getCurrentUser() if not thisuser: return [] repo = self.ensureOwnUserRootNode() res = [{"name": _("My Files"), "key": str(repo.key())}] if "root" in thisuser["access"]: # Add at least some repos from other users repos = db.Query(self.viewNodeSkel.kindName + "_rootNode").filter( "type =", "user").run(100) for repo in repos: if not "user" in repo: continue user = db.Query("user").filter("uid =", repo.user).get() if not user or not "name" in user: continue res.append({"name": user["name"], "key": str(repo.key())}) return res
def doClearSKeys(timeStamp, cursor): gotAtLeastOne = False query = db.Query(securityKeyKindName).filter( "until <", datetime.strptime(timeStamp, "%d.%m.%Y %H:%M:%S")) for oldKey in query.run(100, keysOnly=True): gotAtLeastOne = True db.Delete(oldKey) newCursor = query.getCursor() if gotAtLeastOne and newCursor and newCursor.urlsafe() != cursor: doClearSKeys(timeStamp, newCursor.urlsafe())
def index(self, *args, **kwargs): global _callableTasks, _periodicTasks logging.debug("Starting maintenance-run") checkUpdate() #Let the update-module verify the database layout first logging.debug("Updatecheck complete") for task, intervall in _periodicTasks.items( ): #Call all periodic tasks if intervall: #Ensure this task doesn't get called to often try: lastCall = db.Get( db.Key.from_path("viur-task-interval", task.periodicTaskName)) if lastCall["date"] > datetime.now() - timedelta( minutes=intervall): logging.debug( "Skipping task %s - Has already run recently." % task.periodicTaskName) continue except db.EntityNotFoundError: pass res = self.findBoundTask(task) if res: #Its bound, call it this way :) res[0]() else: task() #It seems it wasnt bound - call it as a static method logging.debug("Successfully called task %s" % task.periodicTaskName) if intervall: # Update its last-call timestamp entry = db.Entity("viur-task-interval", name=task.periodicTaskName) entry["date"] = datetime.now() db.Put(entry) logging.debug("Periodic tasks complete") for currentTask in db.Query( "viur-queued-tasks").iter(): #Look for queued tasks db.Delete(currentTask.key()) if currentTask["taskid"] in _callableTasks: task = _callableTasks[currentTask["taskid"]]() tmpDict = {} for k in currentTask.keys(): if k == "taskid": continue tmpDict[k] = json.loads(currentTask[k]) try: task.execute(**tmpDict) except Exception as e: logging.error("Error executing Task") logging.exception(e) logging.debug("Scheduled tasks complete")
def checkStatusJson(self): userList = list() for entry in userSkel().all().filter("nick >", "user").order( ("nick", db.ASCENDING)).run(limit=10): entry["creationdate"] = entry["creationdate"].strftime( "%Y.%m.%d %H:%M:%S") entry["changedate"] = entry["changedate"].strftime( "%Y.%m.%d %H:%M:%S") userList.append(entry) updateList = db.Query("viur-relations").filter( "viur_src_kind =", "report").order( ("dest.firstname", db.ASCENDING)).run(limit=1000) relationsList = list() for entry in sorted(updateList, key=lambda x: (x["viur_src_property"], x["dest.name"])): relationsList.append(entry) reportList = db.Query("report").run(limit=1) entry = None if reportList: entry = reportList[0] entry["creationdate"] = entry["creationdate"].strftime( "%Y.%m.%d %H:%M:%S") entry["changedate"] = entry["changedate"].strftime( "%Y.%m.%d %H:%M:%S") request.current.get( ).response.headers['content-type'] = 'application/json' return json.dumps( { "userList": userList, "relationList": relationsList, "report": entry }, indent=4)
def updateParentRepo(self, parentNode, newRepoKey, depth=0): """ Recursively fixes the parentrepo key after a move operation. This will delete all entries which are children of *nodeKey*, except *key* nodeKey. :param parentNode: URL-safe key of the node which children should be fixed. :type parentNode: str :param newNode: URL-safe key of the new repository. :type newNode: strg :param depth: Safety level depth preventing infinitive loops. :type depth: int """ if depth > 99: logging.critical( "Maximum recursion depth reached in server.applications.tree/fixParentRepo" ) logging.critical("Your data is corrupt!") logging.critical("Params: parentNode: %s, newRepoKey: %s" % (parentNode, newRepoKey)) return def fixTxn(nodeKey, newRepoKey): node = db.Get(nodeKey) node["parentrepo"] = newRepoKey db.Put(node) # Fix all nodes for repo in db.Query(self.viewNodeSkel().kindName).filter( "parentdir =", parentNode).iter(keysOnly=True): self.updateParentRepo(str(repo), newRepoKey, depth=depth + 1) db.RunInTransaction(fixTxn, str(repo), newRepoKey) # Fix the leafs on this level for repo in db.Query(self.viewLeafSkel().kindName).filter( "parentdir =", parentNode).iter(keysOnly=True): db.RunInTransaction(fixTxn, str(repo), newRepoKey)
def doDeleteWeakReferences(timeStamp, cursor): skelCls = skeletonByKind("file") gotAtLeastOne = False query = skelCls().all().filter("weak =", True).filter("creationdate <", datetime.strptime(timeStamp,"%d.%m.%Y %H:%M:%S")).cursor(cursor) for skel in query.fetch(99): # FIXME: Is that still needed? See hotfix/weakfile anyRel = any(db.Query("viur-relations").filter("dest.key =", skel["key"]).run(1, keysOnly=True)) if anyRel: logging.debug("doDeleteWeakReferences: found relations with that file - don't delete!") continue gotAtLeastOne = True skel.delete() newCursor = query.getCursor() if gotAtLeastOne and newCursor and newCursor.urlsafe() != cursor: doDeleteWeakReferences(timeStamp, newCursor.urlsafe())
def doPayPal(self, token, PayerID, *args, **kwargs): order = db.Query(self.orderHandler.viewSkel().kindName).filter( "paypal_token =", token).get() if not order: return ("NO SUCH ORDER - PAYMENT ABORTED") paypal = PayPal.PayPalHandler() res = paypal.DoExpressCheckoutPayment(token, PayerID, "%.2f" % float(order["price"])) if res["ACK"].lower() == u"success": self.orderHandler.setPayed(str(order.key())) return self.orderHandler.render.getEnv().get_template( self.orderHandler.render.getTemplateFileName( "paypal_okay")).render() else: return self.orderHandler.render.getEnv().get_template( self.orderHandler.render.getTemplateFileName( "paypal_failed")).render()
def iterValues2(self, module, cursor=None, key=None): if not self._checkKey(key, export=True): raise errors.Forbidden() q = db.Query(module) if cursor: q.cursor(cursor) r = [] for res in q.run(limit=32): r.append(self.genDict(res)) return pickle.dumps({ "cursor": str(q.getCursor().urlsafe()), "values": r }).encode("HEX")
def killSessionByUser(user=None): """ Invalidates all active sessions for the given *user*. This means that this user is instantly logged out. If no user is given, it tries to invalidate **all** active sessions. Use "guest" as to kill all sessions not associated with an user. :param user: UserID, "guest" or None. :type user: str | None """ logging.error("Invalidating all sessions for %s" % user) query = db.Query(GaeSession.kindName) if user is not None: query.filter("user =", str(user)) for key in query.iter(keysOnly=True): db.Delete(key)
def deleteRecursive(self, key): """ Recursively processes a delete request. This will delete all entries which are children of *key*, except *key* itself. :param key: URL-safe key of the node which children should be deleted. :type key: str :returns: The number of deleted objects. :rtype: int """ entrys = db.Query(self.viewSkel().kindName).filter( "parententry", str(key)).run() for e in entrys: self.deleteRecursive(str(e.key())) vs = self.editSkel() vs.setValues(e) vs.delete()
def markFileForDeletion(dlkey): """ Adds a marker to the data store that the file specified as *dlkey* can be deleted. Once the mark has been set, the data store is checked four times (default: every 4 hours) if the file is in use somewhere. If it is still in use, the mark goes away, otherwise the mark and the file are removed from the datastore. These delayed checks are necessary due to database inconsistency. :type dlkey: str :param dlkey: Unique download-key of the file that shall be marked for deletion. """ fileObj = db.Query("viur-deleted-files").filter("dlkey", dlkey).get() if fileObj: #Its allready marked return fileObj = db.Entity("viur-deleted-files") fileObj["itercount"] = 0 fileObj["dlkey"] = str(dlkey) db.Put(fileObj)
def postDeletedHandler(self, skel, key, id): db.Delete([ x for x in db.Query("viur-relations").ancestor(db.Key(id)).run( keysOnly=True) ])
def postSavedHandler(self, valuesCache, boneName, skel, key, dbfields): if not valuesCache[boneName]: values = [] elif isinstance(valuesCache[boneName], dict): values = [dict((k, v) for k, v in valuesCache[boneName].items())] else: values = [ dict((k, v) for k, v in x.items()) for x in valuesCache[boneName] ] parentValues = {} for parentKey in dbfields.keys(): if parentKey in self.parentKeys or any( [parentKey.startswith(x + ".") for x in self.parentKeys]): parentValues[parentKey] = dbfields[parentKey] dbVals = db.Query("viur-relations").ancestor( db.Key(key)) #skel.kindName+"_"+self.kind+"_"+key dbVals.filter("viur_src_kind =", skel.kindName) dbVals.filter("viur_dest_kind =", self.kind) dbVals.filter("viur_src_property =", boneName) for dbObj in dbVals.iter(): try: if not dbObj["dest.key"] in [x["dest"]["key"] for x in values ]: #Relation has been removed db.Delete(dbObj.key()) continue except: #This entry is corrupt db.Delete(dbObj.key()) else: # Relation: Updated data = [ x for x in values if x["dest"]["key"] == dbObj["dest.key"] ][0] if self.indexed: #We dont store more than key and kinds, and these dont change #Write our (updated) values in refSkel = self._refSkelCache refSkel.setValuesCache(data["dest"]) for k, v in refSkel.serialize().items(): dbObj["dest." + k] = v for k, v in parentValues.items(): dbObj["src." + k] = v if self.using is not None: usingSkel = self._usingSkelCache usingSkel.setValuesCache(data["rel"]) for k, v in usingSkel.serialize().items(): dbObj["rel." + k] = v dbObj["viur_delayed_update_tag"] = time() db.Put(dbObj) values.remove(data) # Add any new Relation for val in values: dbObj = db.Entity( "viur-relations", parent=db.Key(key)) #skel.kindName+"_"+self.kind+"_"+key if not self.indexed: #Dont store more than key and kinds, as they aren't used anyway dbObj["dest.key"] = val["dest"]["key"] dbObj["src.key"] = key else: refSkel = self._refSkelCache refSkel.setValuesCache(val["dest"]) for k, v in refSkel.serialize().items(): dbObj["dest." + k] = v for k, v in parentValues.items(): dbObj["src." + k] = v if self.using is not None: usingSkel = self._usingSkelCache usingSkel.setValuesCache(val["rel"]) for k, v in usingSkel.serialize().items(): dbObj["rel." + k] = v dbObj["viur_delayed_update_tag"] = time() dbObj[ "viur_src_kind"] = skel.kindName #The kind of the entry referencing #dbObj[ "viur_src_key" ] = str( key ) #The key of the entry referencing dbObj[ "viur_src_property"] = boneName #The key of the bone referencing #dbObj[ "viur_dest_key" ] = val["key"] dbObj["viur_dest_kind"] = self.kind db.Put(dbObj)
def markFileRefsStrong(): """Adds the weak=False marker to all Filerefs without such an marker""" for file in db.Query("file").iter(): if not "weak" in file.keys(): file["weak"] = False db.Put(file)
def pathToKey(self, key=None): """ Returns the recursively expanded path through the Hierarchy from the root-node to a requested node. :param key: URL-safe key of the destination entity. :type key: str :returns: An nested dictionary with information about all nodes in the path from root \ to the requested node. :rtype: dict """ def getName(obj): """ Tries to return a suitable name for the given object. """ if "name" in obj: return obj["name"] skel = self.viewSkel() if "name" in skel: nameBone = getattr(skel, "name") if (isinstance(nameBone, baseBone) and "languages" in dir(nameBone) and nameBone.languages): skel.setValues(obj) return unicode(skel["name"]) return None availableRepos = self.getAvailableRootNodes() if not key: try: key = availableRepos[0]["key"] except: raise errors.NotFound() keylist = [] else: if str(key).isdigit(): key = str(db.Key.from_path(self.viewSkel().kindName, long(key))) keylist = [key] if not self.canList(key): raise errors.Unauthorized() res = [] lastChildren = [] for x in range(0, 99): q = db.Query(self.viewSkel().kindName) q.filter("parententry =", str(key)) q.order("sortindex") entryObjs = q.run(100) lastChildren = res[:] res = [] for obj in entryObjs: if "parententry" in obj: parent = str(obj["parententry"]) else: parent = None r = { "name": getName(obj), "key": str(obj.key()), "parent": parent, "hrk": obj["hrk"] if "hrk" in obj else None, "active": (str(obj.key()) in keylist) } if r["active"]: r["children"] = lastChildren res.append(r) if key in [x["key"] for x in availableRepos]: break else: item = db.Get(str(key)) if item and "parententry" in item: keylist.append(key) key = item["parententry"] else: break return res
def getAppId(self, key, *args, **kwargs): if not self._checkKey(key, export=False): raise errors.Forbidden() return (pickle.dumps(db.Query("SharedConfData").get().key().app()) ) #app_identity.get_application_id()