def save(self, req): """ Writes the session to the memcache/datastore. Does nothing, if the session hasn't been changed in the current request. """ if self.changed: serialized = base64.b64encode( pickle.dumps(self.session, protocol=pickle.HIGHEST_PROTOCOL ) ) self.getSessionKey( req ) # Get the current user id userid = None try: if "user" in dir( conf["viur.mainApp"] ): #Check for our custom user-api userid = conf["viur.mainApp"].user.getCurrentUser()["key"] except: pass try: dbSession = db.Entity( self.kindName, name=self.key ) dbSession["data"] = serialized dbSession["sslkey"] = self.sslKey dbSession["skey"] = self.sessionSecurityKey dbSession["lastseen"] = time() dbSession["user"] = str(userid) or "guest" #Store the userid inside the sessionobj, so we can kill specific sessions if needed dbSession.set_unindexed_properties( ["data","sslkey" ] ) db.Put( dbSession ) except (OverQuotaError, CapabilityDisabledError): pass req.response.headers.add_header( "Set-Cookie", bytes( "%s=%s; Max-Age=99999; Path=/; HttpOnly" % ( self.plainCookieName, self.key ) ) ) if req.isSSLConnection: req.response.headers.add_header( "Set-Cookie", bytes( "%s=%s; Max-Age=99999; Path=/; Secure; HttpOnly" % ( self.sslCookieName, self.sslKey ) ) )
def getKeyTxn( kindName, orderKey ): """Generates and returns a new, unique Key""" seqObj = db.GetOrInsert(kindName, "viur_bill_sequences", count=1000) idx = seqObj["count"] seqObj["count"] += 1 db.Put( seqObj ) return str(idx)
def storeEntry2(self, e, key): if not self._checkKey(key, export=False): raise errors.Forbidden() entry = pickle.loads(e.decode("HEX")) for k in list(entry.keys())[:]: if isinstance(entry[k], str): entry[k] = entry[k].decode("UTF-8") key = db.Key(encoded=utils.normalizeKey(entry["key"])) logging.info(key.kind()) logging.info(key.id()) logging.info(key.name()) dbEntry = db.Entity(kind=key.kind(), parent=key.parent(), id=key.id(), name=key.name()) #maybe some more fixes here ? for k in entry.keys(): if k != "key": val = entry[k] dbEntry[k] = val db.Put(dbEntry) try: skel = skeletonByKind(key.kind())() except: logging.error("Unknown Skeleton - skipping") skel.fromDB(str(dbEntry.key())) skel.refresh() skel.toDB(clearUpdateTag=True)
def upload(self, oldkey, *args, **kwargs): logging.error("got UPLOADS") res = [] for upload in self.getUploads(): fileName = self.decodeFileName(upload.filename) if str(upload.content_type).startswith("image/"): try: servingURL = get_serving_url(upload.key()) except: servingURL = "" else: servingURL = "" res.append({ "name": fileName, "size": upload.size, "mimetype": upload.content_type, "dlkey": str(upload.key()), "servingurl": servingURL, "parentdir": "", "parentrepo": "", "weak": False }) oldKeyHash = sha256(oldkey).hexdigest().encode("hex") e = db.Entity("viur-blobimportmap", name=oldKeyHash) e["newkey"] = str(upload.key()) e["oldkey"] = oldkey e["servingurl"] = servingURL e["available"] = True db.Put(e) return (json.dumps({"action": "addSuccess", "values": res}))
def storeEntry(self, e, key): if not self._checkKey(key, export=False): raise errors.Forbidden() entry = pickle.loads(e) for k in list(entry.keys())[:]: if isinstance(entry[k], str): entry[k] = entry[k].decode("UTF-8") if "key" in entry.keys(): key = db.Key(encoded=entry["key"]) elif "id" in entry.keys(): key = db.Key(encoded=entry["id"]) else: raise AttributeError() logging.error(key.kind()) logging.error(key.id()) logging.error(key.name()) dbEntry = db.Entity(kind=key.kind(), parent=key.parent(), id=key.id(), _app=key.app(), name=key.name()) #maybe some more fixes here ? for k in entry.keys(): if k != "key": val = entry[k] #if isinstance(val, dict) or isinstance(val, list): # val = pickle.dumps( val ) dbEntry[k] = val if dbEntry.key().id(): # Ensure the Datastore knows that it's id is in use datastore._GetConnection()._reserve_keys([dbEntry.key()]) db.Put(dbEntry)
def doCheckForUnreferencedBlobs(cursor = None): def getOldBlobKeysTxn(dbKey): obj = db.Get(dbKey) res = obj["old_blob_references"] or [] if obj["is_stale"]: db.Delete(dbKey) else: obj["has_old_blob_references"] = False obj["old_blob_references"] = [] db.Put(obj) return res gotAtLeastOne = False query = db.Query("viur-blob-locks").filter("has_old_blob_references", True).cursor(cursor) for lockKey in query.run(100, keysOnly=True): gotAtLeastOne = True oldBlobKeys = db.RunInTransaction( getOldBlobKeysTxn, lockKey ) for blobKey in oldBlobKeys: if db.Query("viur-blob-locks").filter("active_blob_references =", blobKey).get(): #This blob is referenced elsewhere logging.info("Stale blob is still referenced, %s" % blobKey) continue # Add a marker and schedule it for deletion fileObj = db.Query( "viur-deleted-files" ).filter( "dlkey", blobKey ).get() if fileObj: #Its already marked logging.info("Stale blob already marked for deletion, %s" % blobKey) return fileObj = db.Entity("viur-deleted-files") fileObj["itercount"] = 0 fileObj["dlkey"] = str(blobKey) logging.info("Stale blob marked dirty, %s" % blobKey) db.Put(fileObj) newCursor = query.getCursor() if gotAtLeastOne and newCursor and newCursor.urlsafe() != cursor: doCheckForUnreferencedBlobs(newCursor.urlsafe())
def setIndex(self, item, index, skey, *args, **kwargs): """ Changes the order of the elements in the current level by changing the index of *item*. .. seealso:: :func:`canSetIndex` :param item: URL-safe key of the item which index should be changed. :type item: str :param index: New index for this item. This value must be cast-able to float. :type index: str :returns: A rendered success result generated by the default renderer. :raises: :exc:`server.errors.NotFound`, when no entry with the given *key* was found. :raises: :exc:`server.errors.Unauthorized`, if the current user does not have the required permissions. :raises: :exc:`server.errors.PreconditionFailed`, if the *skey* could not be verified. """ if not securitykey.validate(skey, acceptSessionKey=True): raise errors.PreconditionFailed() if not self.canSetIndex(item, index): raise errors.Unauthorized() fromItem = db.Get(item) fromItem["sortindex"] = float(index) db.Put(fromItem) skel = self.editSkel() assert skel.fromDB(item) self.onItemSetIndex(skel) self.onItemChanged(skel) return self.render.setIndexSuccess(obj=fromItem)
def doCleanupDeletedFiles(cursor = None): maxIterCount = 2 # How often a file will be checked for deletion gotAtLeastOne = False query = db.Query("viur-deleted-files").cursor( cursor ) for file in query.run(100): gotAtLeastOne = True if not "dlkey" in file.keys(): db.Delete(file.key()) elif db.Query("viur-blob-locks").filter("active_blob_references =", file["dlkey"]).get(): logging.info("is referenced, %s" % file["dlkey"]) db.Delete(file.key()) else: if file["itercount"] > maxIterCount: logging.info("Finally deleting, %s" % file["dlkey"]) blobstore.delete(file["dlkey"]) db.Delete(file.key()) # There should be exactly 1 or 0 of these for f in db.Query("file").filter("dlkey =", file["dlkey"]).iter(keysOnly=True): db.Delete(f) else: logging.error("Increasing count, %s" % file["dlkey"]) file["itercount"] += 1 db.Put(file) newCursor = query.getCursor() if gotAtLeastOne and newCursor and newCursor.urlsafe() != cursor: doCleanupDeletedFiles(newCursor.urlsafe())
def login(self, name=None, password=None, skey="", *args, **kwargs): if self.userModule.getCurrentUser(): #Were already logged in return self.userModule.render.loginSucceeded() if not name or not password or not securitykey.validate(skey): return self.userModule.render.login(self.loginSkel()) query = db.Query(self.userModule.viewSkel().kindName) res = query.filter("name.idx >=", name.lower()).get() if res is None: res = {"password": "", "status": 0, "name": "", "name.idx": ""} if "password_salt" in res.keys(): # Its the new, more secure passwd passwd = pbkdf2(password[:conf["viur.maxPasswordLength"]], res["password_salt"]) else: passwd = sha512(password.encode("UTF-8") + conf["viur.salt"]).hexdigest() isOkay = True # We do this exactly that way to avoid timing attacks # Check if the username matches storedUserName = res.get("name.idx", "") if len(storedUserName) != len(name.lower()): isOkay = False else: for x, y in izip(storedUserName, name.lower()): if x != y: isOkay = False # Check if the password matches storedPasswordHash = res.get("password", "") if len(storedPasswordHash) != len(passwd): isOkay = False else: for x, y in izip(storedPasswordHash, passwd): if x != y: isOkay = False # Verify that this account isn't blocked if res["status"] < 10: isOkay = False if not isOkay: skel = self.loginSkel() skel.fromClient({"name": name, "nomissing": "1"}) return self.userModule.render.login(skel, loginFailed=True) else: if not "password_salt" in res.keys( ): #Update the password to the new, more secure format res["password_salt"] = utils.generateRandomString(13) res["password"] = pbkdf2( password[:conf["viur.maxPasswordLength"]], res["password_salt"]) db.Put(res) return self.userModule.continueAuthenticationFlow(self, res.key())
def create(duration=None, **kwargs): """ Creates a new onetime Securitykey for the current session If duration is not set, this key is valid only for the current session. Otherwise, the key and its data is serialized and saved inside the datastore for up to duration-seconds :param duration: Make this key valid for a fixed timeframe (and independend of the current session) :type duration: int or None :returns: The new onetime key """ key = generateRandomString() if duration is None: sessionDependend = True duration = 30 * 60 # 30 Mins from now else: sessionDependend = False duration = int(duration) dbObj = db.Entity(securityKeyKindName, name=key) for k, v in kwargs.items(): dbObj[k] = v dbObj["until"] = datetime.now() + timedelta(seconds=duration) if sessionDependend: dbObj["session"] = currentSession.getSessionKey() else: dbObj["session"] = None dbObj.set_unindexed_properties( [x for x in dbObj.keys() if not x == "until"]) db.Put(dbObj) return (key)
def updateTransaction(userKey, idx): user = db.Get(userKey) if not "otptimedrift" in user.keys() or not isinstance( user["otptimedrift"], float): user["otptimedrift"] = 0.0 user["otptimedrift"] += min(max(0.1 * idx, -0.3), 0.3) db.Put(user)
def setKeyTxn(orderKey, idx): """Assigns the new order to the given order""" dbObj = db.Get(db.Key(orderKey)) if not dbObj: return dbObj["idx"] = idx db.Put(dbObj)
def txn(orderKey, state, removeState): dbObj = db.Get(db.Key(orderKey)) if not dbObj: return dbObj["state_%s" % state] = "1" if not removeState else "0" dbObj["changedate"] = datetime.now() db.Put(dbObj)
def reparent(self, item, dest, skey, *args, **kwargs): """ Moves an entry *item* (and everything beneath it) to another parent-node *dest*. .. seealso:: :func:`canReparent` :param item: URL-safe key of the item which will be moved. :type item: str :param dest: URL-safe key of the new parent for this item. :type dest: str :returns: A rendered success result generated by the default renderer. :raises: :exc:`server.errors.NotFound`, when no entry with the given *id* was found. :raises: :exc:`server.errors.Unauthorized`, if the current user does not have the required permissions. :raises: :exc:`server.errors.PreconditionFailed`, if the *skey* could not be verified. """ if not securitykey.validate(skey, acceptSessionKey=True): raise errors.PreconditionFailed() if not self.canReparent(item, dest): raise errors.Unauthorized() if not self.isValidParent(dest) or item == dest: raise errors.NotAcceptable() ## Test for recursion isValid = False currLevel = db.Get(dest) for x in range(0, 99): if str(currLevel.key()) == item: break if currLevel.key().kind( ) == self.viewSkel().kindName + "_rootNode": # We reached a rootNode isValid = True break currLevel = db.Get(currLevel["parententry"]) if not isValid: raise errors.NotAcceptable() ## Update entry fromItem = db.Get(item) fromItem["parententry"] = dest fromItem["parentrepo"] = str(self.getRootNode(dest).key()) db.Put(fromItem) skel = self.editSkel() assert skel.fromDB(item) self.onItemReparent(skel) self.onItemChanged(skel) return self.render.reparentSuccess(obj=fromItem)
def getOldBlobKeysTxn(dbKey): obj = db.Get(dbKey) res = obj["old_blob_references"] or [] if obj["is_stale"]: db.Delete(dbKey) else: obj["has_old_blob_references"] = False obj["old_blob_references"] = [] db.Put(obj) return res
def index(self, *args, **kwargs): global _callableTasks, _periodicTasks logging.debug("Starting maintenance-run") checkUpdate() #Let the update-module verify the database layout first logging.debug("Updatecheck complete") for task, intervall in _periodicTasks.items( ): #Call all periodic tasks if intervall: #Ensure this task doesn't get called to often try: lastCall = db.Get( db.Key.from_path("viur-task-interval", task.periodicTaskName)) if lastCall["date"] > datetime.now() - timedelta( minutes=intervall): logging.debug( "Skipping task %s - Has already run recently." % task.periodicTaskName) continue except db.EntityNotFoundError: pass res = self.findBoundTask(task) if res: #Its bound, call it this way :) res[0]() else: task() #It seems it wasnt bound - call it as a static method logging.debug("Successfully called task %s" % task.periodicTaskName) if intervall: # Update its last-call timestamp entry = db.Entity("viur-task-interval", name=task.periodicTaskName) entry["date"] = datetime.now() db.Put(entry) logging.debug("Periodic tasks complete") for currentTask in db.Query( "viur-queued-tasks").iter(): #Look for queued tasks db.Delete(currentTask.key()) if currentTask["taskid"] in _callableTasks: task = _callableTasks[currentTask["taskid"]]() tmpDict = {} for k in currentTask.keys(): if k == "taskid": continue tmpDict[k] = json.loads(currentTask[k]) try: task.execute(**tmpDict) except Exception as e: logging.error("Error executing Task") logging.exception(e) logging.debug("Scheduled tasks complete")
def pwrecover(self, authtoken=None, skey=None, *args, **kwargs): if authtoken: data = securitykey.validate(authtoken) if data and isinstance(data, dict) and "userKey" in data.keys( ) and "password" in data.keys(): skel = self.userModule.editSkel() assert skel.fromDB(data["userKey"]) skel["password"] = data["password"] skel.toDB() return self.userModule.render.view( skel, self.passwordRecoverySuccessTemplate) else: return self.userModule.render.view( None, self.passwordRecoveryInvalidTokenTemplate) else: skel = self.lostPasswordSkel() if len(kwargs) == 0 or not skel.fromClient( kwargs) or not securitykey.validate(skey): return self.userModule.render.passwdRecover( skel, tpl=self.passwordRecoveryTemplate) user = self.userModule.viewSkel().all().filter( "name.idx =", skel["name"].lower()).get() if not user or user[ "status"] < 10: # Unknown user or locked account skel.errors["name"] = _("Unknown user") return self.userModule.render.passwdRecover( skel, tpl=self.passwordRecoveryTemplate) try: if user["changedate"] > (datetime.datetime.now() - datetime.timedelta(days=1)): # This user probably has already requested a password reset # within the last 24 hrss return self.userModule.render.view( skel, self.passwordRecoveryAlreadySendTemplate) except AttributeError: #Some newly generated user-objects dont have such a changedate yet pass user["changedate"] = datetime.datetime.now() db.Put(user) userSkel = self.userModule.viewSkel().ensureIsCloned() assert userSkel.fromDB(user.key()) userSkel.skey = baseBone(descr="Skey") userSkel["skey"] = securitykey.create(60 * 60 * 24, userKey=str(user.key()), password=skel["password"]) utils.sendEMail([userSkel["name"]], self.userModule.passwordRecoveryMail, userSkel) return self.userModule.render.view( {}, self.passwordRecoveryInstuctionsSendTemplate)
def calculateOrderSum(self, step, orderKey, *args, **kwargs): """ Calculates the final price for this order. This function *must* be called before any attempt is made to start a payment process. :param step: Current step within the ordering process :type step: int :param orderKey: order to calculate the price for :type orderKey: str """ price = sum([x[3] for x in self.getBillItems(orderKey)]) orderObj = db.Get(db.Key(str(orderKey))) orderObj["price"] = price db.Put(orderObj)
def wrapF(self, *args, **kwargs): currentRequest = request.current.get() if conf["viur.disableCache"] or currentRequest.disableCache: # Caching disabled if conf["viur.disableCache"]: logging.debug("Caching is disabled by config") return (f(self, *args, **kwargs)) # How many arguments are part of the way to the function called (and how many are just *args) offset = -len(currentRequest.args) or len(currentRequest.pathlist) path = "/" + "/".join(currentRequest.pathlist[:offset]) if not path in urls: # This path (possibly a sub-render) should not be cached logging.debug("Not caching for %s" % path) return (f(self, *args, **kwargs)) key = keyFromArgs(f, userSensitive, languageSensitive, evaluatedArgs, path, args, kwargs) if not key: # Someting is wrong (possibly the parameter-count) # Letz call f, but we knew already that this will clash return (f(self, *args, **kwargs)) try: dbRes = db.Get(db.Key.from_path(viurCacheName, key)) except db.EntityNotFoundError: dbRes = None if dbRes: if not maxCacheTime or \ dbRes["creationtime"] > datetime.now()-timedelta( seconds=maxCacheTime ): # We store it unlimited or the cache is fresh enough logging.debug("This request was served from cache.") currentRequest.response.headers['Content-Type'] = dbRes[ "content-type"].encode("UTF-8") return (dbRes["data"]) # If we made it this far, the request wasnt cached or too old; we need to rebuild it res = f(self, *args, **kwargs) dbEntity = db.Entity(viurCacheName, name=key) dbEntity["data"] = res dbEntity["creationtime"] = datetime.now() dbEntity["path"] = path dbEntity["content-type"] = request.current.get( ).response.headers['Content-Type'] dbEntity.set_unindexed_properties(["data", "content-type" ]) #We can save 2 DB-Writs :) db.Put(dbEntity) logging.debug("This request was a cache-miss. Cache has been updated.") return (res)
def getOrBuildIndex(self, origQuery): """ Builds a specific index based on origQuery AND local variables (self.indexPage and self.indexMaxPage) Returns a list of starting-cursors for each page. You probably shouldn't call this directly. Use cursorForQuery. :param origQuery: Query to build the index for :type origQuery: db.Query :param key: DB-Key to save the index to :type key: string :returns: [] """ key = self.keyFromQuery(origQuery) if key in self._cache.keys(): #We have it cached return (self._cache[key]) #We dont have it cached - try to load it from DB try: index = db.Get(db.Key.from_path(self._dbType, key)) res = json.loads(index["data"]) self._cache[key] = res return (res) except db.EntityNotFoundError: #Its not in the datastore, too pass #We dont have this index yet.. Build it #Clone the original Query queryRes = origQuery.clone(keysOnly=True).datastoreQuery.Run( limit=self.maxPages * self.pageSize) #Build-Up the index res = [] i = 0 previousCursor = None #The first page dosnt have any cursor for discardedKey in queryRes: if i % self.pageSize == 0: res.append(previousCursor) if i % self.pageSize == (self.pageSize - 1) and i > 1: previousCursor = str(queryRes.cursor().urlsafe()) i += 1 if len(res) == 0: # Ensure that the first page exists res.append(None) entry = db.Entity(self._dbType, name=key) entry["data"] = json.dumps(res) entry["creationdate"] = datetime.now() db.Put(entry) return (res)
def txnDelete(key, skel): dbObj = db.Get(db.Key( key)) # Fetch the raw object as we might have to clear locks for boneName, bone in skel.items(): # Ensure that we delete any value-lock objects remaining for this entry if bone.unique: try: logging.error("x1") logging.error(dbObj.keys()) if "%s.uniqueIndexValue" % boneName in dbObj.keys(): logging.error("x2") db.Delete( db.Key.from_path( "%s_%s_uniquePropertyIndex" % (skel.kindName, boneName), dbObj["%s.uniqueIndexValue" % boneName])) except db.EntityNotFoundError: raise pass # Delete the blob-key lock object try: lockObj = db.Get(db.Key.from_path("viur-blob-locks", str(key))) except: lockObj = None if lockObj is not None: if lockObj["old_blob_references"] is None and lockObj[ "active_blob_references"] is None: db.Delete(lockObj) # Nothing to do here else: if lockObj["old_blob_references"] is None: # No old stale entries, move active_blob_references -> old_blob_references lockObj["old_blob_references"] = lockObj[ "active_blob_references"] elif lockObj["active_blob_references"] is not None: # Append the current references to the list of old & stale references lockObj["old_blob_references"] += lockObj[ "active_blob_references"] lockObj["active_blob_references"] = [ ] # There are no active ones left lockObj["is_stale"] = True lockObj["has_old_blob_references"] = True db.Put(lockObj) db.Delete(db.Key(key))
def transaction(): obj = db.Get(key) if check: assert isinstance(check, dict), "'check' has to be a dict, you diggi!" assert all([obj[bone] == value for bone, value in check.items()]) if values: for bone, value in values.items(): if bone[0] == "+": obj[bone[1:]] += value elif bone[0] == "-": obj[bone[1:]] -= value else: obj[bone] = value if func and callable(func): func(obj) db.Put(obj)
def markFileForDeletion(dlkey): """ Adds a marker to the data store that the file specified as *dlkey* can be deleted. Once the mark has been set, the data store is checked four times (default: every 4 hours) if the file is in use somewhere. If it is still in use, the mark goes away, otherwise the mark and the file are removed from the datastore. These delayed checks are necessary due to database inconsistency. :type dlkey: str :param dlkey: Unique download-key of the file that shall be marked for deletion. """ fileObj = db.Query("viur-deleted-files").filter("dlkey", dlkey).get() if fileObj: #Its allready marked return fileObj = db.Entity("viur-deleted-files") fileObj["itercount"] = 0 fileObj["dlkey"] = str(dlkey) db.Put(fileObj)
def postSavedHandler(self, valuesCache, boneName, skel, key, dbfields): if not valuesCache[boneName]: values = [] elif isinstance(valuesCache[boneName], dict): values = [dict((k, v) for k, v in valuesCache[boneName].items())] else: values = [ dict((k, v) for k, v in x.items()) for x in valuesCache[boneName] ] parentValues = {} for parentKey in dbfields.keys(): if parentKey in self.parentKeys or any( [parentKey.startswith(x + ".") for x in self.parentKeys]): parentValues[parentKey] = dbfields[parentKey] dbVals = db.Query("viur-relations").ancestor( db.Key(key)) #skel.kindName+"_"+self.kind+"_"+key dbVals.filter("viur_src_kind =", skel.kindName) dbVals.filter("viur_dest_kind =", self.kind) dbVals.filter("viur_src_property =", boneName) for dbObj in dbVals.iter(): try: if not dbObj["dest.key"] in [x["dest"]["key"] for x in values ]: #Relation has been removed db.Delete(dbObj.key()) continue except: #This entry is corrupt db.Delete(dbObj.key()) else: # Relation: Updated data = [ x for x in values if x["dest"]["key"] == dbObj["dest.key"] ][0] if self.indexed: #We dont store more than key and kinds, and these dont change #Write our (updated) values in refSkel = self._refSkelCache refSkel.setValuesCache(data["dest"]) for k, v in refSkel.serialize().items(): dbObj["dest." + k] = v for k, v in parentValues.items(): dbObj["src." + k] = v if self.using is not None: usingSkel = self._usingSkelCache usingSkel.setValuesCache(data["rel"]) for k, v in usingSkel.serialize().items(): dbObj["rel." + k] = v dbObj["viur_delayed_update_tag"] = time() db.Put(dbObj) values.remove(data) # Add any new Relation for val in values: dbObj = db.Entity( "viur-relations", parent=db.Key(key)) #skel.kindName+"_"+self.kind+"_"+key if not self.indexed: #Dont store more than key and kinds, as they aren't used anyway dbObj["dest.key"] = val["dest"]["key"] dbObj["src.key"] = key else: refSkel = self._refSkelCache refSkel.setValuesCache(val["dest"]) for k, v in refSkel.serialize().items(): dbObj["dest." + k] = v for k, v in parentValues.items(): dbObj["src." + k] = v if self.using is not None: usingSkel = self._usingSkelCache usingSkel.setValuesCache(val["rel"]) for k, v in usingSkel.serialize().items(): dbObj["rel." + k] = v dbObj["viur_delayed_update_tag"] = time() dbObj[ "viur_src_kind"] = skel.kindName #The kind of the entry referencing #dbObj[ "viur_src_key" ] = str( key ) #The key of the entry referencing dbObj[ "viur_src_property"] = boneName #The key of the bone referencing #dbObj[ "viur_dest_key" ] = val["key"] dbObj["viur_dest_kind"] = self.kind db.Put(dbObj)
def markFileRefsStrong(): """Adds the weak=False marker to all Filerefs without such an marker""" for file in db.Query("file").iter(): if not "weak" in file.keys(): file["weak"] = False db.Put(file)
def setTokenTxn(key, token): order = db.Get(key) if not order: return order["paypal_token"] = urllib.unquote(token) db.Put(order)
def txnUpdate(key, mergeFrom, clearUpdateTag): blobList = set() skel = type(mergeFrom)() # Load the current values from Datastore or create a new, empty db.Entity if not key: dbObj = db.Entity(skel.kindName) oldBlobLockObj = None else: k = db.Key(key) assert k.kind() == skel.kindName, "Cannot write to invalid kind!" try: dbObj = db.Get(k) except db.EntityNotFoundError: dbObj = db.Entity(k.kind(), id=k.id(), name=k.name(), parent=k.parent()) else: skel.setValues(dbObj) try: oldBlobLockObj = db.Get(db.Key.from_path("viur-blob-locks", str(k))) except: oldBlobLockObj = None # Remember old hashes for bones that must have an unique value oldUniqeValues = {} for boneName, boneInstance in skel.items(): if boneInstance.unique: if "%s.uniqueIndexValue" % boneName in dbObj: oldUniqeValues[boneName] = dbObj["%s.uniqueIndexValue" % boneName] ## Merge the values from mergeFrom in for key, bone in skel.items(): if key in mergeFrom: bone.mergeFrom(skel.valuesCache, key, mergeFrom) for key, _bone in skel.items(): dbObj = _bone.serialize(skel.valuesCache, key, dbObj) blobList.update(_bone.getReferencedBlobs(self.valuesCache, key)) if clearUpdateTag: dbObj["viur_delayed_update_tag"] = 0 # Mark this entity as Up-to-date. else: dbObj[ "viur_delayed_update_tag"] = time() # Mark this entity as dirty, so the background-task will catch it up and update its references. dbObj = skel.preProcessSerializedData(dbObj) try: ourKey = str(dbObj.key()) except: # Its not an update but an insert, no key yet ourKey = None # Lock hashes from bones that must have unique values newUniqeValues = {} for boneName, boneInstance in skel.items(): if boneInstance.unique: # Check if the property is really unique newUniqeValues[boneName] = boneInstance.getUniquePropertyIndexValue( self.valuesCache, boneName) if newUniqeValues[boneName] is not None: try: lockObj = db.Get(db.Key.from_path( "%s_%s_uniquePropertyIndex" % (skel.kindName, boneName), newUniqeValues[boneName])) if lockObj["references"] != ourKey: # This value has been claimed, and that not by us raise ValueError( "The unique value '%s' of bone '%s' has been recently claimed!" % (self.valuesCache[boneName], boneName)) except db.EntityNotFoundError: # No lockObj found for that value, we can use that pass dbObj["%s.uniqueIndexValue" % boneName] = newUniqeValues[boneName] else: if "%s.uniqueIndexValue" % boneName in dbObj: del dbObj["%s.uniqueIndexValue" % boneName] if not skel.searchIndex: # We generate the searchindex using the full skel, not this (maybe incomplete one) tags = [] for key, _bone in skel.items(): if _bone.searchable: tags += [tag for tag in _bone.getSearchTags(self.valuesCache, key) if (tag not in tags and len(tag) < 400)] dbObj["viur_tags"] = tags db.Put(dbObj) # Write the core entry back # Now write the blob-lock object blobList = skel.preProcessBlobLocks(blobList) if blobList is None: raise ValueError( "Did you forget to return the bloblist somewhere inside getReferencedBlobs()?") if None in blobList: raise ValueError("None is not a valid blobKey.") if oldBlobLockObj is not None: oldBlobs = set(oldBlobLockObj["active_blob_references"] if oldBlobLockObj[ "active_blob_references"] is not None else []) removedBlobs = oldBlobs - blobList oldBlobLockObj["active_blob_references"] = list(blobList) if oldBlobLockObj["old_blob_references"] is None: oldBlobLockObj["old_blob_references"] = [x for x in removedBlobs] else: tmp = set(oldBlobLockObj["old_blob_references"] + [x for x in removedBlobs]) oldBlobLockObj["old_blob_references"] = [x for x in (tmp - blobList)] oldBlobLockObj["has_old_blob_references"] = oldBlobLockObj[ "old_blob_references"] is not None and len( oldBlobLockObj["old_blob_references"]) > 0 oldBlobLockObj["is_stale"] = False db.Put(oldBlobLockObj) else: # We need to create a new blob-lock-object blobLockObj = db.Entity("viur-blob-locks", name=str(dbObj.key())) blobLockObj["active_blob_references"] = list(blobList) blobLockObj["old_blob_references"] = [] blobLockObj["has_old_blob_references"] = False blobLockObj["is_stale"] = False db.Put(blobLockObj) for boneName, boneInstance in skel.items(): if boneInstance.unique: # Update/create/delete missing lock-objects if boneName in oldUniqeValues and oldUniqeValues[boneName] != \ newUniqeValues[boneName]: # We had an old lock and its value changed try: # Try to delete the old lock oldLockObj = db.Get(db.Key.from_path( "%s_%s_uniquePropertyIndex" % (skel.kindName, boneName), oldUniqeValues[boneName])) if oldLockObj["references"] != ourKey: # We've been supposed to have that lock - but we don't. # Don't remove that lock as it now belongs to a different entry logging.critical( "Detected Database corruption! A Value-Lock had been reassigned!") else: # It's our lock which we don't need anymore db.Delete(db.Key.from_path( "%s_%s_uniquePropertyIndex" % ( skel.kindName, boneName), oldUniqeValues[boneName])) except db.EntityNotFoundError as e: logging.critical( "Detected Database corruption! Could not delete stale lock-object!") if newUniqeValues[boneName] is not None: # Lock the new value newLockObj = db.Entity( "%s_%s_uniquePropertyIndex" % (skel.kindName, boneName), name=newUniqeValues[boneName]) newLockObj["references"] = str(dbObj.key()) db.Put(newLockObj) return (str(dbObj.key()), dbObj, skel)
def iterImport(module, target, exportKey, cursor=None, amount=0): """ Processes 100 Entries and calls the next batch """ urlfetch.set_default_fetch_deadline(20) payload = {"module": module, "key": exportKey} if cursor: payload.update({"cursor": cursor}) result = urlfetch.fetch( url=target, payload=urllib.urlencode(payload), method=urlfetch.POST, headers={'Content-Type': 'application/x-www-form-urlencoded'}) if result.status_code == 200: res = pickle.loads(result.content.decode("HEX")) skel = skeletonByKind(module)() logging.info("%s: %d new entries fetched, total %d entries fetched" % (module, len(res["values"]), amount)) if len(res["values"]) == 0: try: utils.sendEMailToAdmins( "Import of kind %s finished with %d entities" % (module, amount), "ViUR finished to import %d entities of " "kind %s from %s.\n" % (amount, module, target)) except: #OverQuota, whatever logging.error("Unable to send Email") return for entry in res["values"]: for k in list(entry.keys())[:]: if isinstance(entry[k], str): entry[k] = entry[k].decode("UTF-8") if not "key" in entry.keys(): entry["key"] = entry["id"] key = db.Key(encoded=utils.normalizeKey(entry["key"])) # Special case: Convert old module root nodes!!! if module.endswith( "_rootNode") and key.name() and "_modul_" in key.name(): name = key.name().replace("_modul_", "_module_") else: name = key.name() dbEntry = db.Entity(kind=key.kind(), parent=key.parent(), id=key.id(), name=name) for k in entry.keys(): if k == "key": continue dbEntry[k] = entry[k] # Special case: Convert old module root nodes!!! if (isinstance(skel, (HierarchySkel, TreeLeafSkel)) and k in ["parentdir", "parententry", "parentrepo"] and entry[k]): key = db.Key(encoded=str(entry[k])) if key.parent(): parent = db.Key( encoded=utils.normalizeKey(key.parent())) else: parent = None if key.id_or_name() and "_modul_" in str(key.id_or_name()): name = key.id_or_name().replace("_modul_", "_module_") else: name = key.id_or_name() dbEntry[k] = str( db.Key.from_path(key.kind(), name, parent=parent)) db.Put(dbEntry) skel.fromDB(str(dbEntry.key())) skel.refresh() skel.toDB(clearUpdateTag=True) amount += 1 iterImport(module, target, exportKey, res["cursor"], amount)
def checkout(self, step=None, key=None, skey=None, *args, **kwargs): """ Performs the checkout process trough the state machine provided by self.steps. :param step: The current step index, None for beginning a new checkout :param key: Key of the current checkout :param skey: Server security key :return: Returns the rendered template or throws redirection exceptions. """ myKindName = self.viewSkel().kindName if step is None: logging.info("Starting new checkout process") billObj = db.Entity(myKindName) billObj["idx"] = "0000000" for state in self.states: billObj["state_%s" % state] = "0" db.Put(billObj) key = str(billObj.key()) #Copy the Cart if "amountSkel" in dir(self): cart = session.current.get("cart_products") or {} s = self.amountSkel() products = [] for prod, atts in cart.items(): for i in range(0, atts["amount"]): products.append(str(prod)) s.fromClient({"product": products}) s.toDB() session.current["order_" + myKindName] = { "key": str(key), "completedSteps": [] } session.current.markChanged() raise errors.Redirect("?step=0&key=%s" % str(key)) elif key: try: orderKey = db.Key(key) step = int(step) assert (step >= 0) assert (step < len(self.steps)) except: raise errors.NotAcceptable() sessionInfo = session.current.get("order_" + myKindName) if not sessionInfo or not sessionInfo.get("key") == str(orderKey): raise errors.Unauthorized() if step in sessionInfo["completedSteps"]: session.current["order_" + myKindName]["completedSteps"] = [ x for x in sessionInfo["completedSteps"] if x < step ] session.current.markChanged() #Make sure that no steps can be skipped if step != 0 and not step - 1 in sessionInfo["completedSteps"]: raise errors.Redirect("?step=0&key=%s" % str(str(orderKey))) currentStep = self.steps[step] if "preHandler" in currentStep.keys(): try: if isinstance(currentStep["preHandler"], list): for handler in currentStep["preHandler"]: handler(self, step, str(orderKey), *args, **kwargs) else: currentStep["preHandler"](self, step, str(orderKey), refkwargs=kwargs, *args, **kwargs) except SkipStepException: session.current["order_" + myKindName]["completedSteps"].append(step) session.current.markChanged() raise errors.Redirect("?step=%s&key=%s" % (str(step + 1), str(orderKey))) except ReturnHtmlException as e: return (e.html) if "requiresSecurityKey" in currentStep and currentStep[ "requiresSecurityKey"]: if not securitykey.validate(skey): raise errors.PreconditionFailed() pass if "mainHandler" in currentStep: if currentStep["mainHandler"]["action"] == "edit": skel = self.getSkelByName( currentStep["mainHandler"]["skeleton"], str(orderKey)) skel.fromDB(str(orderKey)) if not len(kwargs.keys()) or not skel.fromClient(kwargs): return (self.render.edit( skel, tpl=currentStep["mainHandler"]["template"], step=step)) skel.toDB() if currentStep["mainHandler"]["action"] == "view": if not "complete" in kwargs or not kwargs[ "complete"] == u"1": skel = self.getSkelByName( currentStep["mainHandler"]["skeleton"], str(orderKey)) skel.fromDB(str(orderKey)) return (self.render.view( skel, tpl=currentStep["mainHandler"]["template"], step=step)) elif currentStep["mainHandler"]["action"] == "function": res = currentStep["mainHandler"]["function"](self, step, str(orderKey), *args, **kwargs) if res: return res if "postHandler" in currentStep: currentStep["postHandler"](self, step, str(orderKey), *args, **kwargs) session.current["order_" + myKindName]["completedSteps"].append(step) session.current.markChanged() logging.info("next ?step=%s&key=%s" % (str(step + 1), str(orderKey))) raise errors.Redirect("?step=%s&key=%s" % (str(step + 1), str(orderKey)))
def fixTxn(nodeKey, newRepoKey): node = db.Get(nodeKey) node["parentrepo"] = newRepoKey db.Put(node)