Beispiel #1
0
    def refresh(self):
        if self["parententry"]:
            self["parententry"] = utils.normalizeKey(self["parententry"])
        if self["parentrepo"]:
            self["parentrepo"] = utils.normalizeKey(self["parentrepo"])

        super(HierarchySkel, self).refresh()
Beispiel #2
0
    def storeEntry2(self, e, key):
        if not self._checkKey(key, export=False):
            raise errors.Forbidden()
        entry = pickle.loads(e.decode("HEX"))
        for k in list(entry.keys())[:]:
            if isinstance(entry[k], str):
                entry[k] = entry[k].decode("UTF-8")
        key = db.Key(encoded=utils.normalizeKey(entry["key"]))

        logging.info(key.kind())
        logging.info(key.id())
        logging.info(key.name())
        dbEntry = db.Entity(kind=key.kind(),
                            parent=key.parent(),
                            id=key.id(),
                            name=key.name())  #maybe some more fixes here ?
        for k in entry.keys():
            if k != "key":
                val = entry[k]
                dbEntry[k] = val
        db.Put(dbEntry)
        try:
            skel = skeletonByKind(key.kind())()
        except:
            logging.error("Unknown Skeleton - skipping")
        skel.fromDB(str(dbEntry.key()))
        skel.refresh()
        skel.toDB(clearUpdateTag=True)
Beispiel #3
0
        def updateInplace(relDict):
            if isinstance(relDict, dict) and "dest" in relDict:
                valDict = relDict["dest"]
            else:
                logging.error("Invalid dictionary in updateInplace: %s" %
                              relDict)
                return

            if "key" in valDict:
                originalKey = valDict["key"]
            else:
                logging.error("Broken fileBone dict")
                return

            entityKey = normalizeKey(originalKey)
            if originalKey != entityKey:
                logging.info("Rewriting %s to %s" % (originalKey, entityKey))
                valDict["key"] = originalKey

            # Anyway, try to copy a dlkey and servingurl
            # from the corresponding viur-blobimportmap entity.
            if "dlkey" in valDict:
                try:
                    oldKeyHash = sha256(
                        valDict["dlkey"]).hexdigest().encode("hex")

                    logging.info(
                        "Trying to fetch entry from blobimportmap with hash %s"
                        % oldKeyHash)
                    res = db.Get(
                        db.Key.from_path("viur-blobimportmap", oldKeyHash))
                except:
                    res = None

                if res and res["oldkey"] == valDict["dlkey"]:
                    valDict["dlkey"] = res["newkey"]
                    valDict["servingurl"] = res["servingurl"]

                    logging.info("Refreshing file dlkey %s (%s)" %
                                 (valDict["dlkey"], valDict["servingurl"]))
                else:
                    if valDict["servingurl"]:
                        try:
                            valDict["servingurl"] = images.get_serving_url(
                                valDict["dlkey"])
                        except Exception as e:
                            logging.exception(e)
Beispiel #4
0
        def updateInplace(relDict):
            """
				Fetches the entity referenced by valDict["dest.key"] and updates all dest.* keys
				accordingly
			"""
            if isinstance(relDict, dict) and "dest" in relDict.keys():
                valDict = relDict["dest"]
            else:
                logging.error("Invalid dictionary in updateInplace: %s" %
                              relDict)
                return

            if "key" in valDict.keys() and valDict["key"]:
                originalKey = valDict["key"]
            else:
                logging.error("Invalid dictionary in updateInplace: %s" %
                              valDict)
                return

            entityKey = normalizeKey(originalKey)
            if originalKey != entityKey:
                logging.info("Rewriting %s to %s" % (originalKey, entityKey))
                valDict["key"] = entityKey

            # Try to update referenced values;
            # If the entity does not exist with this key, ignore
            # (key was overidden above to have a new appid when transferred).
            newValues = None

            try:
                newValues = db.Get(entityKey)
                assert newValues is not None
            except db.EntityNotFoundError:
                #This entity has been deleted
                logging.info("The key %s does not exist" % entityKey)
            except:
                raise

            if newValues:
                for key in valDict.keys():
                    if key == "key":
                        continue

                    elif key in newValues.keys():
                        getattr(valDict,
                                key).unserialize(valDict.valuesCache, key,
                                                 newValues)
Beispiel #5
0
	def refresh(self, valuesCache, boneName, skel):
		"""
			Refresh all values we might have cached from other entities.
		"""
		if boneName in valuesCache and valuesCache[boneName]:
			valuesCache[boneName] = normalizeKey(valuesCache[boneName])
Beispiel #6
0
def iterImport(module, target, exportKey, cursor=None, amount=0):
    """
		Processes 100 Entries and calls the next batch
	"""
    urlfetch.set_default_fetch_deadline(20)

    payload = {"module": module, "key": exportKey}
    if cursor:
        payload.update({"cursor": cursor})

    result = urlfetch.fetch(
        url=target,
        payload=urllib.urlencode(payload),
        method=urlfetch.POST,
        headers={'Content-Type': 'application/x-www-form-urlencoded'})

    if result.status_code == 200:
        res = pickle.loads(result.content.decode("HEX"))
        skel = skeletonByKind(module)()
        logging.info("%s: %d new entries fetched, total %d entries fetched" %
                     (module, len(res["values"]), amount))

        if len(res["values"]) == 0:
            try:
                utils.sendEMailToAdmins(
                    "Import of kind %s finished with %d entities" %
                    (module, amount), "ViUR finished to import %d entities of "
                    "kind %s from %s.\n" % (amount, module, target))
            except:  #OverQuota, whatever
                logging.error("Unable to send Email")

            return

        for entry in res["values"]:
            for k in list(entry.keys())[:]:
                if isinstance(entry[k], str):
                    entry[k] = entry[k].decode("UTF-8")

            if not "key" in entry.keys():
                entry["key"] = entry["id"]

            key = db.Key(encoded=utils.normalizeKey(entry["key"]))

            # Special case: Convert old module root nodes!!!
            if module.endswith(
                    "_rootNode") and key.name() and "_modul_" in key.name():
                name = key.name().replace("_modul_", "_module_")
            else:
                name = key.name()

            dbEntry = db.Entity(kind=key.kind(),
                                parent=key.parent(),
                                id=key.id(),
                                name=name)

            for k in entry.keys():
                if k == "key":
                    continue

                dbEntry[k] = entry[k]

                # Special case: Convert old module root nodes!!!
                if (isinstance(skel, (HierarchySkel, TreeLeafSkel))
                        and k in ["parentdir", "parententry", "parentrepo"]
                        and entry[k]):

                    key = db.Key(encoded=str(entry[k]))
                    if key.parent():
                        parent = db.Key(
                            encoded=utils.normalizeKey(key.parent()))
                    else:
                        parent = None

                    if key.id_or_name() and "_modul_" in str(key.id_or_name()):
                        name = key.id_or_name().replace("_modul_", "_module_")
                    else:
                        name = key.id_or_name()

                    dbEntry[k] = str(
                        db.Key.from_path(key.kind(), name, parent=parent))

            db.Put(dbEntry)
            skel.fromDB(str(dbEntry.key()))
            skel.refresh()
            skel.toDB(clearUpdateTag=True)
            amount += 1

        iterImport(module, target, exportKey, res["cursor"], amount)
Beispiel #7
0
 def refresh(self):
     if self["parentdir"]:
         self["parentdir"] = utils.normalizeKey(self["parentdir"])
     if self["parentrepo"]:
         self["parentrepo"] = utils.normalizeKey(self["parentrepo"])
     super(TreeLeafSkel, self).refresh()