def refresh(self):
if self["parententry"]:
self["parententry"] = utils.normalizeKey(self["parententry"])
if self["parentrepo"]:
self["parentrepo"] = utils.normalizeKey(self["parentrepo"])
super(HierarchySkel, self).refresh()
def storeEntry2(self, e, key):
if not self._checkKey(key, export=False):
raise errors.Forbidden()
entry = pickle.loads(e.decode("HEX"))
for k in list(entry.keys())[:]:
if isinstance(entry[k], str):
entry[k] = entry[k].decode("UTF-8")
key = db.Key(encoded=utils.normalizeKey(entry["key"]))
logging.info(key.kind())
logging.info(key.id())
logging.info(key.name())
dbEntry = db.Entity(kind=key.kind(),
parent=key.parent(),
id=key.id(),
name=key.name()) #maybe some more fixes here ?
for k in entry.keys():
if k != "key":
val = entry[k]
dbEntry[k] = val
db.Put(dbEntry)
try:
skel = skeletonByKind(key.kind())()
except:
logging.error("Unknown Skeleton - skipping")
skel.fromDB(str(dbEntry.key()))
skel.refresh()
skel.toDB(clearUpdateTag=True)
def updateInplace(relDict):
if isinstance(relDict, dict) and "dest" in relDict:
valDict = relDict["dest"]
else:
logging.error("Invalid dictionary in updateInplace: %s" %
relDict)
return
if "key" in valDict:
originalKey = valDict["key"]
else:
logging.error("Broken fileBone dict")
return
entityKey = normalizeKey(originalKey)
if originalKey != entityKey:
logging.info("Rewriting %s to %s" % (originalKey, entityKey))
valDict["key"] = originalKey
# Anyway, try to copy a dlkey and servingurl
# from the corresponding viur-blobimportmap entity.
if "dlkey" in valDict:
try:
oldKeyHash = sha256(
valDict["dlkey"]).hexdigest().encode("hex")
logging.info(
"Trying to fetch entry from blobimportmap with hash %s"
% oldKeyHash)
res = db.Get(
db.Key.from_path("viur-blobimportmap", oldKeyHash))
except:
res = None
if res and res["oldkey"] == valDict["dlkey"]:
valDict["dlkey"] = res["newkey"]
valDict["servingurl"] = res["servingurl"]
logging.info("Refreshing file dlkey %s (%s)" %
(valDict["dlkey"], valDict["servingurl"]))
else:
if valDict["servingurl"]:
try:
valDict["servingurl"] = images.get_serving_url(
valDict["dlkey"])
except Exception as e:
logging.exception(e)
def updateInplace(relDict):
"""
Fetches the entity referenced by valDict["dest.key"] and updates all dest.* keys
accordingly
"""
if isinstance(relDict, dict) and "dest" in relDict.keys():
valDict = relDict["dest"]
else:
logging.error("Invalid dictionary in updateInplace: %s" %
relDict)
return
if "key" in valDict.keys() and valDict["key"]:
originalKey = valDict["key"]
else:
logging.error("Invalid dictionary in updateInplace: %s" %
valDict)
return
entityKey = normalizeKey(originalKey)
if originalKey != entityKey:
logging.info("Rewriting %s to %s" % (originalKey, entityKey))
valDict["key"] = entityKey
# Try to update referenced values;
# If the entity does not exist with this key, ignore
# (key was overidden above to have a new appid when transferred).
newValues = None
try:
newValues = db.Get(entityKey)
assert newValues is not None
except db.EntityNotFoundError:
#This entity has been deleted
logging.info("The key %s does not exist" % entityKey)
except:
raise
if newValues:
for key in valDict.keys():
if key == "key":
continue
elif key in newValues.keys():
getattr(valDict,
key).unserialize(valDict.valuesCache, key,
newValues)
def refresh(self, valuesCache, boneName, skel): """ Refresh all values we might have cached from other entities. """ if boneName in valuesCache and valuesCache[boneName]: valuesCache[boneName] = normalizeKey(valuesCache[boneName])
def iterImport(module, target, exportKey, cursor=None, amount=0):
"""
Processes 100 Entries and calls the next batch
"""
urlfetch.set_default_fetch_deadline(20)
payload = {"module": module, "key": exportKey}
if cursor:
payload.update({"cursor": cursor})
result = urlfetch.fetch(
url=target,
payload=urllib.urlencode(payload),
method=urlfetch.POST,
headers={'Content-Type': 'application/x-www-form-urlencoded'})
if result.status_code == 200:
res = pickle.loads(result.content.decode("HEX"))
skel = skeletonByKind(module)()
logging.info("%s: %d new entries fetched, total %d entries fetched" %
(module, len(res["values"]), amount))
if len(res["values"]) == 0:
try:
utils.sendEMailToAdmins(
"Import of kind %s finished with %d entities" %
(module, amount), "ViUR finished to import %d entities of "
"kind %s from %s.\n" % (amount, module, target))
except: #OverQuota, whatever
logging.error("Unable to send Email")
return
for entry in res["values"]:
for k in list(entry.keys())[:]:
if isinstance(entry[k], str):
entry[k] = entry[k].decode("UTF-8")
if not "key" in entry.keys():
entry["key"] = entry["id"]
key = db.Key(encoded=utils.normalizeKey(entry["key"]))
# Special case: Convert old module root nodes!!!
if module.endswith(
"_rootNode") and key.name() and "_modul_" in key.name():
name = key.name().replace("_modul_", "_module_")
else:
name = key.name()
dbEntry = db.Entity(kind=key.kind(),
parent=key.parent(),
id=key.id(),
name=name)
for k in entry.keys():
if k == "key":
continue
dbEntry[k] = entry[k]
# Special case: Convert old module root nodes!!!
if (isinstance(skel, (HierarchySkel, TreeLeafSkel))
and k in ["parentdir", "parententry", "parentrepo"]
and entry[k]):
key = db.Key(encoded=str(entry[k]))
if key.parent():
parent = db.Key(
encoded=utils.normalizeKey(key.parent()))
else:
parent = None
if key.id_or_name() and "_modul_" in str(key.id_or_name()):
name = key.id_or_name().replace("_modul_", "_module_")
else:
name = key.id_or_name()
dbEntry[k] = str(
db.Key.from_path(key.kind(), name, parent=parent))
db.Put(dbEntry)
skel.fromDB(str(dbEntry.key()))
skel.refresh()
skel.toDB(clearUpdateTag=True)
amount += 1
iterImport(module, target, exportKey, res["cursor"], amount)
def refresh(self):
if self["parentdir"]:
self["parentdir"] = utils.normalizeKey(self["parentdir"])
if self["parentrepo"]:
self["parentrepo"] = utils.normalizeKey(self["parentrepo"])
super(TreeLeafSkel, self).refresh()