def createUploadURL(
self, node: Union[str, None]) -> Tuple[str, str, Dict[str, str]]:
global bucket
targetKey = utils.generateRandomString()
conditions = [["starts-with", "$key", "%s/source/" % targetKey]]
if isinstance(credentials, ServiceAccountCredentials
): # We run locally with an service-account.json
policy = bucket.generate_upload_policy(conditions)
else: # Use our fixed PolicyGenerator - Google is currently unable to create one itself on its GCE
policy = self.generateUploadPolicy(conditions)
uploadUrl = "https://%s.storage.googleapis.com" % bucket.name
# Create a correspondingfile-lock object early, otherwise we would have to ensure that the file-lock object
# the user creates matches the file he had uploaded
fileSkel = self.addSkel(TreeType.Leaf)
fileSkel["key"] = targetKey
fileSkel["name"] = "pending"
fileSkel["size"] = 0
fileSkel["mimetype"] = "application/octetstream"
fileSkel["dlkey"] = targetKey
fileSkel["parentdir"] = None
fileSkel["pendingparententry"] = db.keyHelper(
node,
self.addSkel(TreeType.Node).kindName) if node else None
fileSkel["pending"] = True
fileSkel["weak"] = True
fileSkel["width"] = 0
fileSkel["height"] = 0
fileSkel.toDB()
# Mark that entry dirty as we might never receive an add
utils.markFileForDeletion(targetKey)
return targetKey, uploadUrl, policy
def createUploadURL(
self, node: Union[str, None]) -> Tuple[str, str, Dict[str, str]]:
targetKey = utils.generateRandomString()
conditions = [["starts-with", "$key", "%s/source/" % targetKey]]
policy = bucket.generate_upload_policy(conditions)
uploadUrl = "https://%s.storage.googleapis.com" % bucket.name
# Create a correspondingfile-lock object early, otherwise we would have to ensure that the file-lock object
# the user creates matches the file he had uploaded
fileSkel = self.addLeafSkel()
fileSkel["key"] = targetKey
fileSkel["name"] = "pending"
fileSkel["size"] = 0
fileSkel["mimetype"] = "application/octetstream"
fileSkel["dlkey"] = targetKey
fileSkel["parentdir"] = None
fileSkel["pendingParentdir"] = db.keyHelper(
node,
self.addNodeSkel().kindName) if node else None
fileSkel["pending"] = True
fileSkel["weak"] = True
fileSkel["width"] = 0
fileSkel["height"] = 0
fileSkel[""] = ""
fileSkel.toDB()
# Mark that entry dirty as we might never receive an add
utils.markFileForDeletion(targetKey)
return targetKey, uploadUrl, policy
def updateInplace(relDict):
"""
Fetches the entity referenced by valDict["dest.key"] and updates all dest.* keys
accordingly
"""
if not (isinstance(relDict, dict) and "dest" in relDict):
logging.error("Invalid dictionary in updateInplace: %s" % relDict)
return
newValues = db.Get(db.keyHelper(relDict["dest"]["key"], self.kind))
if newValues is None:
logging.info("The key %s does not exist" % relDict["dest"]["key"])
return
for boneName in self.refKeys:
if boneName != "key" and boneName in newValues:
relDict["dest"].dbEntity[boneName] = newValues[boneName]
def getRootNode(self, entryKey: db.Key) -> Skeleton:
"""
Returns the root-node for a given child.
:param entryKey: URL-Safe key of the child entry
:type entryKey: str
:returns: The entity of the root-node.
:rtype: :class:`server.db.Entity`
"""
rootNodeSkel = self.nodeSkelCls()
entryKey = db.keyHelper(entryKey, rootNodeSkel.kindName)
repo = db.Get(entryKey)
while repo and "parententry" in repo:
repo = db.Get(repo["parententry"])
rootNodeSkel.fromDB(repo.key)
return rootNodeSkel
def createRelSkelFromKey(self, key: Union[str, db.KeyClass], rel: Union[dict, None] = None):
"""
Creates a relSkel instance valid for this bone from the given database key.
"""
key = db.keyHelper(key, self.kind)
entity = db.Get(key)
if not entity:
logging.error("Key %s not found" % str(key))
return None
relSkel = self._refSkelCache()
relSkel.unserialize(entity)
for k in relSkel.keys():
# Unserialize all bones from refKeys, then drop dbEntity - otherwise all properties will be copied
_ = relSkel[k]
relSkel.dbEntity = None
return {
"dest": relSkel,
"rel": rel or None
}
def restoreSkels(key, usingData, index=None):
refSkel, usingSkel = self._getSkels()
isEntryFromBackup = False # If the referenced entry has been deleted, restore information from backup
entry = None
dbKey = None
errors = []
try:
dbKey = db.keyHelper(key, self.kind)
entry = db.Get(dbKey)
assert entry
except: # Invalid key or something like that
logging.info("Invalid reference key >%s< detected on bone '%s'",
key, name)
if isinstance(oldValues, dict):
if oldValues["dest"].key == dbKey:
entry = oldValues["dest"]
isEntryFromBackup = True
elif isinstance(oldValues, list):
for dbVal in oldValues:
if dbVal["dest"].key == dbKey:
entry = dbVal["dest"]
isEntryFromBackup = True
if isEntryFromBackup:
refSkel = entry
elif entry:
refSkel.dbEntity = entry
for k in refSkel.keys():
# Unserialize all bones from refKeys, then drop dbEntity - otherwise all properties will be copied
_ = refSkel[k]
refSkel.dbEntity = None
else:
if index:
errors.append(
ReadFromClientError(ReadFromClientErrorSeverity.Invalid, "%s.%s" % (name, index),
"Invalid value submitted"))
else:
errors.append(
ReadFromClientError(ReadFromClientErrorSeverity.Invalid, name, "Invalid value submitted"))
return None, None, errors # We could not parse this
if usingSkel:
if not usingSkel.fromClient(usingData):
errors.extend(usingSkel.errors)
return refSkel, usingSkel, errors
def initializeUpload(self,
fileName: str,
mimeType: str,
node: Union[str, None],
size: Union[int, None] = None) -> Tuple[str, str]:
"""
Internal helper that registers a new upload. Will create the pending fileSkel entry (needed to remove any
started uploads from GCS if that file isn't used) and creates a resumable (and signed) uploadURL for that.
:param fileName: Name of the file that will be uploaded
:param mimeType: Mimetype of said file
:param node: If set (to a string-key representation of a file-node) the upload will be written to this directory
:param size: The *exact* filesize we're accepting in Bytes. Used to enforce a filesize limit by getUploadURL
:return: Str-Key of the new file-leaf entry, the signed upload-url
"""
global bucket
fileName = sanitizeFileName(fileName)
targetKey = utils.generateRandomString()
blob = bucket.blob("%s/source/%s" % (targetKey, fileName))
uploadUrl = blob.create_resumable_upload_session(content_type=mimeType,
size=size,
timeout=60)
# Create a corresponding file-lock object early, otherwise we would have to ensure that the file-lock object
# the user creates matches the file he had uploaded
fileSkel = self.addSkel("leaf")
fileSkel["name"] = "pending"
fileSkel["size"] = 0
fileSkel["mimetype"] = "application/octetstream"
fileSkel["dlkey"] = targetKey
fileSkel["parentdir"] = None
fileSkel["pendingparententry"] = db.keyHelper(
node,
self.addSkel("node").kindName) if node else None
fileSkel["pending"] = True
fileSkel["weak"] = True
fileSkel["width"] = 0
fileSkel["height"] = 0
fileSkel.toDB()
# Mark that entry dirty as we might never receive an add
utils.markFileForDeletion(targetKey)
return db.encodeKey(fileSkel["key"]), uploadUrl
def deleteRecursive(self, parentKey):
"""
Recursively processes a delete request.
This will delete all entries which are children of *nodeKey*, except *key* nodeKey.
:param key: URL-safe key of the node which children should be deleted.
:type key: str
"""
nodeKey = db.keyHelper(parentKey, self.viewSkel("node").kindName)
if self.leafSkelCls:
for leaf in db.Query(self.viewSkel("leaf").kindName).filter(
"parententry =", nodeKey).iter():
leafSkel = self.viewSkel("leaf")
if not leafSkel.fromDB(leaf.key):
continue
leafSkel.delete()
for node in db.Query(self.viewSkel("node").kindName).filter(
"parententry =", nodeKey).iter():
self.deleteRecursive(node.key)
nodeSkel = self.viewSkel("node")
if not nodeSkel.fromDB(node.key):
continue
nodeSkel.delete()
def deleteRecursive(self, nodeKey):
"""
Recursively processes a delete request.
This will delete all entries which are children of *nodeKey*, except *key* nodeKey.
:param key: URL-safe key of the node which children should be deleted.
:type key: str
"""
nodeKey = db.keyHelper(nodeKey, self.viewSkel("node").kindName)
if self.leafSkelCls:
for f in db.Query(self.viewSkel("leaf").kindName).filter(
"parententry =", nodeKey).iter(keysOnly=True):
s = self.viewSkel("leaf")
if not s.fromDB(f):
continue
s.delete()
for d in db.Query(self.viewSkel("node").kindName).filter(
"parententry =", nodeKey).iter(keysOnly=True):
self.deleteRecursive(str(d))
s = self.viewSkel("node")
if not s.fromDB(d):
continue
s.delete()
