def createUploadURL(
self, node: Union[str, None]) -> Tuple[str, str, Dict[str, str]]:
global bucket
targetKey = utils.generateRandomString()
conditions = [["starts-with", "$key", "%s/source/" % targetKey]]
if isinstance(credentials, ServiceAccountCredentials
): # We run locally with an service-account.json
policy = bucket.generate_upload_policy(conditions)
else: # Use our fixed PolicyGenerator - Google is currently unable to create one itself on its GCE
policy = self.generateUploadPolicy(conditions)
uploadUrl = "https://%s.storage.googleapis.com" % bucket.name
# Create a correspondingfile-lock object early, otherwise we would have to ensure that the file-lock object
# the user creates matches the file he had uploaded
fileSkel = self.addSkel(TreeType.Leaf)
fileSkel["key"] = targetKey
fileSkel["name"] = "pending"
fileSkel["size"] = 0
fileSkel["mimetype"] = "application/octetstream"
fileSkel["dlkey"] = targetKey
fileSkel["parentdir"] = None
fileSkel["pendingparententry"] = db.keyHelper(
node,
self.addSkel(TreeType.Node).kindName) if node else None
fileSkel["pending"] = True
fileSkel["weak"] = True
fileSkel["width"] = 0
fileSkel["height"] = 0
fileSkel.toDB()
# Mark that entry dirty as we might never receive an add
utils.markFileForDeletion(targetKey)
return targetKey, uploadUrl, policy
def createUploadURL(
self, node: Union[str, None]) -> Tuple[str, str, Dict[str, str]]:
targetKey = utils.generateRandomString()
conditions = [["starts-with", "$key", "%s/source/" % targetKey]]
policy = bucket.generate_upload_policy(conditions)
uploadUrl = "https://%s.storage.googleapis.com" % bucket.name
# Create a correspondingfile-lock object early, otherwise we would have to ensure that the file-lock object
# the user creates matches the file he had uploaded
fileSkel = self.addLeafSkel()
fileSkel["key"] = targetKey
fileSkel["name"] = "pending"
fileSkel["size"] = 0
fileSkel["mimetype"] = "application/octetstream"
fileSkel["dlkey"] = targetKey
fileSkel["parentdir"] = None
fileSkel["pendingParentdir"] = db.keyHelper(
node,
self.addNodeSkel().kindName) if node else None
fileSkel["pending"] = True
fileSkel["weak"] = True
fileSkel["width"] = 0
fileSkel["height"] = 0
fileSkel[""] = ""
fileSkel.toDB()
# Mark that entry dirty as we might never receive an add
utils.markFileForDeletion(targetKey)
return targetKey, uploadUrl, policy
def deleteRecursive(self, parentKey):
files = db.Query(self.leafSkelCls().kindName).filter("parentdir =", parentKey).iter()
for fileEntry in files:
utils.markFileForDeletion(fileEntry["dlkey"])
skel = self.leafSkelCls()
if skel.fromDB(str(fileEntry.key())):
skel.delete()
dirs = db.Query(self.nodeSkelCls().kindName).filter("parentdir", parentKey).iter(keysOnly=True)
for d in dirs:
self.deleteRecursive(str(d))
skel = self.nodeSkelCls()
if skel.fromDB(str(d)):
skel.delete()
def initializeUpload(self,
fileName: str,
mimeType: str,
node: Union[str, None],
size: Union[int, None] = None) -> Tuple[str, str]:
"""
Internal helper that registers a new upload. Will create the pending fileSkel entry (needed to remove any
started uploads from GCS if that file isn't used) and creates a resumable (and signed) uploadURL for that.
:param fileName: Name of the file that will be uploaded
:param mimeType: Mimetype of said file
:param node: If set (to a string-key representation of a file-node) the upload will be written to this directory
:param size: The *exact* filesize we're accepting in Bytes. Used to enforce a filesize limit by getUploadURL
:return: Str-Key of the new file-leaf entry, the signed upload-url
"""
global bucket
fileName = sanitizeFileName(fileName)
targetKey = utils.generateRandomString()
blob = bucket.blob("%s/source/%s" % (targetKey, fileName))
uploadUrl = blob.create_resumable_upload_session(content_type=mimeType,
size=size,
timeout=60)
# Create a corresponding file-lock object early, otherwise we would have to ensure that the file-lock object
# the user creates matches the file he had uploaded
fileSkel = self.addSkel("leaf")
fileSkel["name"] = "pending"
fileSkel["size"] = 0
fileSkel["mimetype"] = "application/octetstream"
fileSkel["dlkey"] = targetKey
fileSkel["parentdir"] = None
fileSkel["pendingparententry"] = db.keyHelper(
node,
self.addSkel("node").kindName) if node else None
fileSkel["pending"] = True
fileSkel["weak"] = True
fileSkel["width"] = 0
fileSkel["height"] = 0
fileSkel.toDB()
# Mark that entry dirty as we might never receive an add
utils.markFileForDeletion(targetKey)
return db.encodeKey(fileSkel["key"]), uploadUrl
def upload(self, node=None, *args, **kwargs):
try:
canAdd = self.canAdd("leaf", node)
except:
canAdd = False
if not canAdd:
for upload in self.getUploads():
upload.delete()
raise errors.Forbidden()
try:
res = []
if node:
# The file is uploaded into a rootNode
nodeSkel = self.editNodeSkel()
if not nodeSkel.fromDB(node):
for upload in self.getUploads():
upload.delete()
raise errors.NotFound()
else:
weak = False
parentDir = str(node)
parentRepo = nodeSkel["parentrepo"]
else:
weak = True
parentDir = None
parentRepo = None
# Handle the actual uploads
for upload in self.getUploads():
fileName = decodeFileName(upload.filename)
height = width = 0
if str(upload.content_type).startswith("image/"):
try:
servingURL = images.get_serving_url(upload.key())
if request.current.get().isDevServer:
# NOTE: changed for Ticket ADMIN-37
servingURL = urlparse(servingURL).path
elif servingURL.startswith("http://"):
# Rewrite Serving-URLs to https if we are live
servingURL = servingURL.replace(
"http://", "https://")
except:
servingURL = ""
try:
# only fetching the file header or all if the file is smaller than 1M
data = blobstore.fetch_data(upload.key(), 0,
min(upload.size, 1000000))
image = images.Image(image_data=data)
height = image.height
width = image.width
except Exception as err:
logging.error(
"some error occurred while trying to fetch the image header with dimensions"
)
logging.exception(err)
else:
servingURL = ""
fileSkel = self.addLeafSkel()
fileSkel.setValues({
"name":
utils.escapeString(fileName),
"size":
upload.size,
"mimetype":
utils.escapeString(upload.content_type),
"dlkey":
str(upload.key()),
"servingurl":
servingURL,
"parentdir":
parentDir,
"parentrepo":
parentRepo,
"weak":
weak,
"width":
width,
"height":
height
})
fileSkel.toDB()
res.append(fileSkel)
self.onItemUploaded(fileSkel)
# Uploads stored successfully, generate response to the client
for r in res:
logging.info("Upload successful: %s (%s)" %
(r["name"], r["dlkey"]))
user = utils.getCurrentUser()
if user:
logging.info("User: %s (%s)" % (user["name"], user["key"]))
return self.render.addItemSuccess(res)
except Exception as err:
logging.exception(err)
for upload in self.getUploads():
upload.delete()
utils.markFileForDeletion(str(upload.key()))
raise errors.InternalServerError()