def createUploadURL( self, node: Union[str, None]) -> Tuple[str, str, Dict[str, str]]: global bucket targetKey = utils.generateRandomString() conditions = [["starts-with", "$key", "%s/source/" % targetKey]] if isinstance(credentials, ServiceAccountCredentials ): # We run locally with an service-account.json policy = bucket.generate_upload_policy(conditions) else: # Use our fixed PolicyGenerator - Google is currently unable to create one itself on its GCE policy = self.generateUploadPolicy(conditions) uploadUrl = "https://%s.storage.googleapis.com" % bucket.name # Create a correspondingfile-lock object early, otherwise we would have to ensure that the file-lock object # the user creates matches the file he had uploaded fileSkel = self.addSkel(TreeType.Leaf) fileSkel["key"] = targetKey fileSkel["name"] = "pending" fileSkel["size"] = 0 fileSkel["mimetype"] = "application/octetstream" fileSkel["dlkey"] = targetKey fileSkel["parentdir"] = None fileSkel["pendingparententry"] = db.keyHelper( node, self.addSkel(TreeType.Node).kindName) if node else None fileSkel["pending"] = True fileSkel["weak"] = True fileSkel["width"] = 0 fileSkel["height"] = 0 fileSkel.toDB() # Mark that entry dirty as we might never receive an add utils.markFileForDeletion(targetKey) return targetKey, uploadUrl, policy
def createUploadURL( self, node: Union[str, None]) -> Tuple[str, str, Dict[str, str]]: targetKey = utils.generateRandomString() conditions = [["starts-with", "$key", "%s/source/" % targetKey]] policy = bucket.generate_upload_policy(conditions) uploadUrl = "https://%s.storage.googleapis.com" % bucket.name # Create a correspondingfile-lock object early, otherwise we would have to ensure that the file-lock object # the user creates matches the file he had uploaded fileSkel = self.addLeafSkel() fileSkel["key"] = targetKey fileSkel["name"] = "pending" fileSkel["size"] = 0 fileSkel["mimetype"] = "application/octetstream" fileSkel["dlkey"] = targetKey fileSkel["parentdir"] = None fileSkel["pendingParentdir"] = db.keyHelper( node, self.addNodeSkel().kindName) if node else None fileSkel["pending"] = True fileSkel["weak"] = True fileSkel["width"] = 0 fileSkel["height"] = 0 fileSkel[""] = "" fileSkel.toDB() # Mark that entry dirty as we might never receive an add utils.markFileForDeletion(targetKey) return targetKey, uploadUrl, policy
def deleteRecursive(self, parentKey): files = db.Query(self.leafSkelCls().kindName).filter("parentdir =", parentKey).iter() for fileEntry in files: utils.markFileForDeletion(fileEntry["dlkey"]) skel = self.leafSkelCls() if skel.fromDB(str(fileEntry.key())): skel.delete() dirs = db.Query(self.nodeSkelCls().kindName).filter("parentdir", parentKey).iter(keysOnly=True) for d in dirs: self.deleteRecursive(str(d)) skel = self.nodeSkelCls() if skel.fromDB(str(d)): skel.delete()
def initializeUpload(self, fileName: str, mimeType: str, node: Union[str, None], size: Union[int, None] = None) -> Tuple[str, str]: """ Internal helper that registers a new upload. Will create the pending fileSkel entry (needed to remove any started uploads from GCS if that file isn't used) and creates a resumable (and signed) uploadURL for that. :param fileName: Name of the file that will be uploaded :param mimeType: Mimetype of said file :param node: If set (to a string-key representation of a file-node) the upload will be written to this directory :param size: The *exact* filesize we're accepting in Bytes. Used to enforce a filesize limit by getUploadURL :return: Str-Key of the new file-leaf entry, the signed upload-url """ global bucket fileName = sanitizeFileName(fileName) targetKey = utils.generateRandomString() blob = bucket.blob("%s/source/%s" % (targetKey, fileName)) uploadUrl = blob.create_resumable_upload_session(content_type=mimeType, size=size, timeout=60) # Create a corresponding file-lock object early, otherwise we would have to ensure that the file-lock object # the user creates matches the file he had uploaded fileSkel = self.addSkel("leaf") fileSkel["name"] = "pending" fileSkel["size"] = 0 fileSkel["mimetype"] = "application/octetstream" fileSkel["dlkey"] = targetKey fileSkel["parentdir"] = None fileSkel["pendingparententry"] = db.keyHelper( node, self.addSkel("node").kindName) if node else None fileSkel["pending"] = True fileSkel["weak"] = True fileSkel["width"] = 0 fileSkel["height"] = 0 fileSkel.toDB() # Mark that entry dirty as we might never receive an add utils.markFileForDeletion(targetKey) return db.encodeKey(fileSkel["key"]), uploadUrl
def upload(self, node=None, *args, **kwargs): try: canAdd = self.canAdd("leaf", node) except: canAdd = False if not canAdd: for upload in self.getUploads(): upload.delete() raise errors.Forbidden() try: res = [] if node: # The file is uploaded into a rootNode nodeSkel = self.editNodeSkel() if not nodeSkel.fromDB(node): for upload in self.getUploads(): upload.delete() raise errors.NotFound() else: weak = False parentDir = str(node) parentRepo = nodeSkel["parentrepo"] else: weak = True parentDir = None parentRepo = None # Handle the actual uploads for upload in self.getUploads(): fileName = decodeFileName(upload.filename) height = width = 0 if str(upload.content_type).startswith("image/"): try: servingURL = images.get_serving_url(upload.key()) if request.current.get().isDevServer: # NOTE: changed for Ticket ADMIN-37 servingURL = urlparse(servingURL).path elif servingURL.startswith("http://"): # Rewrite Serving-URLs to https if we are live servingURL = servingURL.replace( "http://", "https://") except: servingURL = "" try: # only fetching the file header or all if the file is smaller than 1M data = blobstore.fetch_data(upload.key(), 0, min(upload.size, 1000000)) image = images.Image(image_data=data) height = image.height width = image.width except Exception as err: logging.error( "some error occurred while trying to fetch the image header with dimensions" ) logging.exception(err) else: servingURL = "" fileSkel = self.addLeafSkel() fileSkel.setValues({ "name": utils.escapeString(fileName), "size": upload.size, "mimetype": utils.escapeString(upload.content_type), "dlkey": str(upload.key()), "servingurl": servingURL, "parentdir": parentDir, "parentrepo": parentRepo, "weak": weak, "width": width, "height": height }) fileSkel.toDB() res.append(fileSkel) self.onItemUploaded(fileSkel) # Uploads stored successfully, generate response to the client for r in res: logging.info("Upload successful: %s (%s)" % (r["name"], r["dlkey"])) user = utils.getCurrentUser() if user: logging.info("User: %s (%s)" % (user["name"], user["key"])) return self.render.addItemSuccess(res) except Exception as err: logging.exception(err) for upload in self.getUploads(): upload.delete() utils.markFileForDeletion(str(upload.key())) raise errors.InternalServerError()