def archive(project, job_id): """Redirects to the actual job archive. This is done via a redirect, so that templates that offer a download link do not need to know the eventual storage backend link. This makes them render faster, and only requires communication with the storage backend when needed. """ from pillar.api.file_storage_backends import default_storage_backend from . import ARCHIVE_JOB_STATES from .sdk import Job api = pillar_api() job = Job.find(job_id, api=api) if job['status'] not in ARCHIVE_JOB_STATES: raise wz_exceptions.PreconditionFailed('Job is not archived') archive_blob_name = job.archive_blob_name if not archive_blob_name: raise wz_exceptions.NotFound('Job has no archive') bucket = default_storage_backend(project._id) blob = bucket.get_blob(archive_blob_name) archive_url = blob.get_url(is_public=False) return redirect(archive_url, code=303)
def archive(project, job_id): """Redirects to the actual job archive. This is done via a redirect, so that templates that offer a download link do not need to know the eventual storage backend link. This makes them render faster, and only requires communication with the storage backend when needed. """ from pillar.api.file_storage_backends import default_storage_backend from . import ARCHIVE_JOB_STATES from .sdk import Job api = pillar_api() job = Job.find(job_id, api=api) if job['status'] not in ARCHIVE_JOB_STATES: raise wz_exceptions.PreconditionFailed('Job is not archived') archive_blob_name = job.archive_blob_name if not archive_blob_name: raise wz_exceptions.NotFound('Job has no archive') bucket = default_storage_backend(project._id) blob = bucket.get_blob(archive_blob_name) archive_url = blob.get_url(is_public=False) return redirect(archive_url, code=303)
def storage(project_id: ObjectId) -> file_storage_backends.Bucket: """Return the storage bucket for this project. For now this returns a bucket in the default storage backend, since individual projects do not have a 'storage backend' setting (this is set per file, not per project). """ return file_storage_backends.default_storage_backend(str(project_id))
def upload_and_process(local_file: typing.Union[io.BytesIO, typing.BinaryIO], uploaded_file: werkzeug.datastructures.FileStorage, project_id: str): # Figure out the file size, as we need to pass this in explicitly to GCloud. # Otherwise it always uses os.fstat(file_obj.fileno()).st_size, which isn't # supported by a BytesIO object (even though it does have a fileno # attribute). if isinstance(local_file, io.BytesIO): file_size = len(local_file.getvalue()) else: file_size = os.fstat(local_file.fileno()).st_size # Check the file size again, now that we know its size for sure. assert_file_size_allowed(file_size) # Create file document in MongoDB. file_id, internal_fname, status = create_file_doc_for_upload( project_id, uploaded_file) # Copy the file into storage. bucket = default_storage_backend(project_id) blob = bucket.blob(internal_fname) blob.create_from_file(local_file, file_size=file_size, content_type=uploaded_file.mimetype) log.debug( 'Marking uploaded file id=%s, fname=%s, ' 'size=%i as "queued_for_processing"', file_id, internal_fname, file_size) update_file_doc(file_id, status='queued_for_processing', file_path=internal_fname, length=blob.size, content_type=uploaded_file.mimetype) log.debug('Processing uploaded file id=%s, fname=%s, size=%i', file_id, internal_fname, blob.size) process_file(bucket, file_id, local_file) # Local processing is done, we can close the local file so it is removed. if local_file is not None: local_file.close() log.debug('Handled uploaded file id=%s, fname=%s, size=%i, status=%i', file_id, internal_fname, blob.size, status) # Status is 200 if the file already existed, and 201 if it was newly # created. # TODO: add a link to a thumbnail in the response. return dict(status='ok', file_id=str(file_id), status_code=status)
def create_upload_zip(project_id: str, storage_path: str, zip_path: str) -> str: """Uploads the ZIP file to the storage backend. Also stores the link to the ZIP in the job document. Returns the name of the storage blob the ZIP is stored in. """ import itertools import zipfile import secrets import datetime from pillar.api.file_storage_backends import default_storage_backend log.info('Creating ZIP %s', zip_path) spath = pathlib.Path(storage_path) zpath = pathlib.Path(zip_path) with zipfile.ZipFile(zip_path, mode='w', compression=zipfile.ZIP_DEFLATED) as outfile: for fpath in itertools.chain(spath.glob('*.gz'), spath.glob('*.json')): outfile.write(fpath, fpath.name) bucket = default_storage_backend(project_id) blob = bucket.blob(f'flamenco-jobs/{zpath.name}') while blob.exists(): log.warning( 'Blob %s already exists, changing filename to something more unique.', blob) today = datetime.date.today() uniquifier = f'{today:%Y-%m-%d}-{secrets.token_hex(4)}' new_zipname = f'{zpath.stem}-{uniquifier}{zpath.suffix}' blob = bucket.blob(f'flamenco-jobs/{new_zipname}') log.info('Uploading ZIP %s to %s', zpath, blob) file_size = zpath.stat().st_size with zpath.open(mode='rb') as stream_to_upload: blob.create_from_file(stream_to_upload, file_size=file_size, content_type='application/zip') return blob.name
def create_upload_zip(project_id: str, storage_path: str, zip_path: str) -> str: """Uploads the ZIP file to the storage backend. Also stores the link to the ZIP in the job document. Returns the name of the storage blob the ZIP is stored in. """ import itertools import zipfile import secrets import datetime from pillar.api.file_storage_backends import default_storage_backend log.info('Creating ZIP %s', zip_path) spath = pathlib.Path(storage_path) zpath = pathlib.Path(zip_path) with zipfile.ZipFile(zip_path, mode='w', compression=zipfile.ZIP_DEFLATED) as outfile: for fpath in itertools.chain(spath.glob('*.gz'), spath.glob('*.json')): outfile.write(fpath, fpath.name) bucket = default_storage_backend(project_id) blob = bucket.blob(f'flamenco-jobs/{zpath.name}') while blob.exists(): log.warning('Blob %s already exists, changing filename to something more unique.', blob) today = datetime.date.today() uniquifier = f'{today:%Y-%m-%d}-{secrets.token_hex(4)}' new_zipname = f'{zpath.stem}-{uniquifier}{zpath.suffix}' blob = bucket.blob(f'flamenco-jobs/{new_zipname}') log.info('Uploading ZIP %s to %s', zpath, blob) file_size = zpath.stat().st_size with zpath.open(mode='rb') as stream_to_upload: blob.create_from_file(stream_to_upload, file_size=file_size, content_type='application/zip') return blob.name
def remove_deleted_files(): """Iterate over every _deleted file, remove file from the system and delete doc.""" from pillar.api.file_storage_backends import default_storage_backend from pillar.api.file_storage_backends.local import LocalBlob file_collection = current_app.db()['files'] files = file_collection.find({'_deleted': True}) for f in files: bucket = default_storage_backend(f['project']) blob = bucket.get_blob(f['name']) if not isinstance(blob, LocalBlob): log.info('Skipping non local blob') continue file_abspath = blob.abspath() log.info('Removing %s from filesystem' % file_abspath) try: file_abspath.unlink() except OSError: log.error('File %s not found' % file_abspath) log.info('Deleting file document for file %s' % f['_id']) file_collection.delete_one({'_id': f['_id']})
def after_inserting_project(project, db_user): from pillar.auth import UserClass project_id = project['_id'] user_id = db_user['_id'] # Create a project-specific admin group (with name matching the project id) result, _, _, status = current_app.post_internal('groups', {'name': str(project_id)}) if status != 201: log.error('Unable to create admin group for new project %s: %s', project_id, result) return abort_with_error(status) admin_group_id = result['_id'] log.debug('Created admin group %s for project %s', admin_group_id, project_id) # Assign the current user to the group db_user.setdefault('groups', []).append(admin_group_id) result, _, _, status = current_app.patch_internal( 'users', {'groups': db_user['groups']}, _id=user_id) if status != 200: log.error( 'Unable to add user %s as member of admin group %s for new project %s: %s', user_id, admin_group_id, project_id, result) return abort_with_error(status) log.debug('Made user %s member of group %s', user_id, admin_group_id) # Assign the group to the project with admin rights owner_user = UserClass.construct('', db_user) is_admin = authorization.is_admin(owner_user) world_permissions = ['GET'] if is_admin else [] permissions = { 'world': world_permissions, 'users': [], 'groups': [ { 'group': admin_group_id, 'methods': DEFAULT_ADMIN_GROUP_PERMISSIONS[:] }, ] } def with_permissions(node_type): copied = copy.deepcopy(node_type) copied['permissions'] = permissions return copied # Assign permissions to the project itself, as well as to the node_types project['permissions'] = permissions project['node_types'] = [ with_permissions(node_type_group), with_permissions(node_type_asset), with_permissions(node_type_comment), with_permissions(node_type_texture), with_permissions(node_type_group_texture), ] # Allow admin users to use whatever url they want. if not is_admin or not project.get('url'): if project.get('category', '') == 'home': project['url'] = 'home' else: project['url'] = "p-{!s}".format(project_id) # Initialize storage using the default specified in STORAGE_BACKEND default_storage_backend(str(project_id)) # Commit the changes directly to the MongoDB; a PUT is not allowed yet, # as the project doesn't have a valid permission structure. projects_collection = current_app.data.driver.db['projects'] result = projects_collection.update_one( {'_id': project_id}, {'$set': remove_private_keys(project)}) if result.matched_count != 1: log.error('Unable to update project %s: %s', project_id, result.raw_result) abort_with_error(500)