def create_file_doc_for_upload(project_id, uploaded_file): """Creates a secure filename and a document in MongoDB for the file. The (project_id, filename) tuple should be unique. If such a document already exists, it is updated with the new file. :param uploaded_file: file from request.files['form-key'] :type uploaded_file: werkzeug.datastructures.FileStorage :returns: a tuple (file_id, filename, status), where 'filename' is the internal filename used on GCS. """ project_id = ObjectId(project_id) # Hash the filename with path info to get the internal name. This should # be unique for the project. # internal_filename = uploaded_file.filename _, ext = os.path.splitext(uploaded_file.filename) internal_filename = uuid.uuid4().hex + ext # For now, we don't support overwriting files, and create a new one every time. # # See if we can find a pre-existing file doc. # files = current_app.data.driver.db['files'] # file_doc = files.find_one({'project': project_id, # 'name': internal_filename}) file_doc = None # TODO: at some point do name-based and content-based content-type sniffing. new_props = { 'filename': uploaded_file.filename, 'content_type': uploaded_file.mimetype, 'length': uploaded_file.content_length, 'project': project_id, 'status': 'uploading' } if file_doc is None: # Create a file document on MongoDB for this file. file_doc = create_file_doc(name=internal_filename, **new_props) file_fields, _, _, status = current_app.post_internal( 'files', file_doc) else: file_doc.update(new_props) file_fields, _, _, status = current_app.put_internal( 'files', remove_private_keys(file_doc)) if status not in (200, 201): log.error( 'Unable to create new file document in MongoDB, status=%i: %s', status, file_fields) raise wz_exceptions.InternalServerError() log.debug( 'Created file document %s for uploaded file %s; internal name %s', file_fields['_id'], uploaded_file.filename, internal_filename) return file_fields['_id'], internal_filename, status
def edit_comment(user_id, node_id, patch): """Edits a single comment. Doesn't do permission checking; users are allowed to edit their own comment, and this is not something you want to revoke anyway. Admins can edit all comments. """ # Find the node. We need to fetch some more info than we use here, so that # we can pass this stuff to Eve's patch_internal; that way the validation & # authorisation system has enough info to work. nodes_coll = current_app.data.driver.db['nodes'] node = nodes_coll.find_one(node_id) if node is None: log.warning('User %s wanted to patch non-existing node %s' % (user_id, node_id)) raise wz_exceptions.NotFound('Node %s not found' % node_id) if node['user'] != user_id and not authorization.user_has_role('admin'): raise wz_exceptions.Forbidden('You can only edit your own comments.') node = remove_private_keys(node) node['properties']['content'] = patch['content'] node['properties']['attachments'] = patch.get('attachments', {}) # Use Eve to PUT this node, as that also updates the etag and we want to replace attachments. r, _, _, status = current_app.put_internal('nodes', node, concurrency_check=False, _id=node_id) if status != 200: log.error('Error %i editing comment %s for user %s: %s', status, node_id, user_id, r) raise wz_exceptions.InternalServerError('Internal error %i from Eve' % status) else: log.info('User %s edited comment %s', user_id, node_id) # Fetch the new content, so the client can show these without querying again. node = nodes_coll.find_one(node_id, projection={ 'properties.content': 1, 'properties._content_html': 1, }) return status, node
def _update_project(project): """Updates a project in the database, or SystemExit()s. :param project: the project data, should be the entire project document :type: dict :return: the project :rtype: dict """ from pillar.api.utils import remove_private_keys project_id = ObjectId(project['_id']) project = remove_private_keys(project) result, _, _, status_code = current_app.put_internal('projects', project, _id=project_id) if status_code != 200: raise RuntimeError(f"Can't update project {project_id}, " f"status {status_code} with issues: {result}")
def process_file(bucket: Bucket, file_id: typing.Union[str, ObjectId], local_file: tempfile._TemporaryFileWrapper): """Process the file by creating thumbnails, sending to Zencoder, etc. :param file_id: '_id' key of the file :param local_file: locally stored file, or None if no local processing is needed. """ file_id = ObjectId(file_id) # Fetch the src_file document from MongoDB. files = current_app.data.driver.db['files'] src_file = files.find_one(file_id) if not src_file: log.warning('process_file(%s): no such file document found, ignoring.') return src_file = utils.remove_private_keys(src_file) # Update the 'format' field from the content type. # TODO: overrule the content type based on file extention & magic numbers. mime_category, src_file['format'] = src_file['content_type'].split('/', 1) # Only allow video encoding when the user has the correct capability. if not current_user.has_cap('encode-video') and mime_category == 'video': if src_file['format'].startswith('x-'): xified = src_file['format'] else: xified = 'x-' + src_file['format'] src_file['content_type'] = 'application/%s' % xified mime_category = 'application' log.info('Not processing video file %s for non-video-encoding user', file_id) # Run the required processor, based on the MIME category. processors: typing.Mapping[str, typing.Callable] = { 'image': _process_image, 'video': _process_video, } try: processor = processors[mime_category] except KeyError: log.info( "POSTed file %s was of type %r, which isn't " "thumbnailed/encoded.", file_id, mime_category) src_file['status'] = 'complete' else: log.debug('process_file(%s): marking file status as "processing"', file_id) src_file['status'] = 'processing' update_file_doc(file_id, status='processing') try: processor(bucket, file_id, local_file, src_file) except Exception: log.warning( 'process_file(%s): error when processing file, ' 'resetting status to ' '"queued_for_processing"', file_id, exc_info=True) update_file_doc(file_id, status='queued_for_processing') return # Update the original file with additional info, e.g. image resolution r, _, _, status = current_app.put_internal('files', src_file, _id=file_id) if status not in (200, 201): log.warning( 'process_file(%s): status %i when saving processed file ' 'info to MongoDB: %s', file_id, status, r)
def create_home_project(user_id, write_access): """Creates a home project for the given user. :param user_id: the user ID of the owner :param write_access: whether the user has full write access to the home project. :type write_access: bool :returns: the project :rtype: dict """ log.info('Creating home project for user %s', user_id) overrides = { 'category': 'home', 'url': 'home', 'summary': HOME_PROJECT_SUMMARY, 'description': HOME_PROJECT_DESCRIPTION } # Maybe the user has a deleted home project. proj_coll = current_app.data.driver.db['projects'] deleted_proj = proj_coll.find_one({ 'user': user_id, 'category': 'home', '_deleted': True }) if deleted_proj: log.info('User %s has a deleted project %s, restoring', user_id, deleted_proj['_id']) project = deleted_proj else: log.debug('User %s does not have a deleted project', user_id) project = proj_utils.create_new_project(project_name='Home', user_id=ObjectId(user_id), overrides=overrides) # Re-validate the authentication token, so that the put_internal call sees the # new group created for the project. authentication.validate_token(force=True) # There are a few things in the on_insert_projects hook we need to adjust. # Ensure that the project is private, even for admins. project['permissions']['world'] = [] # Set up the correct node types. No need to set permissions for them, # as the inherited project permissions are fine. from pillar.api.node_types.group import node_type_group from pillar.api.node_types.asset import node_type_asset # from pillar.api.node_types.text import node_type_text from pillar.api.node_types.comment import node_type_comment # For non-subscribers: take away write access from the admin group, # and grant it to certain node types. project['permissions']['groups'][0]['methods'] = home_project_permissions( write_access) # Everybody should be able to comment on anything in this project. # This allows people to comment on shared images and see comments. node_type_comment = assign_permissions(node_type_comment, subscriber_methods=['GET', 'POST'], world_methods=['GET']) project['node_types'] = [ node_type_group, node_type_asset, # node_type_text, node_type_comment, ] result, _, _, status = current_app.put_internal( 'projects', utils.remove_private_keys(project), _id=project['_id']) if status != 200: log.error('Unable to update home project %s for user %s: %s', project['_id'], user_id, result) raise wz_exceptions.InternalServerError( 'Unable to update home project') project.update(result) # Create the Blender Sync node, with explicit write permissions on the node itself. create_blender_sync_node(project['_id'], project['permissions']['groups'][0]['group'], user_id) return project
def zencoder_notifications(): """ See: https://app.zencoder.com/docs/guides/getting-started/notifications#api_version_2 """ if current_app.config['ENCODING_BACKEND'] != 'zencoder': log.warning('Received notification from Zencoder but app not configured for Zencoder.') return abort(403) if not current_app.config['DEBUG']: # If we are in production, look for the Zencoder header secret try: notification_secret_request = request.headers[ 'X-Zencoder-Notification-Secret'] except KeyError: log.warning('Received Zencoder notification without secret.') return abort(401) # If the header is found, check it agains the one in the config notification_secret = current_app.config['ZENCODER_NOTIFICATIONS_SECRET'] if notification_secret_request != notification_secret: log.warning('Received Zencoder notification with incorrect secret.') return abort(401) # Cast request data into a dict data = request.get_json() if log.isEnabledFor(logging.DEBUG): from pprint import pformat log.debug('Zencoder job JSON: %s', pformat(data)) files_collection = current_app.data.driver.db['files'] # Find the file object based on processing backend and job_id zencoder_job_id = data['job']['id'] lookup = {'processing.backend': 'zencoder', 'processing.job_id': str(zencoder_job_id)} file_doc = files_collection.find_one(lookup) if not file_doc: log.warning('Unknown Zencoder job id %r', zencoder_job_id) # Return 200 OK when debugging, or Zencoder will keep trying and trying and trying... # which is what we want in production. return "Not found, but that's okay.", 200 if current_app.config['DEBUG'] else 404 file_id = ObjectId(file_doc['_id']) # Remove internal keys (so that we can run put internal) file_doc = utils.remove_private_keys(file_doc) # Update processing status job_state = data['job']['state'] file_doc['processing']['status'] = job_state if job_state == 'failed': log.warning('Zencoder job %i for file %s failed.', zencoder_job_id, file_id) # Log what Zencoder told us went wrong. for output in data['outputs']: if not any('error' in key for key in output): continue log.warning('Errors for output %s:', output['url']) for key in output: if 'error' in key: log.info(' %s: %s', key, output[key]) file_doc['status'] = 'failed' current_app.put_internal('files', file_doc, _id=file_id) return "You failed, but that's okay.", 200 log.info('Zencoder job %s for file %s completed with status %s.', zencoder_job_id, file_id, job_state) # For every variation encoded, try to update the file object root, _ = os.path.splitext(file_doc['file_path']) for output in data['outputs']: video_format = output['format'] # Change the zencoder 'mpeg4' format to 'mp4' used internally video_format = 'mp4' if video_format == 'mpeg4' else video_format # Find a variation matching format and resolution variation = next((v for v in file_doc['variations'] if v['format'] == format and v['width'] == output['width']), None) # Fall back to a variation matching just the format if variation is None: variation = next((v for v in file_doc['variations'] if v['format'] == video_format), None) if variation is None: log.warning('Unable to find variation for video format %s for file %s', video_format, file_id) continue # Rename the file to include the now-known size descriptor. size = size_descriptor(output['width'], output['height']) new_fname = '{}-{}.{}'.format(root, size, video_format) # Rename on Google Cloud Storage try: rename_on_gcs(file_doc['project'], '_/' + variation['file_path'], '_/' + new_fname) except Exception: log.warning('Unable to rename GCS blob %r to %r. Keeping old name.', variation['file_path'], new_fname, exc_info=True) else: variation['file_path'] = new_fname # TODO: calculate md5 on the storage variation.update({ 'height': output['height'], 'width': output['width'], 'length': output['file_size_in_bytes'], 'duration': data['input']['duration_in_ms'] / 1000, 'md5': output['md5_checksum'] or '', # they don't do MD5 for GCS... 'size': size, }) file_doc['status'] = 'complete' # Force an update of the links on the next load of the file. file_doc['link_expires'] = datetime.datetime.now(tz=tz_util.utc) - datetime.timedelta(days=1) current_app.put_internal('files', file_doc, _id=file_id) return '', 204
def zencoder_notifications(): """ See: https://app.zencoder.com/docs/guides/getting-started/notifications#api_version_2 """ if current_app.config['ENCODING_BACKEND'] != 'zencoder': log.warning('Received notification from Zencoder but app not configured for Zencoder.') return abort(403) if not current_app.config['DEBUG']: # If we are in production, look for the Zencoder header secret try: notification_secret_request = request.headers[ 'X-Zencoder-Notification-Secret'] except KeyError: log.warning('Received Zencoder notification without secret.') return abort(401) # If the header is found, check it agains the one in the config notification_secret = current_app.config['ZENCODER_NOTIFICATIONS_SECRET'] if notification_secret_request != notification_secret: log.warning('Received Zencoder notification with incorrect secret.') return abort(401) # Cast request data into a dict data = request.get_json() if log.isEnabledFor(logging.DEBUG): from pprint import pformat log.debug('Zencoder job JSON: %s', pformat(data)) files_collection = current_app.data.driver.db['files'] # Find the file object based on processing backend and job_id zencoder_job_id = data['job']['id'] lookup = {'processing.backend': 'zencoder', 'processing.job_id': str(zencoder_job_id)} file_doc = files_collection.find_one(lookup) if not file_doc: log.warning('Unknown Zencoder job id %r', zencoder_job_id) # Return 200 OK when debugging, or Zencoder will keep trying and trying and trying... # which is what we want in production. return "Not found, but that's okay.", 200 if current_app.config['DEBUG'] else 404 file_id = ObjectId(file_doc['_id']) # Remove internal keys (so that we can run put internal) file_doc = utils.remove_private_keys(file_doc) # Update processing status job_state = data['job']['state'] file_doc['processing']['status'] = job_state if job_state == 'failed': log.warning('Zencoder job %s for file %s failed: %s', zencoder_job_id, file_id, json.dumps(data, sort_keys=True, indent=4)) file_doc['status'] = 'failed' current_app.put_internal('files', file_doc, _id=file_id) # This is 'okay' because we handled the Zencoder notification properly. return "You failed, but that's okay.", 200 log.info('Zencoder job %s for file %s completed with status %s.', zencoder_job_id, file_id, job_state) # For every variation encoded, try to update the file object storage_name, _ = os.path.splitext(file_doc['file_path']) nice_name, _ = os.path.splitext(file_doc['filename']) bucket_class = Bucket.for_backend(file_doc['backend']) bucket = bucket_class(str(file_doc['project'])) for output in data['outputs']: video_format = output['format'] # Change the zencoder 'mpeg4' format to 'mp4' used internally video_format = 'mp4' if video_format == 'mpeg4' else video_format # Find a variation matching format and resolution variation = next((v for v in file_doc['variations'] if v['format'] == format and v['width'] == output['width']), None) # Fall back to a variation matching just the format if variation is None: variation = next((v for v in file_doc['variations'] if v['format'] == video_format), None) if variation is None: log.warning('Unable to find variation for video format %s for file %s', video_format, file_id) continue # Rename the file to include the now-known size descriptor. size = size_descriptor(output['width'], output['height']) new_fname = f'{storage_name}-{size}.{video_format}' # Rename the file on the storage. blob = bucket.blob(variation['file_path']) try: new_blob = bucket.rename_blob(blob, new_fname) new_blob.update_filename(f'{nice_name}-{size}.{video_format}') except Exception: log.warning('Unable to rename blob %r to %r. Keeping old name.', blob, new_fname, exc_info=True) else: variation['file_path'] = new_fname # TODO: calculate md5 on the storage variation.update({ 'height': output['height'], 'width': output['width'], 'length': output['file_size_in_bytes'], 'duration': data['input']['duration_in_ms'] / 1000, 'md5': output['md5_checksum'] or '', # they don't do MD5 for GCS... 'size': size, }) file_doc['status'] = 'complete' # Force an update of the links on the next load of the file. file_doc['link_expires'] = utils.utcnow() - datetime.timedelta(days=1) r, _, _, status = current_app.put_internal('files', file_doc, _id=file_id) if status != 200: log.error('unable to save file %s after Zencoder notification: %s', file_id, r) return json.dumps(r), 500 return '', 204
def upsert_user(db_user): """Inserts/updates the user in MongoDB. Retries a few times when there are uniqueness issues in the username. :returns: the user's database ID and the status of the PUT/POST. The status is 201 on insert, and 200 on update. :type: (ObjectId, int) """ if 'subscriber' in db_user.get('groups', []): log.error('Non-ObjectID string found in user.groups: %s', db_user) raise wz_exceptions.InternalServerError( 'Non-ObjectID string found in user.groups: %s' % db_user) if not db_user['full_name']: # Blender ID doesn't need a full name, but we do. db_user['full_name'] = db_user['username'] r = {} for retry in range(5): if '_id' in db_user: # Update the existing user attempted_eve_method = 'PUT' db_id = db_user['_id'] r, _, _, status = current_app.put_internal( 'users', remove_private_keys(db_user), _id=db_id) if status == 422: log.error( 'Status %i trying to PUT user %s with values %s, should not happen! %s', status, db_id, remove_private_keys(db_user), r) else: # Create a new user, retry for non-unique usernames. attempted_eve_method = 'POST' r, _, _, status = current_app.post_internal('users', db_user) if status not in {200, 201}: log.error('Status %i trying to create user with values %s: %s', status, db_user, r) raise wz_exceptions.InternalServerError() db_id = r['_id'] db_user.update(r) # update with database/eve-generated fields. if status == 422: # Probably non-unique username, so retry a few times with different usernames. log.info('Error creating new user: %s', r) username_issue = r.get('_issues', {}).get('username', '') if 'not unique' in username_issue: # Retry db_user['username'] = make_unique_username(db_user['email']) continue # Saving was successful, or at least didn't break on a non-unique username. break else: log.error('Unable to create new user %s: %s', db_user, r) raise wz_exceptions.InternalServerError() if status not in (200, 201): log.error('internal response from %s to Eve: %r %r', attempted_eve_method, status, r) raise wz_exceptions.InternalServerError() return db_id, status