def api_update_rna_overrides(self, job_id: ObjectId, rna_overrides: typing.List[str]): """API-level call to create or update an RNA override task of a Blender Render job.""" new_etag = random_etag() now = utcnow() jobs_coll = current_flamenco.db('jobs') # Check that the job exists and is a Blender-related job. job = jobs_coll.find_one({'_id': job_id}) if not job: self._log.warning('Unable to update RNA overrides of non-existing job %s', job_id) return None compiler = job_compilers.construct_job_compiler(job) if not isinstance(compiler, blender_render.AbstractBlenderJobCompiler): self._log.warning('Job compiler %r is not an AbstractBlenderJobCompiler, unable ' 'to update RNA overrides for job %s of type %r', type(compiler), job_id, job['job_type']) return None # Update the job itself before updating its tasks. Ideally this would happen in the # same transaction. # TODO(Sybren): put into one transaction when we upgrade to MongoDB 4+. job['settings']['rna_overrides'] = rna_overrides result = jobs_coll.update_one({'_id': job_id}, {'$set': { 'settings.rna_overrides': rna_overrides, '_updated': now, '_etag': new_etag, }}) if result.matched_count != 1: self._log.warning('Matched %d jobs while setting job %s RNA overrides', result.matched_count, job_id) compiler.update_rna_overrides_task(job)
def _insert_rna_overrides_task(self, job: dict, parent_task_selector: dict) -> bson.ObjectId: # Find the task that is supposed to be the parent of the new task. tasks_coll = current_flamenco.db('tasks') if parent_task_selector: parent_task = tasks_coll.find_one({'job': job['_id'], **parent_task_selector}, projection={'_id': True}) if not parent_task: raise ValueError('unable to find move-out-of-way task, cannot update this job') parents_kwargs = {'parents': [parent_task['_id']]} else: parents_kwargs = {} # Construct the new task. cmd = rna_overrides_command(job) task_id = self._create_task(job, [cmd], RNA_OVERRIDES_TASK_NAME, 'file-management', priority=80, status='queued', **parents_kwargs) self._log.info('Inserted RNA Overrides task %s into job %s', task_id, job['_id']) # Update existing render tasks to have the new task as parent. new_etag = random_etag() now = utcnow() result = tasks_coll.update_many({ 'job': job['_id'], 'task_type': 'blender-render', **parents_kwargs, }, {'$set': { '_etag': new_etag, '_updated': now, 'parents': [task_id], }}) self._log.debug('Updated %d task parent pointers to %s', result.modified_count, task_id) return task_id
def api_set_job_priority(self, job_id: ObjectId, new_priority: int): """API-level call to updates the job priority.""" assert isinstance(new_priority, int) self._log.debug('Setting job %s priority to %r', job_id, new_priority) jobs_coll = current_flamenco.db('jobs') curr_job = jobs_coll.find_one({'_id': job_id}, projection={'priority': 1}) old_priority = curr_job['priority'] if old_priority == new_priority: self._log.debug('Job %s is already at priority %r', job_id, old_priority) return new_etag = random_etag() now = utcnow() jobs_coll = current_flamenco.db('jobs') result = jobs_coll.update_one({'_id': job_id}, {'$set': {'priority': new_priority, '_updated': now, '_etag': new_etag, }}) if result.matched_count != 1: self._log.warning('Matched %d jobs while setting job %s to priority %r', result.matched_count, job_id, new_priority) tasks_coll = current_flamenco.db('tasks') result = tasks_coll.update_many({'job': job_id}, {'$set': {'job_priority': new_priority, '_updated': now, '_etag': new_etag, }}) self._log.debug('Matched %d tasks while setting job %s to priority %r', result.matched_count, job_id, new_priority)
def update_rna_overrides_task(self, job: dict): """Update or create an RNA Overrides task of an existing job.""" tasks_coll = current_flamenco.db('tasks') task = tasks_coll.find_one( { 'job': job['_id'], 'name': RNA_OVERRIDES_TASK_NAME }, projection={'_id': True}) if not task: self.insert_rna_overrides_task(job) return cmd = rna_overrides_command(job) new_etag = random_etag() now = utcnow() result = tasks_coll.update_one( task, { '$set': { '_etag': new_etag, '_updated': now, 'status': 'queued', 'commands': [cmd.to_dict()], } }) self._log.info('Modified %d RNA override task (%s) of job %s', result.modified_count, task['_id'], job['_id'])
def merge_project(pid_from: ObjectId, pid_to: ObjectId): """Move nodes and files from one project to another. Note that this may invalidate the nodes, as their node type definition may differ between projects. """ log.info('Moving project contents from %s to %s', pid_from, pid_to) assert isinstance(pid_from, ObjectId) assert isinstance(pid_to, ObjectId) files_coll = current_app.db('files') nodes_coll = current_app.db('nodes') # Move the files first. Since this requires API calls to an external # service, this is more likely to go wrong than moving the nodes. to_move = files_coll.find({'project': pid_from}, projection={'_id': 1}) log.info('Moving %d files to project %s', to_move.count(), pid_to) for file_doc in to_move: fid = file_doc['_id'] log.debug('moving file %s to project %s', fid, pid_to) move_to_bucket(fid, pid_to) # Mass-move the nodes. etag = random_etag() result = nodes_coll.update_many( {'project': pid_from}, {'$set': {'project': pid_to, '_etag': etag, '_updated': utcnow(), }} ) log.info('Moved %d nodes to project %s', result.modified_count, pid_to)
def _create_image_node(self): file_id, _ = self.ensure_file_exists( file_overrides={ '_id': ObjectId(), 'variations': [ { 'format': 'jpeg' }, ], }) node = { 'name': 'Just a node name', 'project': self.pid, 'description': '', 'node_type': 'asset', '_etag': random_etag(), } props = { 'status': 'published', 'file': file_id, 'content_type': 'image', 'order': 0 } return self.create_node({'properties': props, **node})
def _create_video_node(self, file_duration=None, node_duration=None, include_file=True): file_id, _ = self.ensure_file_exists( file_overrides={ '_id': ObjectId(), 'content_type': 'video/mp4', 'variations': [ { 'format': 'mp4', 'duration': file_duration }, ], }) node = { 'name': 'Just a node name', 'project': self.pid, 'description': '', 'node_type': 'asset', '_etag': random_etag(), } props = {'status': 'published', 'content_type': 'video', 'order': 0} if node_duration is not None: props['duration_seconds'] = node_duration if include_file: props['file'] = file_id return self.create_node({'properties': props, **node})
def _task_log_request(self, manager_id: bson.ObjectId, operation: dict): managers_coll = current_flamenco.db('managers') managers_coll.update_one({'_id': manager_id}, { **operation, '$set': { '_updated': utcnow(), '_etag': random_etag(), }, })
def _insert_rna_overrides_task( self, job: dict, parent_task_selector: dict) -> bson.ObjectId: # Find the task that is supposed to be the parent of the new task. tasks_coll = current_flamenco.db('tasks') if parent_task_selector: parent_task = tasks_coll.find_one( { 'job': job['_id'], **parent_task_selector }, projection={'_id': True}) if not parent_task: raise ValueError( 'unable to find move-out-of-way task, cannot update this job' ) parents_kwargs = {'parents': [parent_task['_id']]} else: parents_kwargs = {} # Construct the new task. cmd = rna_overrides_command(job) task_id = self._create_task(job, [cmd], RNA_OVERRIDES_TASK_NAME, 'file-management', priority=80, status='queued', **parents_kwargs) self._log.info('Inserted RNA Overrides task %s into job %s', task_id, job['_id']) # Update existing render tasks to have the new task as parent. new_etag = random_etag() now = utcnow() result = tasks_coll.update_many( { 'job': job['_id'], 'task_type': 'blender-render', **parents_kwargs, }, { '$set': { '_etag': new_etag, '_updated': now, 'parents': [task_id], } }) self._log.debug('Updated %d task parent pointers to %s', result.modified_count, task_id) return task_id
def api_update_rna_overrides(self, job_id: ObjectId, rna_overrides: typing.List[str]): """API-level call to create or update an RNA override task of a Blender Render job.""" new_etag = random_etag() now = utcnow() jobs_coll = current_flamenco.db('jobs') # Check that the job exists and is a Blender-related job. job = jobs_coll.find_one({'_id': job_id}) if not job: self._log.warning( 'Unable to update RNA overrides of non-existing job %s', job_id) return None compiler = job_compilers.construct_job_compiler(job) if not isinstance(compiler, blender_render.AbstractBlenderJobCompiler): self._log.warning( 'Job compiler %r is not an AbstractBlenderJobCompiler, unable ' 'to update RNA overrides for job %s of type %r', type(compiler), job_id, job['job_type']) return None # Update the job itself before updating its tasks. Ideally this would happen in the # same transaction. # TODO(Sybren): put into one transaction when we upgrade to MongoDB 4+. job['settings']['rna_overrides'] = rna_overrides result = jobs_coll.update_one({'_id': job_id}, { '$set': { 'settings.rna_overrides': rna_overrides, '_updated': now, '_etag': new_etag, } }) if result.matched_count != 1: self._log.warning( 'Matched %d jobs while setting job %s RNA overrides', result.matched_count, job_id) compiler.update_rna_overrides_task(job)
def update_rna_overrides_task(self, job: dict): """Update or create an RNA Overrides task of an existing job.""" tasks_coll = current_flamenco.db('tasks') task = tasks_coll.find_one({'job': job['_id'], 'name': RNA_OVERRIDES_TASK_NAME}, projection={'_id': True}) if not task: self.insert_rna_overrides_task(job) return cmd = rna_overrides_command(job) new_etag = random_etag() now = utcnow() result = tasks_coll.update_one(task, {'$set': { '_etag': new_etag, '_updated': now, 'status': 'queued', 'commands': [cmd.to_dict()], }}) self._log.info('Modified %d RNA override task (%s) of job %s', result.modified_count, task['_id'], job['_id'])
def api_set_job_priority(self, job_id: ObjectId, new_priority: int): """API-level call to updates the job priority.""" assert isinstance(new_priority, int) self._log.debug('Setting job %s priority to %r', job_id, new_priority) jobs_coll = current_flamenco.db('jobs') curr_job = jobs_coll.find_one({'_id': job_id}, projection={'priority': 1}) old_priority = curr_job['priority'] if old_priority == new_priority: self._log.debug('Job %s is already at priority %r', job_id, old_priority) return new_etag = random_etag() now = utcnow() jobs_coll = current_flamenco.db('jobs') result = jobs_coll.update_one({'_id': job_id}, { '$set': { 'priority': new_priority, '_updated': now, '_etag': new_etag, } }) if result.matched_count != 1: self._log.warning( 'Matched %d jobs while setting job %s to priority %r', result.matched_count, job_id, new_priority) tasks_coll = current_flamenco.db('tasks') result = tasks_coll.update_many({'job': job_id}, { '$set': { 'job_priority': new_priority, '_updated': now, '_etag': new_etag, } }) self._log.debug('Matched %d tasks while setting job %s to priority %r', result.matched_count, job_id, new_priority)
def remove_project_references(node): project_id = node.get('project') if not project_id: return node_id = node['_id'] log.info('Removing references to node %s from project %s', node_id, project_id) projects_col = current_app.db('projects') project = projects_col.find_one({'_id': project_id}) updates = collections.defaultdict(dict) if project.get('header_node') == node_id: updates['$unset']['header_node'] = node_id project_reference_lists = ('nodes_blog', 'nodes_featured', 'nodes_latest') for list_name in project_reference_lists: references = project.get(list_name) if not references: continue try: references.remove(node_id) except ValueError: continue updates['$set'][list_name] = references if not updates: return updates['$set']['_etag'] = random_etag() result = projects_col.update_one({'_id': project_id}, updates) if result.modified_count != 1: log.warning( 'Removing references to node %s from project %s resulted in %d modified documents (expected 1)', node_id, project_id, result.modified_count)
def handle_task_update_batch(manager_id, task_updates): """Performs task updates. Task status changes are generally always accepted. The only exception is when the task ID is contained in 'tasks_to_cancel'; in that case only a transition to either 'canceled', 'completed' or 'failed' is accepted. :returns: tuple (total nr of modified tasks, handled update IDs) """ if not task_updates: return 0, [] import dateutil.parser from pillar.api.utils import str2id from flamenco import current_flamenco, eve_settings log.debug('Received %i task updates from manager %s', len(task_updates), manager_id) tasks_coll = current_flamenco.db('tasks') logs_coll = current_flamenco.db('task_logs') valid_statuses = set(eve_settings.tasks_schema['status']['allowed']) handled_update_ids = [] total_modif_count = 0 for task_update in task_updates: # Check that this task actually belongs to this manager, before we accept any updates. update_id = str2id(task_update['_id']) task_id = str2id(task_update['task_id']) task_info = tasks_coll.find_one({'_id': task_id}, projection={'manager': 1, 'status': 1, 'job': 1}) # For now, we just ignore updates to non-existing tasks. Someone might have just deleted # one, for example. This is not a reason to reject the entire batch. if task_info is None: log.warning('Manager %s sent update for non-existing task %s; accepting but ignoring', manager_id, task_id) handled_update_ids.append(update_id) continue if task_info['manager'] != manager_id: log.warning('Manager %s sent update for task %s which belongs to other manager %s', manager_id, task_id, task_info['manager']) continue if task_update.get('received_on_manager'): received_on_manager = dateutil.parser.parse(task_update['received_on_manager']) else: # Fake a 'received on manager' field; it really should have been in the JSON payload. received_on_manager = utcnow() # Store the log for this task, allowing for duplicate log reports. # # NOTE: is deprecated and will be removed in a future version of Flamenco; # only periodically send the last few lines of logging in 'log_tail' and # store the entire log on the Manager itself. task_log = task_update.get('log') if task_log: log_doc = { '_id': update_id, 'task': task_id, 'received_on_manager': received_on_manager, 'log': task_log } logs_coll.replace_one({'_id': update_id}, log_doc, upsert=True) # Modify the task, and append the log to the logs collection. updates = { 'task_progress_percentage': task_update.get('task_progress_percentage', 0), 'current_command_index': task_update.get('current_command_index', 0), 'command_progress_percentage': task_update.get('command_progress_percentage', 0), '_updated': received_on_manager, '_etag': random_etag(), } new_status = determine_new_task_status(manager_id, task_id, task_info, task_update.get('task_status'), valid_statuses) if new_status: updates['status'] = new_status new_activity = task_update.get('activity') if new_activity: updates['activity'] = new_activity worker = task_update.get('worker') if worker: updates['worker'] = worker # Store the last lines of logging on the task itself. task_log_tail: str = task_update.get('log_tail') if not task_log_tail and task_log: task_log_tail = '\n'.join(task_log.split('\n')[-LOG_TAIL_LINES:]) if task_log_tail: updates['log'] = task_log_tail result = tasks_coll.update_one({'_id': task_id}, {'$set': updates}) total_modif_count += result.modified_count handled_update_ids.append(update_id) # Update the task's job after updating the task itself. if new_status: current_flamenco.job_manager.update_job_after_task_status_change( task_info['job'], task_id, new_status) return total_modif_count, handled_update_ids
def patch_project(project_id: str): """Undelete a project. This is done via a custom PATCH due to the lack of transactions of MongoDB; we cannot undelete both project-referenced files and file-referenced projects in one atomic operation. """ # Parse the request pid = str2id(project_id) patch = request.get_json() if not patch: raise wz_exceptions.BadRequest('Expected JSON body') log.debug('User %s wants to PATCH project %s: %s', current_user, pid, patch) # 'undelete' is the only operation we support now, so no fancy handler registration. op = patch.get('op', '') if op != 'undelete': log.warning('User %s sent unsupported PATCH op %r to project %s: %s', current_user, op, pid, patch) raise wz_exceptions.BadRequest(f'unsupported operation {op!r}') # Get the project to find the user's permissions. proj_coll = current_app.db('projects') proj = proj_coll.find_one({'_id': pid}) if not proj: raise wz_exceptions.NotFound(f'project {pid} not found') allowed = authorization.compute_allowed_methods('projects', proj) if 'PUT' not in allowed: log.warning( 'User %s tried to undelete project %s but only has permissions %r', current_user, pid, allowed) raise wz_exceptions.Forbidden(f'no PUT access to project {pid}') if not proj.get('_deleted', False): raise wz_exceptions.BadRequest( f'project {pid} was not deleted, unable to undelete') # Undelete the files. We cannot do this via Eve, as it doesn't support # PATCHing collections, so direct MongoDB modification is used to set # _deleted=False and provide new _etag and _updated values. new_etag = random_etag() log.debug('undeleting files before undeleting project %s', pid) files_coll = current_app.db('files') update_result = files_coll.update_many( {'project': pid}, {'$set': { '_deleted': False, '_etag': new_etag, '_updated': utcnow() }}) log.info('undeleted %d of %d file documents of project %s', update_result.modified_count, update_result.matched_count, pid) log.info('undeleting project %s on behalf of user %s', pid, current_user) update_result = proj_coll.update_one({'_id': pid}, {'$set': { '_deleted': False }}) log.info('undeleted %d project document %s', update_result.modified_count, pid) resp = flask.Response('', status=204) resp.location = flask.url_for('projects.view', project_url=proj['url']) return resp
def handle_task_update_batch(manager_id, task_updates): """Performs task updates. Task status changes are generally always accepted. The only exception is when the task ID is contained in 'tasks_to_cancel'; in that case only a transition to either 'canceled', 'completed' or 'failed' is accepted. :returns: tuple (total nr of modified tasks, handled update IDs) """ if not task_updates: return 0, [] import dateutil.parser from pillar.api.utils import str2id from flamenco import current_flamenco, eve_settings log.debug('Received %i task updates from manager %s', len(task_updates), manager_id) tasks_coll = current_flamenco.db('tasks') logs_coll = current_flamenco.db('task_logs') valid_statuses = set(eve_settings.tasks_schema['status']['allowed']) handled_update_ids = [] total_modif_count = 0 for task_update in task_updates: # Check that this task actually belongs to this manager, before we accept any updates. update_id = str2id(task_update['_id']) task_id = str2id(task_update['task_id']) task_info = tasks_coll.find_one({'_id': task_id}, projection={'manager': 1, 'status': 1, 'job': 1}) # For now, we just ignore updates to non-existing tasks. Someone might have just deleted # one, for example. This is not a reason to reject the entire batch. if task_info is None: log.warning('Manager %s sent update for non-existing task %s; accepting but ignoring', manager_id, task_id) handled_update_ids.append(update_id) continue if task_info['manager'] != manager_id: log.warning('Manager %s sent update for task %s which belongs to other manager %s', manager_id, task_id, task_info['manager']) continue if task_update.get('received_on_manager'): received_on_manager = dateutil.parser.parse(task_update['received_on_manager']) else: # Fake a 'received on manager' field; it really should have been in the JSON payload. received_on_manager = utcnow() # Store the log for this task, allowing for duplicate log reports. # # NOTE: is deprecated and will be removed in a future version of Flamenco; # only periodically send the last few lines of logging in 'log_tail' and # store the entire log on the Manager itself. task_log = task_update.get('log') if task_log: log_doc = { '_id': update_id, 'task': task_id, 'received_on_manager': received_on_manager, 'log': task_log } logs_coll.replace_one({'_id': update_id}, log_doc, upsert=True) # Modify the task, and append the log to the logs collection. updates = { 'task_progress_percentage': task_update.get('task_progress_percentage', 0), 'current_command_index': task_update.get('current_command_index', 0), 'command_progress_percentage': task_update.get('command_progress_percentage', 0), '_updated': received_on_manager, '_etag': random_etag(), } new_status = determine_new_task_status(manager_id, task_id, task_info, task_update.get('task_status'), valid_statuses) if new_status: updates['status'] = new_status new_activity = task_update.get('activity') if new_activity: updates['activity'] = new_activity worker = task_update.get('worker') if worker: updates['worker'] = worker metrics_timing = task_update.get('metrics', {}).get('timing') if metrics_timing: updates['metrics.timing'] = metrics_timing fbw = task_update.get('failed_by_workers') if fbw is not None: updates['failed_by_workers'] = fbw # Store the last lines of logging on the task itself. task_log_tail: str = task_update.get('log_tail') if not task_log_tail and task_log: task_log_tail = '\n'.join(task_log.split('\n')[-LOG_TAIL_LINES:]) if task_log_tail: updates['log'] = task_log_tail result = tasks_coll.update_one({'_id': task_id}, {'$set': updates}) total_modif_count += result.modified_count handled_update_ids.append(update_id) # Update the task's job after updating the task itself. if new_status: current_flamenco.job_manager.update_job_after_task_status_change( task_info['job'], task_id, new_status) return total_modif_count, handled_update_ids