def api_update_rna_overrides(self, job_id: ObjectId, rna_overrides: typing.List[str]):
"""API-level call to create or update an RNA override task of a Blender Render job."""
new_etag = random_etag()
now = utcnow()
jobs_coll = current_flamenco.db('jobs')
# Check that the job exists and is a Blender-related job.
job = jobs_coll.find_one({'_id': job_id})
if not job:
self._log.warning('Unable to update RNA overrides of non-existing job %s', job_id)
return None
compiler = job_compilers.construct_job_compiler(job)
if not isinstance(compiler, blender_render.AbstractBlenderJobCompiler):
self._log.warning('Job compiler %r is not an AbstractBlenderJobCompiler, unable '
'to update RNA overrides for job %s of type %r',
type(compiler), job_id, job['job_type'])
return None
# Update the job itself before updating its tasks. Ideally this would happen in the
# same transaction.
# TODO(Sybren): put into one transaction when we upgrade to MongoDB 4+.
job['settings']['rna_overrides'] = rna_overrides
result = jobs_coll.update_one({'_id': job_id}, {'$set': {
'settings.rna_overrides': rna_overrides,
'_updated': now,
'_etag': new_etag,
}})
if result.matched_count != 1:
self._log.warning('Matched %d jobs while setting job %s RNA overrides',
result.matched_count, job_id)
compiler.update_rna_overrides_task(job)
def _insert_rna_overrides_task(self, job: dict,
parent_task_selector: dict) -> bson.ObjectId:
# Find the task that is supposed to be the parent of the new task.
tasks_coll = current_flamenco.db('tasks')
if parent_task_selector:
parent_task = tasks_coll.find_one({'job': job['_id'], **parent_task_selector},
projection={'_id': True})
if not parent_task:
raise ValueError('unable to find move-out-of-way task, cannot update this job')
parents_kwargs = {'parents': [parent_task['_id']]}
else:
parents_kwargs = {}
# Construct the new task.
cmd = rna_overrides_command(job)
task_id = self._create_task(job, [cmd], RNA_OVERRIDES_TASK_NAME,
'file-management', priority=80,
status='queued', **parents_kwargs)
self._log.info('Inserted RNA Overrides task %s into job %s', task_id, job['_id'])
# Update existing render tasks to have the new task as parent.
new_etag = random_etag()
now = utcnow()
result = tasks_coll.update_many({
'job': job['_id'],
'task_type': 'blender-render',
**parents_kwargs,
}, {'$set': {
'_etag': new_etag,
'_updated': now,
'parents': [task_id],
}})
self._log.debug('Updated %d task parent pointers to %s', result.modified_count, task_id)
return task_id
def api_set_job_priority(self, job_id: ObjectId, new_priority: int):
"""API-level call to updates the job priority."""
assert isinstance(new_priority, int)
self._log.debug('Setting job %s priority to %r', job_id, new_priority)
jobs_coll = current_flamenco.db('jobs')
curr_job = jobs_coll.find_one({'_id': job_id}, projection={'priority': 1})
old_priority = curr_job['priority']
if old_priority == new_priority:
self._log.debug('Job %s is already at priority %r', job_id, old_priority)
return
new_etag = random_etag()
now = utcnow()
jobs_coll = current_flamenco.db('jobs')
result = jobs_coll.update_one({'_id': job_id},
{'$set': {'priority': new_priority,
'_updated': now,
'_etag': new_etag,
}})
if result.matched_count != 1:
self._log.warning('Matched %d jobs while setting job %s to priority %r',
result.matched_count, job_id, new_priority)
tasks_coll = current_flamenco.db('tasks')
result = tasks_coll.update_many({'job': job_id},
{'$set': {'job_priority': new_priority,
'_updated': now,
'_etag': new_etag,
}})
self._log.debug('Matched %d tasks while setting job %s to priority %r',
result.matched_count, job_id, new_priority)
def update_rna_overrides_task(self, job: dict):
"""Update or create an RNA Overrides task of an existing job."""
tasks_coll = current_flamenco.db('tasks')
task = tasks_coll.find_one(
{
'job': job['_id'],
'name': RNA_OVERRIDES_TASK_NAME
},
projection={'_id': True})
if not task:
self.insert_rna_overrides_task(job)
return
cmd = rna_overrides_command(job)
new_etag = random_etag()
now = utcnow()
result = tasks_coll.update_one(
task, {
'$set': {
'_etag': new_etag,
'_updated': now,
'status': 'queued',
'commands': [cmd.to_dict()],
}
})
self._log.info('Modified %d RNA override task (%s) of job %s',
result.modified_count, task['_id'], job['_id'])
def merge_project(pid_from: ObjectId, pid_to: ObjectId):
"""Move nodes and files from one project to another.
Note that this may invalidate the nodes, as their node type definition
may differ between projects.
"""
log.info('Moving project contents from %s to %s', pid_from, pid_to)
assert isinstance(pid_from, ObjectId)
assert isinstance(pid_to, ObjectId)
files_coll = current_app.db('files')
nodes_coll = current_app.db('nodes')
# Move the files first. Since this requires API calls to an external
# service, this is more likely to go wrong than moving the nodes.
to_move = files_coll.find({'project': pid_from}, projection={'_id': 1})
log.info('Moving %d files to project %s', to_move.count(), pid_to)
for file_doc in to_move:
fid = file_doc['_id']
log.debug('moving file %s to project %s', fid, pid_to)
move_to_bucket(fid, pid_to)
# Mass-move the nodes.
etag = random_etag()
result = nodes_coll.update_many(
{'project': pid_from},
{'$set': {'project': pid_to,
'_etag': etag,
'_updated': utcnow(),
}}
)
log.info('Moved %d nodes to project %s', result.modified_count, pid_to)
def _create_image_node(self):
file_id, _ = self.ensure_file_exists(
file_overrides={
'_id': ObjectId(),
'variations': [
{
'format': 'jpeg'
},
],
})
node = {
'name': 'Just a node name',
'project': self.pid,
'description': '',
'node_type': 'asset',
'_etag': random_etag(),
}
props = {
'status': 'published',
'file': file_id,
'content_type': 'image',
'order': 0
}
return self.create_node({'properties': props, **node})
def _create_video_node(self,
file_duration=None,
node_duration=None,
include_file=True):
file_id, _ = self.ensure_file_exists(
file_overrides={
'_id': ObjectId(),
'content_type': 'video/mp4',
'variations': [
{
'format': 'mp4',
'duration': file_duration
},
],
})
node = {
'name': 'Just a node name',
'project': self.pid,
'description': '',
'node_type': 'asset',
'_etag': random_etag(),
}
props = {'status': 'published', 'content_type': 'video', 'order': 0}
if node_duration is not None:
props['duration_seconds'] = node_duration
if include_file:
props['file'] = file_id
return self.create_node({'properties': props, **node})
def _task_log_request(self, manager_id: bson.ObjectId, operation: dict):
managers_coll = current_flamenco.db('managers')
managers_coll.update_one({'_id': manager_id}, {
**operation,
'$set': {
'_updated': utcnow(),
'_etag': random_etag(),
},
})
def _insert_rna_overrides_task(
self, job: dict, parent_task_selector: dict) -> bson.ObjectId:
# Find the task that is supposed to be the parent of the new task.
tasks_coll = current_flamenco.db('tasks')
if parent_task_selector:
parent_task = tasks_coll.find_one(
{
'job': job['_id'],
**parent_task_selector
},
projection={'_id': True})
if not parent_task:
raise ValueError(
'unable to find move-out-of-way task, cannot update this job'
)
parents_kwargs = {'parents': [parent_task['_id']]}
else:
parents_kwargs = {}
# Construct the new task.
cmd = rna_overrides_command(job)
task_id = self._create_task(job, [cmd],
RNA_OVERRIDES_TASK_NAME,
'file-management',
priority=80,
status='queued',
**parents_kwargs)
self._log.info('Inserted RNA Overrides task %s into job %s', task_id,
job['_id'])
# Update existing render tasks to have the new task as parent.
new_etag = random_etag()
now = utcnow()
result = tasks_coll.update_many(
{
'job': job['_id'],
'task_type': 'blender-render',
**parents_kwargs,
}, {
'$set': {
'_etag': new_etag,
'_updated': now,
'parents': [task_id],
}
})
self._log.debug('Updated %d task parent pointers to %s',
result.modified_count, task_id)
return task_id
def api_update_rna_overrides(self, job_id: ObjectId,
rna_overrides: typing.List[str]):
"""API-level call to create or update an RNA override task of a Blender Render job."""
new_etag = random_etag()
now = utcnow()
jobs_coll = current_flamenco.db('jobs')
# Check that the job exists and is a Blender-related job.
job = jobs_coll.find_one({'_id': job_id})
if not job:
self._log.warning(
'Unable to update RNA overrides of non-existing job %s',
job_id)
return None
compiler = job_compilers.construct_job_compiler(job)
if not isinstance(compiler, blender_render.AbstractBlenderJobCompiler):
self._log.warning(
'Job compiler %r is not an AbstractBlenderJobCompiler, unable '
'to update RNA overrides for job %s of type %r',
type(compiler), job_id, job['job_type'])
return None
# Update the job itself before updating its tasks. Ideally this would happen in the
# same transaction.
# TODO(Sybren): put into one transaction when we upgrade to MongoDB 4+.
job['settings']['rna_overrides'] = rna_overrides
result = jobs_coll.update_one({'_id': job_id}, {
'$set': {
'settings.rna_overrides': rna_overrides,
'_updated': now,
'_etag': new_etag,
}
})
if result.matched_count != 1:
self._log.warning(
'Matched %d jobs while setting job %s RNA overrides',
result.matched_count, job_id)
compiler.update_rna_overrides_task(job)
def update_rna_overrides_task(self, job: dict):
"""Update or create an RNA Overrides task of an existing job."""
tasks_coll = current_flamenco.db('tasks')
task = tasks_coll.find_one({'job': job['_id'], 'name': RNA_OVERRIDES_TASK_NAME},
projection={'_id': True})
if not task:
self.insert_rna_overrides_task(job)
return
cmd = rna_overrides_command(job)
new_etag = random_etag()
now = utcnow()
result = tasks_coll.update_one(task, {'$set': {
'_etag': new_etag,
'_updated': now,
'status': 'queued',
'commands': [cmd.to_dict()],
}})
self._log.info('Modified %d RNA override task (%s) of job %s',
result.modified_count, task['_id'], job['_id'])
def api_set_job_priority(self, job_id: ObjectId, new_priority: int):
"""API-level call to updates the job priority."""
assert isinstance(new_priority, int)
self._log.debug('Setting job %s priority to %r', job_id, new_priority)
jobs_coll = current_flamenco.db('jobs')
curr_job = jobs_coll.find_one({'_id': job_id},
projection={'priority': 1})
old_priority = curr_job['priority']
if old_priority == new_priority:
self._log.debug('Job %s is already at priority %r', job_id,
old_priority)
return
new_etag = random_etag()
now = utcnow()
jobs_coll = current_flamenco.db('jobs')
result = jobs_coll.update_one({'_id': job_id}, {
'$set': {
'priority': new_priority,
'_updated': now,
'_etag': new_etag,
}
})
if result.matched_count != 1:
self._log.warning(
'Matched %d jobs while setting job %s to priority %r',
result.matched_count, job_id, new_priority)
tasks_coll = current_flamenco.db('tasks')
result = tasks_coll.update_many({'job': job_id}, {
'$set': {
'job_priority': new_priority,
'_updated': now,
'_etag': new_etag,
}
})
self._log.debug('Matched %d tasks while setting job %s to priority %r',
result.matched_count, job_id, new_priority)
def remove_project_references(node):
project_id = node.get('project')
if not project_id:
return
node_id = node['_id']
log.info('Removing references to node %s from project %s', node_id,
project_id)
projects_col = current_app.db('projects')
project = projects_col.find_one({'_id': project_id})
updates = collections.defaultdict(dict)
if project.get('header_node') == node_id:
updates['$unset']['header_node'] = node_id
project_reference_lists = ('nodes_blog', 'nodes_featured', 'nodes_latest')
for list_name in project_reference_lists:
references = project.get(list_name)
if not references:
continue
try:
references.remove(node_id)
except ValueError:
continue
updates['$set'][list_name] = references
if not updates:
return
updates['$set']['_etag'] = random_etag()
result = projects_col.update_one({'_id': project_id}, updates)
if result.modified_count != 1:
log.warning(
'Removing references to node %s from project %s resulted in %d modified documents (expected 1)',
node_id, project_id, result.modified_count)
def handle_task_update_batch(manager_id, task_updates):
"""Performs task updates.
Task status changes are generally always accepted. The only exception is when the
task ID is contained in 'tasks_to_cancel'; in that case only a transition to either
'canceled', 'completed' or 'failed' is accepted.
:returns: tuple (total nr of modified tasks, handled update IDs)
"""
if not task_updates:
return 0, []
import dateutil.parser
from pillar.api.utils import str2id
from flamenco import current_flamenco, eve_settings
log.debug('Received %i task updates from manager %s', len(task_updates), manager_id)
tasks_coll = current_flamenco.db('tasks')
logs_coll = current_flamenco.db('task_logs')
valid_statuses = set(eve_settings.tasks_schema['status']['allowed'])
handled_update_ids = []
total_modif_count = 0
for task_update in task_updates:
# Check that this task actually belongs to this manager, before we accept any updates.
update_id = str2id(task_update['_id'])
task_id = str2id(task_update['task_id'])
task_info = tasks_coll.find_one({'_id': task_id},
projection={'manager': 1, 'status': 1, 'job': 1})
# For now, we just ignore updates to non-existing tasks. Someone might have just deleted
# one, for example. This is not a reason to reject the entire batch.
if task_info is None:
log.warning('Manager %s sent update for non-existing task %s; accepting but ignoring',
manager_id, task_id)
handled_update_ids.append(update_id)
continue
if task_info['manager'] != manager_id:
log.warning('Manager %s sent update for task %s which belongs to other manager %s',
manager_id, task_id, task_info['manager'])
continue
if task_update.get('received_on_manager'):
received_on_manager = dateutil.parser.parse(task_update['received_on_manager'])
else:
# Fake a 'received on manager' field; it really should have been in the JSON payload.
received_on_manager = utcnow()
# Store the log for this task, allowing for duplicate log reports.
#
# NOTE: is deprecated and will be removed in a future version of Flamenco;
# only periodically send the last few lines of logging in 'log_tail' and
# store the entire log on the Manager itself.
task_log = task_update.get('log')
if task_log:
log_doc = {
'_id': update_id,
'task': task_id,
'received_on_manager': received_on_manager,
'log': task_log
}
logs_coll.replace_one({'_id': update_id}, log_doc, upsert=True)
# Modify the task, and append the log to the logs collection.
updates = {
'task_progress_percentage': task_update.get('task_progress_percentage', 0),
'current_command_index': task_update.get('current_command_index', 0),
'command_progress_percentage': task_update.get('command_progress_percentage', 0),
'_updated': received_on_manager,
'_etag': random_etag(),
}
new_status = determine_new_task_status(manager_id, task_id, task_info,
task_update.get('task_status'), valid_statuses)
if new_status:
updates['status'] = new_status
new_activity = task_update.get('activity')
if new_activity:
updates['activity'] = new_activity
worker = task_update.get('worker')
if worker:
updates['worker'] = worker
# Store the last lines of logging on the task itself.
task_log_tail: str = task_update.get('log_tail')
if not task_log_tail and task_log:
task_log_tail = '\n'.join(task_log.split('\n')[-LOG_TAIL_LINES:])
if task_log_tail:
updates['log'] = task_log_tail
result = tasks_coll.update_one({'_id': task_id}, {'$set': updates})
total_modif_count += result.modified_count
handled_update_ids.append(update_id)
# Update the task's job after updating the task itself.
if new_status:
current_flamenco.job_manager.update_job_after_task_status_change(
task_info['job'], task_id, new_status)
return total_modif_count, handled_update_ids
def patch_project(project_id: str):
"""Undelete a project.
This is done via a custom PATCH due to the lack of transactions of MongoDB;
we cannot undelete both project-referenced files and file-referenced
projects in one atomic operation.
"""
# Parse the request
pid = str2id(project_id)
patch = request.get_json()
if not patch:
raise wz_exceptions.BadRequest('Expected JSON body')
log.debug('User %s wants to PATCH project %s: %s', current_user, pid,
patch)
# 'undelete' is the only operation we support now, so no fancy handler registration.
op = patch.get('op', '')
if op != 'undelete':
log.warning('User %s sent unsupported PATCH op %r to project %s: %s',
current_user, op, pid, patch)
raise wz_exceptions.BadRequest(f'unsupported operation {op!r}')
# Get the project to find the user's permissions.
proj_coll = current_app.db('projects')
proj = proj_coll.find_one({'_id': pid})
if not proj:
raise wz_exceptions.NotFound(f'project {pid} not found')
allowed = authorization.compute_allowed_methods('projects', proj)
if 'PUT' not in allowed:
log.warning(
'User %s tried to undelete project %s but only has permissions %r',
current_user, pid, allowed)
raise wz_exceptions.Forbidden(f'no PUT access to project {pid}')
if not proj.get('_deleted', False):
raise wz_exceptions.BadRequest(
f'project {pid} was not deleted, unable to undelete')
# Undelete the files. We cannot do this via Eve, as it doesn't support
# PATCHing collections, so direct MongoDB modification is used to set
# _deleted=False and provide new _etag and _updated values.
new_etag = random_etag()
log.debug('undeleting files before undeleting project %s', pid)
files_coll = current_app.db('files')
update_result = files_coll.update_many(
{'project': pid},
{'$set': {
'_deleted': False,
'_etag': new_etag,
'_updated': utcnow()
}})
log.info('undeleted %d of %d file documents of project %s',
update_result.modified_count, update_result.matched_count, pid)
log.info('undeleting project %s on behalf of user %s', pid, current_user)
update_result = proj_coll.update_one({'_id': pid},
{'$set': {
'_deleted': False
}})
log.info('undeleted %d project document %s', update_result.modified_count,
pid)
resp = flask.Response('', status=204)
resp.location = flask.url_for('projects.view', project_url=proj['url'])
return resp
def handle_task_update_batch(manager_id, task_updates):
"""Performs task updates.
Task status changes are generally always accepted. The only exception is when the
task ID is contained in 'tasks_to_cancel'; in that case only a transition to either
'canceled', 'completed' or 'failed' is accepted.
:returns: tuple (total nr of modified tasks, handled update IDs)
"""
if not task_updates:
return 0, []
import dateutil.parser
from pillar.api.utils import str2id
from flamenco import current_flamenco, eve_settings
log.debug('Received %i task updates from manager %s', len(task_updates), manager_id)
tasks_coll = current_flamenco.db('tasks')
logs_coll = current_flamenco.db('task_logs')
valid_statuses = set(eve_settings.tasks_schema['status']['allowed'])
handled_update_ids = []
total_modif_count = 0
for task_update in task_updates:
# Check that this task actually belongs to this manager, before we accept any updates.
update_id = str2id(task_update['_id'])
task_id = str2id(task_update['task_id'])
task_info = tasks_coll.find_one({'_id': task_id},
projection={'manager': 1, 'status': 1, 'job': 1})
# For now, we just ignore updates to non-existing tasks. Someone might have just deleted
# one, for example. This is not a reason to reject the entire batch.
if task_info is None:
log.warning('Manager %s sent update for non-existing task %s; accepting but ignoring',
manager_id, task_id)
handled_update_ids.append(update_id)
continue
if task_info['manager'] != manager_id:
log.warning('Manager %s sent update for task %s which belongs to other manager %s',
manager_id, task_id, task_info['manager'])
continue
if task_update.get('received_on_manager'):
received_on_manager = dateutil.parser.parse(task_update['received_on_manager'])
else:
# Fake a 'received on manager' field; it really should have been in the JSON payload.
received_on_manager = utcnow()
# Store the log for this task, allowing for duplicate log reports.
#
# NOTE: is deprecated and will be removed in a future version of Flamenco;
# only periodically send the last few lines of logging in 'log_tail' and
# store the entire log on the Manager itself.
task_log = task_update.get('log')
if task_log:
log_doc = {
'_id': update_id,
'task': task_id,
'received_on_manager': received_on_manager,
'log': task_log
}
logs_coll.replace_one({'_id': update_id}, log_doc, upsert=True)
# Modify the task, and append the log to the logs collection.
updates = {
'task_progress_percentage': task_update.get('task_progress_percentage', 0),
'current_command_index': task_update.get('current_command_index', 0),
'command_progress_percentage': task_update.get('command_progress_percentage', 0),
'_updated': received_on_manager,
'_etag': random_etag(),
}
new_status = determine_new_task_status(manager_id, task_id, task_info,
task_update.get('task_status'), valid_statuses)
if new_status:
updates['status'] = new_status
new_activity = task_update.get('activity')
if new_activity:
updates['activity'] = new_activity
worker = task_update.get('worker')
if worker:
updates['worker'] = worker
metrics_timing = task_update.get('metrics', {}).get('timing')
if metrics_timing:
updates['metrics.timing'] = metrics_timing
fbw = task_update.get('failed_by_workers')
if fbw is not None:
updates['failed_by_workers'] = fbw
# Store the last lines of logging on the task itself.
task_log_tail: str = task_update.get('log_tail')
if not task_log_tail and task_log:
task_log_tail = '\n'.join(task_log.split('\n')[-LOG_TAIL_LINES:])
if task_log_tail:
updates['log'] = task_log_tail
result = tasks_coll.update_one({'_id': task_id}, {'$set': updates})
total_modif_count += result.modified_count
handled_update_ids.append(update_id)
# Update the task's job after updating the task itself.
if new_status:
current_flamenco.job_manager.update_job_after_task_status_change(
task_info['job'], task_id, new_status)
return total_modif_count, handled_update_ids