def get_depsgraph(manager_id, request_json):
"""Returns the dependency graph of all tasks assigned to the given Manager.
Use the HTTP header X-Flamenco-If-Updated-Since to limit the dependency
graph to tasks that have been modified since that timestamp.
"""
import dateutil.parser
from pillar.api.utils import jsonify, bsonify
from flamenco import current_flamenco
from flamenco.utils import report_duration
modified_since = request.headers.get('X-Flamenco-If-Updated-Since')
with report_duration(log, 'depsgraph query'):
tasks_coll = current_flamenco.db('tasks')
jobs_coll = current_flamenco.db('jobs')
# Get runnable jobs first, as non-runnable jobs are not interesting.
# Note that jobs going from runnable to non-runnable should have their
# tasks set to cancel-requested, which is communicated to the Manager
# through a different channel.
jobs = jobs_coll.find({
'manager': manager_id,
'status': {'$in': DEPSGRAPH_RUNNABLE_JOB_STATUSES}},
projection={'_id': 1},
)
job_ids = [job['_id'] for job in jobs]
if not job_ids:
log.debug('Returning empty depsgraph')
return '', 204 # empty response
log.debug('Requiring jobs to be in %s', job_ids)
task_query = {
'manager': manager_id,
'status': {'$nin': ['active']},
'job': {'$in': job_ids},
}
if modified_since is None:
# "Clean slate" query.
task_query['status'] = {'$in': DEPSGRAPH_CLEAN_SLATE_TASK_STATUSES}
else:
# Not clean slate, just give all updated tasks assigned to this manager.
log.debug('Modified-since header: %s', modified_since)
modified_since = dateutil.parser.parse(modified_since)
task_query['_updated'] = {'$gt': modified_since}
task_query['status'] = {'$in': DEPSGRAPH_MODIFIED_SINCE_TASK_STATUSES}
log.debug('Querying all tasks changed since %s', modified_since)
cursor = tasks_coll.find(task_query)
depsgraph = list(cursor)
if len(depsgraph) == 0:
log.debug('Returning empty depsgraph')
if modified_since is not None:
return '', 304 # Not Modified
else:
log.info('Returning depsgraph of %i tasks', len(depsgraph))
# Update the task status in the database to move queued tasks to claimed-by-manager.
# This also erases the link to any previously uploaded log files, to ensure the
# log is fresh and represents the current execution of the task.
task_query['status'] = 'queued'
tasks_coll.update_many(task_query, {
'$set': {'status': 'claimed-by-manager'},
'$unset': {'log_file': True},
})
# Update the returned task statuses. Unfortunately Mongo doesn't support
# find_and_modify() on multiple documents.
for task in depsgraph:
if task['status'] == 'queued':
task['status'] = 'claimed-by-manager'
# Must be a dict to convert to BSON.
respdoc = {
'depsgraph': depsgraph,
}
if request.accept_mimetypes.best == 'application/bson':
resp = bsonify(respdoc)
else:
resp = jsonify(respdoc)
if depsgraph:
last_modification = max(task['_updated'] for task in depsgraph)
log.debug('Last modification was %s', last_modification)
# We need a format that can handle sub-second precision, which is not provided by the
# HTTP date format (RFC 1123). This means that we can't use the Last-Modified header, as
# it may be incorrectly interpreted and rewritten by HaProxy, Apache or other software
# in the path between client & server.
resp.headers['X-Flamenco-Last-Updated'] = last_modification.isoformat()
resp.headers['X-Flamenco-Last-Updated-Format'] = 'ISO-8601'
return resp
def get_depsgraph(manager_id, request_json):
"""Returns the dependency graph of all tasks assigned to the given Manager.
Use the HTTP header X-Flamenco-If-Updated-Since to limit the dependency
graph to tasks that have been modified since that timestamp.
"""
import dateutil.parser
from pillar.api.utils import jsonify, bsonify
from flamenco import current_flamenco
from flamenco.utils import report_duration
modified_since = request.headers.get('X-Flamenco-If-Updated-Since')
with report_duration(log, 'depsgraph query'):
tasks_coll = current_flamenco.db('tasks')
jobs_coll = current_flamenco.db('jobs')
# Get runnable jobs first, as non-runnable jobs are not interesting.
# Note that jobs going from runnable to non-runnable should have their
# tasks set to cancel-requested, which is communicated to the Manager
# through a different channel.
jobs = jobs_coll.find({
'manager': manager_id,
'status': {'$in': DEPSGRAPH_RUNNABLE_JOB_STATUSES}},
projection={'_id': 1},
)
job_ids = [job['_id'] for job in jobs]
if not job_ids:
log.debug('Returning empty depsgraph')
return '', 204 # empty response
log.debug('Requiring jobs to be in %s', job_ids)
task_query = {
'manager': manager_id,
'status': {'$nin': ['active']},
'job': {'$in': job_ids},
}
if modified_since is None:
# "Clean slate" query.
task_query['status'] = {'$in': DEPSGRAPH_CLEAN_SLATE_TASK_STATUSES}
else:
# Not clean slate, just give all updated tasks assigned to this manager.
log.debug('Modified-since header: %s', modified_since)
modified_since = dateutil.parser.parse(modified_since)
task_query['_updated'] = {'$gt': modified_since}
task_query['status'] = {'$in': DEPSGRAPH_MODIFIED_SINCE_TASK_STATUSES}
log.debug('Querying all tasks changed since %s', modified_since)
cursor = tasks_coll.find(task_query)
depsgraph = list(cursor)
if len(depsgraph) == 0:
log.debug('Returning empty depsgraph')
if modified_since is not None:
return '', 304 # Not Modified
else:
log.info('Returning depsgraph of %i tasks', len(depsgraph))
# Update the task status in the database to move queued tasks to claimed-by-manager.
# This also erases the link to any previously uploaded log files, to ensure the
# log is fresh and represents the current execution of the task.
task_query['status'] = 'queued'
tasks_coll.update_many(task_query, {
'$set': {'status': 'claimed-by-manager'},
'$unset': {'log_file': True},
})
# Update the returned task statuses. Unfortunately Mongo doesn't support
# find_and_modify() on multiple documents.
for task in depsgraph:
if task['status'] == 'queued':
task['status'] = 'claimed-by-manager'
# Must be a dict to convert to BSON.
respdoc = {
'depsgraph': depsgraph,
}
if request.accept_mimetypes.best == 'application/bson':
resp = bsonify(respdoc)
else:
resp = jsonify(respdoc)
if depsgraph:
last_modification = max(task['_updated'] for task in depsgraph)
log.debug('Last modification was %s', last_modification)
# We need a format that can handle sub-second precision, which is not provided by the
# HTTP date format (RFC 1123). This means that we can't use the Last-Modified header, as
# it may be incorrectly interpreted and rewritten by HaProxy, Apache or other software
# in the path between client & server.
resp.headers['X-Flamenco-Last-Updated'] = last_modification.isoformat()
resp.headers['X-Flamenco-Last-Updated-Format'] = 'ISO-8601'
return resp