def get(self, request):
'''Gets jobs and their associated latest execution
:param request: the HTTP GET request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
'''
started = rest_util.parse_timestamp(request, u'started', required=False)
ended = rest_util.parse_timestamp(request, u'ended', required=False)
rest_util.check_time_range(started, ended)
job_status = rest_util.parse_string(request, u'status', required=False)
job_type_ids = rest_util.parse_int_list(request, u'job_type_id', required=False)
job_type_names = rest_util.parse_string_list(request, u'job_type_name', required=False)
job_type_categories = rest_util.parse_string_list(request, u'job_type_category', required=False)
order = rest_util.parse_string_list(request, u'order', required=False)
jobs = Job.objects.get_jobs(started, ended, job_status, job_type_ids, job_type_names, job_type_categories,
order)
page = rest_util.perform_paging(request, jobs)
# Add the latest execution for each matching job
paged_jobs = list(page.object_list)
job_exes_dict = JobExecution.objects.get_latest(page.object_list)
for job in paged_jobs:
job.latest_job_exe = job_exes_dict[job.id] if job.id in job_exes_dict else None
page.object_list = paged_jobs
serializer = JobWithExecutionListSerializer(page, context={u'request': request})
return Response(serializer.data, status=status.HTTP_200_OK)
def get(self, request):
'''Retrieves the list of all nodes with execution status and returns it in JSON form
:param request: the HTTP GET request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
'''
# Get a list of all node status counts
started = rest_util.parse_timestamp(request, 'started', 'PT3H0M0S')
ended = rest_util.parse_timestamp(request, 'ended', required=False)
node_statuses = Node.objects.get_status(started, ended)
# Get the online nodes
try:
sched = Scheduler.objects.get_master()
slaves = mesos_api.get_slaves(sched.master_hostname, sched.master_port)
slaves_dict = {s.hostname for s in slaves}
except:
logger.exception('Unable to fetch nodes online status')
slaves_dict = dict()
# Add the online status to each node
for node_status in node_statuses:
node_status.is_online = node_status.node.hostname in slaves_dict
page = rest_util.perform_paging(request, node_statuses)
serializer = NodeStatusListSerializer(page, context={'request': request})
return Response(serializer.data, status=status.HTTP_200_OK)
def get(self, request):
'''Gets job executions and their associated job_type id, name, and version
:param request: the HTTP GET request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
'''
started = rest_util.parse_timestamp(request, u'started', required=False)
ended = rest_util.parse_timestamp(request, u'ended', required=False)
rest_util.check_time_range(started, ended)
job_status = rest_util.parse_string(request, u'status', required=False)
job_type_ids = rest_util.parse_int_list(request, u'job_type_id', required=False)
job_type_names = rest_util.parse_string_list(request, u'job_type_name', required=False)
job_type_categories = rest_util.parse_string_list(request, u'job_type_category', required=False)
node_ids = rest_util.parse_int_list(request, u'node_id', required=False)
order = rest_util.parse_string_list(request, u'order', required=False)
job_exes = JobExecution.objects.get_exes(started, ended, job_status, job_type_ids, job_type_names,
job_type_categories, node_ids, order)
page = rest_util.perform_paging(request, job_exes)
serializer = JobExecutionListSerializer(page, context={u'request': request})
return Response(serializer.data, status=status.HTTP_200_OK)
def get(self, request):
'''Retrieves the job updates for a given time range and returns it in JSON form
:param request: the HTTP GET request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
'''
started = rest_util.parse_timestamp(request, u'started', required=False)
ended = rest_util.parse_timestamp(request, u'ended', required=False)
rest_util.check_time_range(started, ended)
job_status = rest_util.parse_string(request, u'status', required=False)
job_type_ids = rest_util.parse_int_list(request, u'job_type_id', required=False)
job_type_names = rest_util.parse_string_list(request, u'job_type_name', required=False)
job_type_categories = rest_util.parse_string_list(request, u'job_type_category', required=False)
order = rest_util.parse_string_list(request, u'order', required=False)
jobs = Job.objects.get_job_updates(started, ended, job_status, job_type_ids, job_type_names,
job_type_categories, order)
page = rest_util.perform_paging(request, jobs)
Job.objects.populate_input_files(page)
serializer = JobUpdateListSerializer(page, context={'request': request})
return Response(serializer.data, status=status.HTTP_200_OK)
def get(self, request, name):
'''Retrieves the plot values for metrics and return them in JSON form
:param request: the HTTP GET request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
'''
started = rest_util.parse_timestamp(request, 'started', required=False)
ended = rest_util.parse_timestamp(request, 'ended', required=False)
rest_util.check_time_range(started, ended)
choice_ids = rest_util.parse_string_list(request, 'choice_id', required=False)
column_names = rest_util.parse_string_list(request, 'column', required=False)
group_names = rest_util.parse_string_list(request, 'group', required=False)
try:
provider = registry.get_provider(name)
metrics_type = provider.get_metrics_type(include_choices=False)
except MetricsTypeError:
raise Http404
# Build a unique set of column names from groups
columns = metrics_type.get_column_set(column_names, group_names)
# Get the actual plot values
metrics_values = provider.get_plot_data(started, ended, choice_ids, columns)
page = rest_util.perform_paging(request, metrics_values)
if len(choice_ids) > 1:
serializer = MetricsPlotMultiListSerializer(page, context={'request': request})
else:
serializer = MetricsPlotListSerializer(page, context={'request': request})
return Response(serializer.data, status=status.HTTP_200_OK)
def get(self, request):
'''Retrieves the product updates for a given time range and returns it in JSON form
:param request: the HTTP GET request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
'''
started = rest_util.parse_timestamp(request, u'started', required=False)
ended = rest_util.parse_timestamp(request, u'ended', required=False)
rest_util.check_time_range(started, ended)
job_type_ids = rest_util.parse_int_list(request, u'job_type_id', required=False)
job_type_names = rest_util.parse_string_list(request, u'job_type_name', required=False)
job_type_categories = rest_util.parse_string_list(request, u'job_type_category', required=False)
is_operational = rest_util.parse_bool(request, u'is_operational', required=False)
file_name = rest_util.parse_string(request, u'file_name', required=False)
order = rest_util.parse_string_list(request, u'order', required=False)
products = ProductFile.objects.get_products(started, ended, job_type_ids, job_type_names, job_type_categories,
is_operational, file_name, order)
page = rest_util.perform_paging(request, products)
ProductFile.objects.populate_source_ancestors(page)
serializer = ProductFileUpdateListSerializer(page, context={'request': request})
return Response(serializer.data, status=status.HTTP_200_OK)
def get(self, request):
'''Retrieves the metrics types and returns it in JSON form
:param request: the HTTP GET request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
'''
metrics_types = registry.get_metrics_types()
page = rest_util.perform_paging(request, metrics_types)
serializer = MetricsTypeListSerializer(page, context={'request': request})
return Response(serializer.data, status=status.HTTP_200_OK)
def get(self, request):
'''Retrieves the job types that have failed with system errors and returns them in JSON form
:param request: the HTTP GET request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
'''
# Get all the failed job types with statistics
failed_status = JobType.objects.get_failed_status()
# Wrap the response with paging information
page = rest_util.perform_paging(request, failed_status)
serializer = JobTypeFailedStatusListSerializer(page, context={u'request': request})
return Response(serializer.data, status=status.HTTP_200_OK)
def get(self, request):
'''Retrieves the current status of running job types and returns it in JSON form
:param request: the HTTP GET request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
'''
# Get all the running job types with statistics
running_status = JobType.objects.get_running_status()
# Wrap the response with paging information
page = rest_util.perform_paging(request, running_status)
serializer = JobTypeRunningStatusListSerializer(page, context={u'request': request})
return Response(serializer.data, status=status.HTTP_200_OK)
def get(self, request):
'''Retrieves the list of all job types with status and returns it in JSON form
:param request: the HTTP GET request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
'''
# Get a list of all job type status counts
started = rest_util.parse_timestamp(request, u'started', u'PT3H0M0S')
ended = rest_util.parse_timestamp(request, u'ended', required=False)
job_type_statuses = JobType.objects.get_status(started, ended)
page = rest_util.perform_paging(request, job_type_statuses)
serializer = JobTypeStatusListSerializer(page, context={u'request': request})
return Response(serializer.data, status=status.HTTP_200_OK)
def get(self, request):
'''Retrieves the list of all recipe types returns it in JSON form
:param request: the HTTP GET request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
'''
started = rest_util.parse_timestamp(request, 'started', required=False)
ended = rest_util.parse_timestamp(request, 'ended', required=False)
rest_util.check_time_range(started, ended)
order = rest_util.parse_string_list(request, 'order', ['name', 'version'])
recipe_types = RecipeType.objects.get_recipe_types(started, ended, order)
page = rest_util.perform_paging(request, recipe_types)
serializer = RecipeTypeListSerializer(page, context={'request': request})
return Response(serializer.data, status=status.HTTP_200_OK)
def get(self, request):
'''Retrieves the list of all nodes and returns it in JSON form
:param request: the HTTP GET request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
'''
started = rest_util.parse_timestamp(request, 'started', required=False)
ended = rest_util.parse_timestamp(request, 'ended', required=False)
rest_util.check_time_range(started, ended)
include_inactive = rest_util.parse_bool(request, 'include_inactive', False, False)
order = rest_util.parse_string_list(request, 'order', required=False)
nodes = Node.objects.get_nodes(started, ended, order, include_inactive=include_inactive)
page = rest_util.perform_paging(request, nodes)
serializer = NodeListSerializer(page, context={'request': request})
return Response(serializer.data, status=status.HTTP_200_OK)
def get(self, request):
'''Retrieves the ingest status information and returns it in JSON form
:param request: the HTTP GET request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
'''
started = rest_util.parse_timestamp(request, 'started', rest_util.get_relative_days(7))
ended = rest_util.parse_timestamp(request, 'ended', required=False)
rest_util.check_time_range(started, ended, max_duration=datetime.timedelta(days=31))
use_ingest_time = rest_util.parse_bool(request, 'use_ingest_time', default_value=False)
ingests = Ingest.objects.get_status(started, ended, use_ingest_time)
page = rest_util.perform_paging(request, ingests)
serializer = IngestStatusListSerializer(page, context={'request': request})
return Response(serializer.data, status=status.HTTP_200_OK)
def get(self, request):
'''Retrieves the list of all errors and returns it in JSON form
:param request: the HTTP GET request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
'''
started = rest_util.parse_timestamp(request, u'started', required=False)
ended = rest_util.parse_timestamp(request, u'ended', required=False)
rest_util.check_time_range(started, ended)
order = rest_util.parse_string_list(request, u'order', required=False)
errors = Error.objects.get_errors(started, ended, order)
page = rest_util.perform_paging(request, errors)
serializer = ErrorListSerializer(page, context={u'request': request})
return Response(serializer.data, status=status.HTTP_200_OK)
def get(self, request):
'''Retrieves the source file updates for a given time range and returns it in JSON form
:param request: the HTTP GET request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
'''
started = rest_util.parse_timestamp(request, u'started', required=False)
ended = rest_util.parse_timestamp(request, u'ended', required=False)
rest_util.check_time_range(started, ended)
is_parsed = rest_util.parse_bool(request, u'is_parsed', required=False)
file_name = rest_util.parse_string(request, u'file_name', required=False)
order = rest_util.parse_string_list(request, u'order', required=False)
sources = SourceFile.objects.get_sources(started, ended, is_parsed, file_name, order)
page = rest_util.perform_paging(request, sources)
serializer = SourceFileUpdateListSerializer(page, context={'request': request})
return Response(serializer.data, status=status.HTTP_200_OK)
def get(self, request):
'''Retrieves the list of all job types and returns it in JSON form
:param request: the HTTP GET request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
'''
started = rest_util.parse_timestamp(request, u'started', required=False)
ended = rest_util.parse_timestamp(request, u'ended', required=False)
rest_util.check_time_range(started, ended)
names = rest_util.parse_string_list(request, u'name', required=False)
categories = rest_util.parse_string_list(request, u'category', required=False)
order = rest_util.parse_string_list(request, u'order', [u'name', u'version'])
job_types = JobType.objects.get_job_types(started, ended, names, categories, order)
page = rest_util.perform_paging(request, job_types)
serializer = JobTypeListSerializer(page, context={u'request': request})
return Response(serializer.data, status=status.HTTP_200_OK)
def get(self, request):
"""Retrieves the job load for a given time range and returns it in JSON form
:param request: the HTTP GET request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
"""
started = rest_util.parse_timestamp(request, 'started', default_value=rest_util.get_relative_days(7))
ended = rest_util.parse_timestamp(request, 'ended', required=False)
rest_util.check_time_range(started, ended, max_duration=datetime.timedelta(days=31))
job_type_ids = rest_util.parse_int_list(request, 'job_type_id', required=False)
job_type_names = rest_util.parse_string_list(request, 'job_type_name', required=False)
job_type_categories = rest_util.parse_string_list(request, 'job_type_category', required=False)
job_type_priorities = rest_util.parse_string_list(request, 'job_type_priority', required=False)
job_loads = JobLoad.objects.get_job_loads(started, ended, job_type_ids, job_type_names, job_type_categories,
job_type_priorities)
job_loads_grouped = JobLoad.objects.group_by_time(job_loads)
page = rest_util.perform_paging(request, job_loads_grouped)
serializer = JobLoadGroupListSerializer(page, context={'request': request})
return Response(serializer.data, status=status.HTTP_200_OK)
def post(self, request):
"""Increase max_tries, place it on the queue, and returns the new job information in JSON form
:param request: the HTTP GET request
:type request: :class:`rest_framework.request.Request`
:returns: the HTTP response to send back to the user
"""
started = rest_util.parse_timestamp(request, 'started', required=False)
ended = rest_util.parse_timestamp(request, 'ended', required=False)
rest_util.check_time_range(started, ended)
job_status = rest_util.parse_string(request, 'status', required=False)
job_ids = rest_util.parse_int_list(request, 'job_ids', required=False)
job_type_ids = rest_util.parse_int_list(request, 'job_type_ids', required=False)
job_type_names = rest_util.parse_string_list(request, 'job_type_names', required=False)
job_type_categories = rest_util.parse_string_list(request, 'job_type_categories', required=False)
priority = rest_util.parse_int(request, 'priority', required=False)
# Fetch all the jobs matching the filters
jobs = Job.objects.get_jobs(started, ended, job_status, job_ids, job_type_ids, job_type_names,
job_type_categories)
if not jobs:
raise Http404
# Attempt to queue all jobs matching the filters
requested_job_ids = {job.id for job in jobs}
Queue.objects.requeue_jobs(requested_job_ids, priority)
# Refresh models to get the new status information for all originally requested jobs
jobs = Job.objects.get_jobs(job_ids=requested_job_ids)
page = rest_util.perform_paging(request, jobs)
serializer = JobListSerializer(page, context={'request': request})
return Response(serializer.data, status=status.HTTP_200_OK)