def create(self, request):
"""Creates a new Strike process and returns a link to the detail URL
:param request: the HTTP POST request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
"""
name = rest_util.parse_string(request, 'name')
title = rest_util.parse_string(request, 'title', required=False)
description = rest_util.parse_string(request, 'description', required=False)
configuration = rest_util.parse_dict(request, 'configuration')
try:
strike = Strike.objects.create_strike(name, title, description, configuration)
except InvalidStrikeConfiguration as ex:
raise BadParameter('Strike configuration invalid: %s' % unicode(ex))
# Fetch the full strike process with details
try:
strike = Strike.objects.get_details(strike.id)
except Strike.DoesNotExist:
raise Http404
serializer = StrikeDetailsSerializer(strike)
strike_url = urlresolvers.reverse('strike_details_view', args=[strike.id])
return Response(serializer.data, status=status.HTTP_201_CREATED, headers=dict(location=strike_url))
def post(self, request):
"""Validates a new Strike process and returns any warnings discovered
:param request: the HTTP POST request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
"""
name = rest_util.parse_string(request, 'name')
configuration = rest_util.parse_dict(request, 'configuration')
rest_util.parse_string(request, 'title', required=False)
rest_util.parse_string(request, 'description', required=False)
# Validate the Strike configuration
try:
config = StrikeConfiguration(configuration)
warnings = config.validate()
except InvalidStrikeConfiguration as ex:
logger.exception('Unable to validate new Strike process: %s', name)
raise BadParameter(unicode(ex))
results = [{'id': w.key, 'details': w.details} for w in warnings]
return Response({'warnings': results})
def patch(self, request, strike_id):
"""Edits an existing Strike process and returns the updated details
:param request: the HTTP GET request
:type request: :class:`rest_framework.request.Request`
:param strike_id: The ID of the Strike process
:type strike_id: int encoded as a str
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
"""
title = rest_util.parse_string(request, 'title', required=False)
description = rest_util.parse_string(request, 'description', required=False)
configuration = rest_util.parse_dict(request, 'configuration', required=False)
try:
Strike.objects.edit_strike(strike_id, title, description, configuration)
strike = Strike.objects.get_details(strike_id)
except Strike.DoesNotExist:
raise Http404
except InvalidStrikeConfiguration as ex:
logger.exception('Unable to edit Strike process: %s', strike_id)
raise BadParameter(unicode(ex))
serializer = self.get_serializer(strike)
return Response(serializer.data)
def patch(self, request, workspace_id):
"""Edits an existing workspace and returns the updated details
:param request: the HTTP GET request
:type request: :class:`rest_framework.request.Request`
:param workspace_id: The ID for the workspace.
:type workspace_id: int encoded as a str
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
"""
title = rest_util.parse_string(request, "title", required=False)
description = rest_util.parse_string(request, "description", required=False)
json_config = rest_util.parse_dict(request, "json_config", required=False)
base_url = rest_util.parse_string(request, "base_url", required=False)
is_active = rest_util.parse_string(request, "is_active", required=False)
try:
Workspace.objects.edit_workspace(workspace_id, title, description, json_config, base_url, is_active)
workspace = Workspace.objects.get_details(workspace_id)
except Workspace.DoesNotExist:
raise Http404
except InvalidWorkspaceConfiguration as ex:
logger.exception("Unable to edit workspace: %s", workspace_id)
raise BadParameter(unicode(ex))
serializer = self.get_serializer(workspace)
return Response(serializer.data)
def patch(self, request, error_id):
"""Edits an existing error and returns the updated model
:param request: the HTTP PATCH request
:type request: :class:`rest_framework.request.Request`
:param error_id: The id of the error
:type error_id: int encoded as a str
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
"""
try:
error = Error.objects.get(pk=error_id)
except Error.DoesNotExist:
raise Http404
# Do not allow editing of SYSTEM level errors
if error.category == 'SYSTEM':
raise BadParameter('System level errors cannot be edited.')
title = rest_util.parse_string(request, 'title', required=False)
description = rest_util.parse_string(request, 'description', required=False)
category = rest_util.parse_string(request, 'category', required=False,
accepted_values=[c for c, _ in Error.CATEGORIES])
# Do not allow editing of SYSTEM level errors
if category == 'SYSTEM':
raise BadParameter('Errors cannot be changed to system level.')
error.title = title or error.title
error.description = description or error.description
error.category = category or error.category
error.save()
serializer = self.get_serializer(error)
return Response(serializer.data)
def create(self, request):
"""Creates a new Workspace and returns it in JSON form
:param request: the HTTP POST request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
"""
name = rest_util.parse_string(request, "name")
title = rest_util.parse_string(request, "title", required=False)
description = rest_util.parse_string(request, "description", required=False)
json_config = rest_util.parse_dict(request, "json_config")
base_url = rest_util.parse_string(request, "base_url", required=False)
is_active = rest_util.parse_bool(request, "is_active", default_value=True, required=False)
try:
workspace = Workspace.objects.create_workspace(name, title, description, json_config, base_url, is_active)
except InvalidWorkspaceConfiguration as ex:
logger.exception("Unable to create new workspace: %s", name)
raise BadParameter(unicode(ex))
# Fetch the full workspace with details
try:
workspace = Workspace.objects.get_details(workspace.id)
except Workspace.DoesNotExist:
raise Http404
serializer = WorkspaceDetailsSerializer(workspace)
workspace_url = reverse("workspace_details_view", args=[workspace.id], request=request)
return Response(serializer.data, status=status.HTTP_201_CREATED, headers=dict(location=workspace_url))
def post(self, request):
"""Creates a new recipe type and returns a link to the detail URL
:param request: the HTTP POST request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
"""
name = rest_util.parse_string(request, 'name')
version = rest_util.parse_string(request, 'version')
title = rest_util.parse_string(request, 'title', default_value=name)
description = rest_util.parse_string(request, 'description', required=False)
definition_dict = rest_util.parse_dict(request, 'definition')
# Check for optional trigger rule parameters
trigger_rule_dict = rest_util.parse_dict(request, 'trigger_rule', required=False)
if (('type' in trigger_rule_dict and 'configuration' not in trigger_rule_dict) or
('type' not in trigger_rule_dict and 'configuration' in trigger_rule_dict)):
raise BadParameter('Trigger type and configuration are required together.')
is_active = trigger_rule_dict['is_active'] if 'is_active' in trigger_rule_dict else True
# Attempt to look up the trigger handler for the type
rule_handler = None
if trigger_rule_dict and 'type' in trigger_rule_dict:
try:
rule_handler = trigger_handler.get_trigger_rule_handler(trigger_rule_dict['type'])
except InvalidTriggerType as ex:
logger.exception('Invalid trigger type for new recipe type: %s', name)
raise BadParameter(unicode(ex))
try:
with transaction.atomic():
# Validate the recipe definition
recipe_def = RecipeDefinition(definition_dict)
# Attempt to create the trigger rule
trigger_rule = None
if rule_handler and 'configuration' in trigger_rule_dict:
trigger_rule = rule_handler.create_trigger_rule(trigger_rule_dict['configuration'], name, is_active)
# Create the recipe type
recipe_type = RecipeType.objects.create_recipe_type(name, version, title, description, recipe_def,
trigger_rule)
except (InvalidDefinition, InvalidTriggerType, InvalidTriggerRule, InvalidRecipeConnection) as ex:
logger.exception('Unable to create new recipe type: %s', name)
raise BadParameter(unicode(ex))
# Fetch the full recipe type with details
try:
recipe_type = RecipeType.objects.get_details(recipe_type.id)
except RecipeType.DoesNotExist:
raise Http404
url = urlresolvers.reverse('recipe_type_details_view', args=[recipe_type.id])
serializer = RecipeTypeDetailsSerializer(recipe_type)
return Response(serializer.data, status=status.HTTP_201_CREATED, headers=dict(location=url))
def post(self, request):
"""Validates a new workspace and returns any warnings discovered
:param request: the HTTP POST request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
"""
name = rest_util.parse_string(request, "name")
json_config = rest_util.parse_dict(request, "json_config")
rest_util.parse_string(request, "title", required=False)
rest_util.parse_string(request, "description", required=False)
rest_util.parse_string(request, "base_url", required=False)
rest_util.parse_string(request, "is_active", required=False)
# Validate the workspace configuration
try:
warnings = Workspace.objects.validate_workspace(name, json_config)
except InvalidWorkspaceConfiguration as ex:
logger.exception("Unable to validate new workspace: %s", name)
raise BadParameter(unicode(ex))
results = [{"id": w.key, "details": w.details} for w in warnings]
return Response({"warnings": results})
def post(self, request):
"""Validates a new recipe type and returns any warnings discovered
:param request: the HTTP POST request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
"""
name = rest_util.parse_string(request, 'name')
version = rest_util.parse_string(request, 'version')
title = rest_util.parse_string(request, 'title', default_value=name)
description = rest_util.parse_string(request, 'description', required=False)
definition_dict = rest_util.parse_dict(request, 'definition')
# Check for optional trigger rule parameters
trigger_rule_dict = rest_util.parse_dict(request, 'trigger_rule', required=False)
if (('type' in trigger_rule_dict and 'configuration' not in trigger_rule_dict) or
('type' not in trigger_rule_dict and 'configuration' in trigger_rule_dict)):
raise BadParameter('Trigger type and configuration are required together.')
# Attempt to look up the trigger handler for the type
rule_handler = None
if trigger_rule_dict and 'type' in trigger_rule_dict:
try:
rule_handler = trigger_handler.get_trigger_rule_handler(trigger_rule_dict['type'])
except InvalidTriggerType as ex:
logger.exception('Invalid trigger type for recipe validation: %s', name)
raise BadParameter(unicode(ex))
# Attempt to look up the trigger rule configuration
trigger_config = None
if rule_handler and 'configuration' in trigger_rule_dict:
try:
trigger_config = rule_handler.create_configuration(trigger_rule_dict['configuration'])
except InvalidTriggerRule as ex:
logger.exception('Invalid trigger rule configuration for recipe validation: %s', name)
raise BadParameter(unicode(ex))
# Validate the recipe definition
try:
recipe_def = RecipeDefinition(definition_dict)
warnings = RecipeType.objects.validate_recipe_type(name, title, version, description, recipe_def,
trigger_config)
except (InvalidDefinition, InvalidTriggerType, InvalidTriggerRule, InvalidRecipeConnection) as ex:
logger.exception('Unable to validate new recipe type: %s', name)
raise BadParameter(unicode(ex))
results = [{'id': w.key, 'details': w.details} for w in warnings]
return Response({'warnings': results})
def get(self, request):
'''Gets job executions and their associated job_type id, name, and version
:param request: the HTTP GET request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
'''
started = rest_util.parse_timestamp(request, u'started', required=False)
ended = rest_util.parse_timestamp(request, u'ended', required=False)
rest_util.check_time_range(started, ended)
job_status = rest_util.parse_string(request, u'status', required=False)
job_type_ids = rest_util.parse_int_list(request, u'job_type_id', required=False)
job_type_names = rest_util.parse_string_list(request, u'job_type_name', required=False)
job_type_categories = rest_util.parse_string_list(request, u'job_type_category', required=False)
node_ids = rest_util.parse_int_list(request, u'node_id', required=False)
order = rest_util.parse_string_list(request, u'order', required=False)
job_exes = JobExecution.objects.get_exes(started, ended, job_status, job_type_ids, job_type_names,
job_type_categories, node_ids, order)
page = rest_util.perform_paging(request, job_exes)
serializer = JobExecutionListSerializer(page, context={u'request': request})
return Response(serializer.data, status=status.HTTP_200_OK)
def get(self, request):
'''Gets jobs and their associated latest execution
:param request: the HTTP GET request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
'''
started = rest_util.parse_timestamp(request, u'started', required=False)
ended = rest_util.parse_timestamp(request, u'ended', required=False)
rest_util.check_time_range(started, ended)
job_status = rest_util.parse_string(request, u'status', required=False)
job_type_ids = rest_util.parse_int_list(request, u'job_type_id', required=False)
job_type_names = rest_util.parse_string_list(request, u'job_type_name', required=False)
job_type_categories = rest_util.parse_string_list(request, u'job_type_category', required=False)
order = rest_util.parse_string_list(request, u'order', required=False)
jobs = Job.objects.get_jobs(started, ended, job_status, job_type_ids, job_type_names, job_type_categories,
order)
page = rest_util.perform_paging(request, jobs)
# Add the latest execution for each matching job
paged_jobs = list(page.object_list)
job_exes_dict = JobExecution.objects.get_latest(page.object_list)
for job in paged_jobs:
job.latest_job_exe = job_exes_dict[job.id] if job.id in job_exes_dict else None
page.object_list = paged_jobs
serializer = JobWithExecutionListSerializer(page, context={u'request': request})
return Response(serializer.data, status=status.HTTP_200_OK)
def get(self, request):
'''Retrieves the job updates for a given time range and returns it in JSON form
:param request: the HTTP GET request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
'''
started = rest_util.parse_timestamp(request, u'started', required=False)
ended = rest_util.parse_timestamp(request, u'ended', required=False)
rest_util.check_time_range(started, ended)
job_status = rest_util.parse_string(request, u'status', required=False)
job_type_ids = rest_util.parse_int_list(request, u'job_type_id', required=False)
job_type_names = rest_util.parse_string_list(request, u'job_type_name', required=False)
job_type_categories = rest_util.parse_string_list(request, u'job_type_category', required=False)
order = rest_util.parse_string_list(request, u'order', required=False)
jobs = Job.objects.get_job_updates(started, ended, job_status, job_type_ids, job_type_names,
job_type_categories, order)
page = rest_util.perform_paging(request, jobs)
Job.objects.populate_input_files(page)
serializer = JobUpdateListSerializer(page, context={'request': request})
return Response(serializer.data, status=status.HTTP_200_OK)
def list(self, request):
"""Gets jobs and their associated latest execution
:param request: the HTTP GET request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
"""
started = rest_util.parse_timestamp(request, 'started', required=False)
ended = rest_util.parse_timestamp(request, 'ended', required=False)
rest_util.check_time_range(started, ended)
job_status = rest_util.parse_string(request, 'status', required=False)
job_ids = rest_util.parse_int_list(request, 'job_id', required=False)
job_type_ids = rest_util.parse_int_list(request, 'job_type_id', required=False)
job_type_names = rest_util.parse_string_list(request, 'job_type_name', required=False)
job_type_categories = rest_util.parse_string_list(request, 'job_type_category', required=False)
order = rest_util.parse_string_list(request, 'order', required=False)
jobs = Job.objects.get_jobs(started, ended, job_status, job_ids, job_type_ids, job_type_names,
job_type_categories, order)
# Add the latest execution for each matching job
page = self.paginate_queryset(jobs)
job_exes_dict = JobExecution.objects.get_latest(page)
for job in page:
job.latest_job_exe = job_exes_dict[job.id] if job.id in job_exes_dict else None
serializer = self.get_serializer(page, many=True)
return self.get_paginated_response(serializer.data)
def list(self, request):
"""Gets job executions and their associated job_type id, name, and version
:param request: the HTTP GET request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
"""
started = rest_util.parse_timestamp(request, 'started', required=False)
ended = rest_util.parse_timestamp(request, 'ended', required=False)
rest_util.check_time_range(started, ended)
job_status = rest_util.parse_string(request, 'status', required=False)
job_type_ids = rest_util.parse_int_list(request, 'job_type_id', required=False)
job_type_names = rest_util.parse_string_list(request, 'job_type_name', required=False)
job_type_categories = rest_util.parse_string_list(request, 'job_type_category', required=False)
node_ids = rest_util.parse_int_list(request, 'node_id', required=False)
order = rest_util.parse_string_list(request, 'order', required=False)
job_exes = JobExecution.objects.get_exes(started, ended, job_status, job_type_ids, job_type_names,
job_type_categories, node_ids, order)
page = self.paginate_queryset(job_exes)
serializer = self.get_serializer(page, many=True)
return self.get_paginated_response(serializer.data)
def list(self, request):
"""Retrieves the product updates for a given time range and returns it in JSON form
:param request: the HTTP GET request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
"""
started = rest_util.parse_timestamp(request, u'started', required=False)
ended = rest_util.parse_timestamp(request, u'ended', required=False)
rest_util.check_time_range(started, ended)
job_type_ids = rest_util.parse_int_list(request, u'job_type_id', required=False)
job_type_names = rest_util.parse_string_list(request, u'job_type_name', required=False)
job_type_categories = rest_util.parse_string_list(request, u'job_type_category', required=False)
is_operational = rest_util.parse_bool(request, u'is_operational', required=False)
file_name = rest_util.parse_string(request, u'file_name', required=False)
order = rest_util.parse_string_list(request, u'order', required=False)
products = ProductFile.objects.get_products(started, ended, job_type_ids, job_type_names, job_type_categories,
is_operational, file_name, order)
page = self.paginate_queryset(products)
ProductFile.objects.populate_source_ancestors(page)
serializer = self.get_serializer(page, many=True)
return self.get_paginated_response(serializer.data)
def get(self, request):
"""Retrieves the job updates for a given time range and returns it in JSON form
:param request: the HTTP GET request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
"""
started = rest_util.parse_timestamp(request, 'started', required=False)
ended = rest_util.parse_timestamp(request, 'ended', required=False)
rest_util.check_time_range(started, ended)
job_status = rest_util.parse_string(request, 'status', required=False)
job_type_ids = rest_util.parse_int_list(request, 'job_type_id', required=False)
job_type_names = rest_util.parse_string_list(request, 'job_type_name', required=False)
job_type_categories = rest_util.parse_string_list(request, 'job_type_category', required=False)
order = rest_util.parse_string_list(request, 'order', required=False)
jobs = Job.objects.get_job_updates(started, ended, job_status, job_type_ids, job_type_names,
job_type_categories, order)
page = self.paginate_queryset(jobs)
Job.objects.populate_input_files(page)
serializer = self.get_serializer(page, many=True)
return self.get_paginated_response(serializer.data)
def get(self, request):
'''Retrieves the product updates for a given time range and returns it in JSON form
:param request: the HTTP GET request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
'''
started = rest_util.parse_timestamp(request, u'started', required=False)
ended = rest_util.parse_timestamp(request, u'ended', required=False)
rest_util.check_time_range(started, ended)
job_type_ids = rest_util.parse_int_list(request, u'job_type_id', required=False)
job_type_names = rest_util.parse_string_list(request, u'job_type_name', required=False)
job_type_categories = rest_util.parse_string_list(request, u'job_type_category', required=False)
is_operational = rest_util.parse_bool(request, u'is_operational', required=False)
file_name = rest_util.parse_string(request, u'file_name', required=False)
order = rest_util.parse_string_list(request, u'order', required=False)
products = ProductFile.objects.get_products(started, ended, job_type_ids, job_type_names, job_type_categories,
is_operational, file_name, order)
page = rest_util.perform_paging(request, products)
ProductFile.objects.populate_source_ancestors(page)
serializer = ProductFileUpdateListSerializer(page, context={'request': request})
return Response(serializer.data, status=status.HTTP_200_OK)
def patch(self, request, job_id):
"""Modify job info with a subset of fields
:param request: the HTTP GET request
:type request: :class:`rest_framework.request.Request`
:param job_id: The ID for the job.
:type job_id: int encoded as a str
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
"""
# Validate that no extra fields are included
rest_util.check_update(request, ['status'])
# Validate JSON
status_code = rest_util.parse_string(request, 'status')
if status_code != 'CANCELED':
raise rest_util.BadParameter('Invalid or read-only status. Allowed values: CANCELED')
try:
Queue.objects.handle_job_cancellation(job_id, datetime.utcnow())
job = Job.objects.get_details(job_id)
except (Job.DoesNotExist, JobExecution.DoesNotExist):
raise Http404
serializer = self.get_serializer(job)
return Response(serializer.data)
def test_parse_string_optional(self):
'''Tests parsing an optional string parameter that is missing.'''
request = MagicMock(Request)
request.QUERY_PARAMS = QueryDict('', mutable=True)
request.QUERY_PARAMS.update({
'test': 'value1',
})
self.assertIsNone(rest_util.parse_string(request, 'test2', required=False))
def test_parse_string_post(self):
'''Tests parsing a required string parameter that is provided via POST.'''
request = MagicMock(Request)
request.DATA = QueryDict('', mutable=True)
request.DATA.update({
'test': 'value1',
})
self.assertEqual(rest_util.parse_string(request, 'test'), 'value1')
def test_parse_string(self):
"""Tests parsing a required string parameter that is provided via GET."""
request = MagicMock(Request)
request.query_params = QueryDict('', mutable=True)
request.query_params.update({
'test': 'value1',
})
self.assertEqual(rest_util.parse_string(request, 'test'), 'value1')
def test_parse_string_default(self):
'''Tests parsing an optional string parameter that is provided via default value.'''
request = MagicMock(Request)
request.QUERY_PARAMS = QueryDict('', mutable=True)
request.QUERY_PARAMS.update({
'test': 'value1',
})
self.assertEqual(rest_util.parse_string(request, 'test2', 'value2'), 'value2')
def test_parse_string_accepted_all(self):
'''Tests parsing a string parameter where the value is acceptable.'''
request = MagicMock(Request)
request.QUERY_PARAMS = QueryDict('', mutable=True)
request.QUERY_PARAMS.update({
'test': 'value1',
})
self.assertEqual(rest_util.parse_string(request, 'test', accepted_values=['value1']), 'value1')
def post(self, request):
'''Creates a new Strike process and returns its ID in JSON form
:param request: the HTTP POST request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
'''
name = rest_util.parse_string(request, 'name')
title = rest_util.parse_string(request, 'title', required=False)
description = rest_util.parse_string(request, 'description', required=False)
configuration = rest_util.parse_dict(request, 'configuration')
try:
strike = Strike.objects.create_strike_process(name, title, description, configuration)
except InvalidStrikeConfiguration:
raise BadParameter('Configuration failed to validate.')
return Response({'strike_id': strike.id})
def list(self, request):
"""Retrieves the list of all ingests and returns it in JSON form
:param request: the HTTP GET request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
"""
started = rest_util.parse_timestamp(request, 'started', required=False)
ended = rest_util.parse_timestamp(request, 'ended', required=False)
rest_util.check_time_range(started, ended)
ingest_status = rest_util.parse_string(request, 'status', required=False)
file_name = rest_util.parse_string(request, 'file_name', required=False)
order = rest_util.parse_string_list(request, 'order', required=False)
ingests = Ingest.objects.get_ingests(started, ended, ingest_status, file_name, order)
page = self.paginate_queryset(ingests)
serializer = self.get_serializer(page, many=True)
return self.get_paginated_response(serializer.data)
def create(self, request):
"""Creates a new batch and returns a link to the detail URL
:param request: the HTTP POST request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
"""
recipe_type_id = rest_util.parse_int(request, 'recipe_type_id')
title = rest_util.parse_string(request, 'title', required=False)
description = rest_util.parse_string(request, 'description', required=False)
# Make sure the recipe type exists
try:
recipe_type = RecipeType.objects.get(pk=recipe_type_id)
except RecipeType.DoesNotExist:
raise BadParameter('Unknown recipe type: %i' % recipe_type_id)
# Validate the batch definition
definition_dict = rest_util.parse_dict(request, 'definition')
definition = None
try:
if definition_dict:
definition = BatchDefinition(definition_dict)
except InvalidDefinition as ex:
raise BadParameter('Batch definition invalid: %s' % unicode(ex))
# Create the batch
batch = Batch.objects.create_batch(recipe_type, definition, title=title, description=description)
# Fetch the full batch with details
try:
batch = Batch.objects.get_details(batch.id)
except Batch.DoesNotExist:
raise Http404
url = reverse('batch_details_view', args=[batch.id], request=request)
serializer = BatchDetailsSerializer(batch)
return Response(serializer.data, status=status.HTTP_201_CREATED, headers=dict(location=url))
def post(self, request):
'''Validates a new recipe type and returns any warnings discovered
:param request: the HTTP POST request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
'''
name = rest_util.parse_string(request, u'name')
version = rest_util.parse_string(request, u'version')
description = rest_util.parse_string(request, u'description', '')
definition = rest_util.parse_dict(request, u'definition')
try:
recipe_def = RecipeDefinition(definition)
warnings = RecipeType.objects.validate_recipe_type(name, version, description, recipe_def)
except InvalidDefinition as ex:
logger.exception('Unable to validate new recipe type: %s', name)
raise BadParameter(unicode(ex))
results = [{'id': w.key, 'details': w.details} for w in warnings]
return Response({u'warnings': results}, status=status.HTTP_200_OK)
def post(self, request):
"""Creates a new error and returns a link to the info URL
:param request: the HTTP POST request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
"""
name = rest_util.parse_string(request, 'name')
title = rest_util.parse_string(request, 'title')
description = rest_util.parse_string(request, 'description')
category = rest_util.parse_string(request, 'category', accepted_values=[c for c, _ in Error.CATEGORIES])
# Do not allow the creation of SYSTEM level errors
if category == 'SYSTEM':
raise BadParameter('System level errors cannot be created.')
error = Error.objects.create_error(name, title, description, category)
serializer = ErrorDetailsSerializer(error)
error_url = urlresolvers.reverse('error_details_view', args=[error.id])
return Response(serializer.data, status=status.HTTP_201_CREATED, headers=dict(location=error_url))
def post(self, request):
'''Creates a new recipe type and returns a link to the detail URL
:param request: the HTTP POST request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
'''
name = rest_util.parse_string(request, u'name')
version = rest_util.parse_string(request, u'version')
title = rest_util.parse_string(request, u'title')
description = rest_util.parse_string(request, u'description', '')
definition = rest_util.parse_dict(request, u'definition')
try:
recipe_def = RecipeDefinition(definition)
recipe_type = RecipeType.objects.create_recipe_type(name, version, title, description, recipe_def, None)
except InvalidDefinition:
logger.exception('Unable to create new recipe type: %s', name)
raise BadParameter('Invalid recipe type definition')
url = urlresolvers.reverse('recipe_type_details_view', args=[recipe_type.id])
return Response({u'id': recipe_type.id}, status=status.HTTP_201_CREATED, headers=dict(location=url))
def post(self, request):
'''Creates a new error and returns a link to the info URL
:param request: the HTTP POST request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
'''
name = rest_util.parse_string(request, u'name')
title = rest_util.parse_string(request, u'title')
description = rest_util.parse_string(request, u'description')
category = rest_util.parse_string(request, u'category')
if category not in map(lambda x: x[1], Error.CATEGORIES):
return Response(u'category is not valid', status=status.HTTP_400_BAD_REQUEST)
try:
error_id = Error.objects.create_error(name, title, description, category).id
except InvalidData:
return Response('Unable to create error.', status=status.HTTP_400_BAD_REQUEST)
error_url = urlresolvers.reverse('error_details_view', args=[error_id])
return Response({u'error_id': error_id}, status=status.HTTP_201_CREATED, headers=dict(location=error_url))
def patch(self, request, name, version):
"""Edits an existing seed job type and returns the updated details
:param request: the HTTP PATCH request
:type request: :class:`rest_framework.request.Request`
:param job_type_id: The ID for the job type.
:type job_type_id: int encoded as a str
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
"""
auto_update = rest_util.parse_bool(request, 'auto_update', required=False, default_value=True)
icon_code = rest_util.parse_string(request, 'icon_code', required=False)
is_published = rest_util.parse_string(request, 'is_published', required=False)
is_active = rest_util.parse_bool(request, 'is_active', required=False)
is_paused = rest_util.parse_bool(request, 'is_paused', required=False)
max_scheduled = rest_util.parse_int(request, 'max_scheduled', required=False)
docker_image = rest_util.parse_string(request, 'docker_image', required=False)
# Validate the manifest
manifest_dict = rest_util.parse_dict(request, 'manifest', required=False)
manifest = None
if manifest_dict:
try:
manifest = SeedManifest(manifest_dict, do_validate=True)
except InvalidSeedManifestDefinition as ex:
message = 'Seed Manifest invalid'
logger.exception(message)
raise BadParameter('%s: %s' % (message, unicode(ex)))
# validate manifest name/version matches job type
manifest_name = manifest.get_name()
if name != manifest_name:
raise BadParameter('Manifest name %s does not match current Job Type name %s.' % (manifest_name, name))
manifest_version = manifest.get_job_version()
if manifest_version != version:
raise BadParameter('Manifest version %s does not match current Job Type version %s.' % (manifest_version, version))
# Validate the job configuration and pull out secrets
configuration_dict = rest_util.parse_dict(request, 'configuration', required=False)
configuration = None
try:
if configuration_dict:
configuration = JobConfigurationV6(configuration_dict).get_configuration()
except InvalidJobConfiguration as ex:
raise BadParameter('Job type configuration invalid: %s' % unicode(ex))
# Fetch the current job type model
try:
job_type = JobType.objects.get(name=name, version=version)
except JobType.DoesNotExist:
raise Http404
# Check for invalid fields
fields = {'icon_code', 'is_published', 'is_active', 'is_paused', 'max_scheduled', 'configuration', 'manifest',
'docker_image', 'auto_update'}
for key, value in request.data.iteritems():
if key not in fields:
raise BadParameter('%s is not a valid field. Valid fields are: %s' % (key, fields))
try:
with transaction.atomic():
# Edit the job type
validation = JobType.objects.edit_job_type_v6(job_type_id=job_type.id, manifest=manifest, is_published=is_published,
docker_image=docker_image, icon_code=icon_code, is_active=is_active,
is_paused=is_paused, max_scheduled=max_scheduled,
configuration=configuration, auto_update=auto_update)
except (InvalidJobField, InvalidSecretsConfiguration, ValueError,
InvalidJobConfiguration, InvalidInterfaceDefinition) as ex:
logger.exception('Unable to update job type: %i', job_type.id)
raise BadParameter(unicode(ex))
resp_dict = {'is_valid': validation.is_valid, 'errors': [e.to_dict() for e in validation.errors],
'warnings': [w.to_dict() for w in validation.warnings]}
return Response(resp_dict)
def create(self, request):
"""Creates a new job type and returns a link to the detail URL
:param request: the HTTP POST request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
"""
name = rest_util.parse_string(request, 'name')
version = rest_util.parse_string(request, 'version')
# Validate the job interface
interface_dict = rest_util.parse_dict(request, 'interface')
interface = None
try:
if interface_dict:
interface = JobInterface(interface_dict)
except InvalidInterfaceDefinition as ex:
raise BadParameter('Job type interface invalid: %s' % unicode(ex))
# Validate the error mapping
error_dict = rest_util.parse_dict(request,
'error_mapping',
required=False)
error_mapping = None
try:
if error_dict:
error_mapping = ErrorInterface(error_dict)
error_mapping.validate()
except InvalidInterfaceDefinition as ex:
raise BadParameter('Job type error mapping invalid: %s' %
unicode(ex))
# Check for optional trigger rule parameters
trigger_rule_dict = rest_util.parse_dict(request,
'trigger_rule',
required=False)
if (('type' in trigger_rule_dict
and 'configuration' not in trigger_rule_dict)
or ('type' not in trigger_rule_dict
and 'configuration' in trigger_rule_dict)):
raise BadParameter(
'Trigger type and configuration are required together.')
is_active = trigger_rule_dict[
'is_active'] if 'is_active' in trigger_rule_dict else True
# Attempt to look up the trigger handler for the type
rule_handler = None
if trigger_rule_dict and 'type' in trigger_rule_dict:
try:
rule_handler = trigger_handler.get_trigger_rule_handler(
trigger_rule_dict['type'])
except InvalidTriggerType as ex:
logger.exception('Invalid trigger type for new job type: %s',
name)
raise BadParameter(unicode(ex))
# Extract the fields that should be updated as keyword arguments
extra_fields = {}
base_fields = {
'name', 'version', 'interface', 'trigger_rule', 'error_mapping'
}
for key, value in request.data.iteritems():
if key not in base_fields and key not in JobType.UNEDITABLE_FIELDS:
extra_fields[key] = value
try:
with transaction.atomic():
# Attempt to create the trigger rule
trigger_rule = None
if rule_handler and 'configuration' in trigger_rule_dict:
trigger_rule = rule_handler.create_trigger_rule(
trigger_rule_dict['configuration'], name, is_active)
# Create the job type
job_type = JobType.objects.create_job_type(
name, version, interface, trigger_rule, error_mapping,
**extra_fields)
except (InvalidJobField, InvalidTriggerType, InvalidTriggerRule,
InvalidConnection, ValueError) as ex:
logger.exception('Unable to create new job type: %s', name)
raise BadParameter(unicode(ex))
# Fetch the full job type with details
try:
job_type = JobType.objects.get_details(job_type.id)
except JobType.DoesNotExist:
raise Http404
url = urlresolvers.reverse('job_type_details_view', args=[job_type.id])
serializer = JobTypeDetailsSerializer(job_type)
return Response(serializer.data,
status=status.HTTP_201_CREATED,
headers=dict(location=url))
def patch(self, request, recipe_type_id):
"""Edits an existing recipe type and returns the updated details
:param request: the HTTP PATCH request
:type request: :class:`rest_framework.request.Request`
:param recipe_type_id: The ID for the recipe type.
:type recipe_type_id: int encoded as a str
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
"""
title = rest_util.parse_string(request, 'title', required=False)
description = rest_util.parse_string(request,
'description',
required=False)
definition_dict = rest_util.parse_dict(request,
'definition',
required=False)
# Check for optional trigger rule parameters
trigger_rule_dict = rest_util.parse_dict(request,
'trigger_rule',
required=False)
if (('type' in trigger_rule_dict
and 'configuration' not in trigger_rule_dict)
or ('type' not in trigger_rule_dict
and 'configuration' in trigger_rule_dict)):
raise BadParameter(
'Trigger type and configuration are required together.')
is_active = trigger_rule_dict[
'is_active'] if 'is_active' in trigger_rule_dict else True
remove_trigger_rule = rest_util.has_params(
request, 'trigger_rule') and not trigger_rule_dict
# Fetch the current recipe type model
try:
recipe_type = RecipeType.objects.select_related(
'trigger_rule').get(pk=recipe_type_id)
except RecipeType.DoesNotExist:
raise Http404
# Attempt to look up the trigger handler for the type
rule_handler = None
if trigger_rule_dict and 'type' in trigger_rule_dict:
try:
rule_handler = trigger_handler.get_trigger_rule_handler(
trigger_rule_dict['type'])
except InvalidTriggerType as ex:
logger.exception('Invalid trigger type for recipe type: %i',
recipe_type_id)
raise BadParameter(unicode(ex))
try:
with transaction.atomic():
# Validate the recipe definition
recipe_def = None
if definition_dict:
recipe_def = RecipeDefinitionSunset.create(definition_dict)
# Attempt to create the trigger rule
trigger_rule = None
if rule_handler and 'configuration' in trigger_rule_dict:
trigger_rule = rule_handler.create_trigger_rule(
trigger_rule_dict['configuration'], recipe_type.name,
is_active)
# Update the active state separately if that is only given trigger field
if not trigger_rule and recipe_type.trigger_rule and 'is_active' in trigger_rule_dict:
recipe_type.trigger_rule.is_active = is_active
recipe_type.trigger_rule.save()
# Edit the recipe type
RecipeType.objects.edit_recipe_type(recipe_type_id, title,
description, recipe_def,
trigger_rule,
remove_trigger_rule)
except (InvalidDefinition, InvalidTriggerType, InvalidTriggerRule,
InvalidRecipeConnection) as ex:
logger.exception('Unable to update recipe type: %i',
recipe_type_id)
raise BadParameter(unicode(ex))
# Fetch the full recipe type with details
try:
recipe_type = RecipeType.objects.get_details(recipe_type_id)
except RecipeType.DoesNotExist:
raise Http404
serializer = self.get_serializer(recipe_type)
return Response(serializer.data)
def post(self, request):
"""Validates a new recipe type and returns any warnings discovered
:param request: the HTTP POST request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
"""
name = rest_util.parse_string(request, 'name')
version = rest_util.parse_string(request, 'version')
title = rest_util.parse_string(request, 'title', default_value=name)
description = rest_util.parse_string(request,
'description',
required=False)
definition_dict = rest_util.parse_dict(request, 'definition')
# Check for optional trigger rule parameters
trigger_rule_dict = rest_util.parse_dict(request,
'trigger_rule',
required=False)
if (('type' in trigger_rule_dict
and 'configuration' not in trigger_rule_dict)
or ('type' not in trigger_rule_dict
and 'configuration' in trigger_rule_dict)):
raise BadParameter(
'Trigger type and configuration are required together.')
# Attempt to look up the trigger handler for the type
rule_handler = None
if trigger_rule_dict and 'type' in trigger_rule_dict:
try:
rule_handler = trigger_handler.get_trigger_rule_handler(
trigger_rule_dict['type'])
except InvalidTriggerType as ex:
logger.exception(
'Invalid trigger type for recipe validation: %s', name)
raise BadParameter(unicode(ex))
# Attempt to look up the trigger rule configuration
trigger_config = None
if rule_handler and 'configuration' in trigger_rule_dict:
try:
trigger_config = rule_handler.create_configuration(
trigger_rule_dict['configuration'])
except InvalidTriggerRule as ex:
logger.exception(
'Invalid trigger rule configuration for recipe validation: %s',
name)
raise BadParameter(unicode(ex))
# Validate the recipe definition
try:
recipe_def = RecipeDefinition(definition_dict)
warnings = RecipeType.objects.validate_recipe_type(
name, title, version, description, recipe_def, trigger_config)
except (InvalidDefinition, InvalidTriggerType, InvalidTriggerRule,
InvalidRecipeConnection) as ex:
logger.exception('Unable to validate new recipe type: %s', name)
raise BadParameter(unicode(ex))
results = [{'id': w.key, 'details': w.details} for w in warnings]
return Response({'warnings': results})
def post(self, request):
"""Creates a new recipe type and returns a link to the detail URL
:param request: the HTTP POST request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
"""
name = rest_util.parse_string(request, 'name')
version = rest_util.parse_string(request, 'version')
title = rest_util.parse_string(request, 'title', default_value=name)
description = rest_util.parse_string(request,
'description',
required=False)
definition_dict = rest_util.parse_dict(request, 'definition')
# Check for optional trigger rule parameters
trigger_rule_dict = rest_util.parse_dict(request,
'trigger_rule',
required=False)
if (('type' in trigger_rule_dict
and 'configuration' not in trigger_rule_dict)
or ('type' not in trigger_rule_dict
and 'configuration' in trigger_rule_dict)):
raise BadParameter(
'Trigger type and configuration are required together.')
is_active = trigger_rule_dict[
'is_active'] if 'is_active' in trigger_rule_dict else True
# Attempt to look up the trigger handler for the type
rule_handler = None
if trigger_rule_dict and 'type' in trigger_rule_dict:
try:
rule_handler = trigger_handler.get_trigger_rule_handler(
trigger_rule_dict['type'])
except InvalidTriggerType as ex:
logger.exception(
'Invalid trigger type for new recipe type: %s', name)
raise BadParameter(unicode(ex))
try:
with transaction.atomic():
# Validate the recipe definition
recipe_def = RecipeDefinition(definition_dict)
# Attempt to create the trigger rule
trigger_rule = None
if rule_handler and 'configuration' in trigger_rule_dict:
trigger_rule = rule_handler.create_trigger_rule(
trigger_rule_dict['configuration'], name, is_active)
# Create the recipe type
recipe_type = RecipeType.objects.create_recipe_type(
name, version, title, description, recipe_def,
trigger_rule)
except (InvalidDefinition, InvalidTriggerType, InvalidTriggerRule,
InvalidRecipeConnection) as ex:
logger.exception('Unable to create new recipe type: %s', name)
raise BadParameter(unicode(ex))
# Fetch the full recipe type with details
try:
recipe_type = RecipeType.objects.get_details(recipe_type.id)
except RecipeType.DoesNotExist:
raise Http404
url = reverse('recipe_type_details_view',
args=[recipe_type.id],
request=request)
serializer = RecipeTypeDetailsSerializer(recipe_type)
return Response(serializer.data,
status=status.HTTP_201_CREATED,
headers=dict(location=url))
def create_v6(self, request):
"""Creates or edits a Seed job type and returns a link to the detail URL
:param request: the HTTP POST request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
"""
# Optional icon code value
icon_code = rest_util.parse_string(request, 'icon_code', required=False)
# Optional is published value
is_published = rest_util.parse_string(request, 'is_published', required=False)
# Optional max scheduled value
max_scheduled = rest_util.parse_int(request, 'max_scheduled', required=False)
# Require docker image value
docker_image = rest_util.parse_string(request, 'docker_image', required=True)
# Validate the job interface / manifest
manifest_dict = rest_util.parse_dict(request, 'manifest', required=True)
# If editing an existing job type, automatically update recipes containing said job type
auto_update = rest_util.parse_bool(request, 'auto_update', required=False, default_value=True)
# Optional setting job type active if editing existing job
is_active = rest_util.parse_bool(request, 'is_active', required=False)
# Optional setting job type to paused if editing an existing job
is_paused = rest_util.parse_bool(request, 'is_paused', required=False)
manifest = None
try:
manifest = SeedManifest(manifest_dict, do_validate=True)
except InvalidSeedManifestDefinition as ex:
message = 'Seed Manifest invalid'
logger.exception(message)
raise BadParameter('%s: %s' % (message, unicode(ex)))
# Validate the job configuration and pull out secrets
configuration_dict = rest_util.parse_dict(request, 'configuration', required=False)
configuration = None
if configuration_dict:
try:
configuration = JobConfigurationV6(configuration_dict, do_validate=True).get_configuration()
except InvalidJobConfiguration as ex:
message = 'Job type configuration invalid'
logger.exception(message)
raise BadParameter('%s: %s' % (message, unicode(ex)))
# Check for invalid fields
fields = {'icon_code', 'is_published', 'max_scheduled', 'docker_image', 'configuration', 'manifest',
'auto_update', 'is_active', 'is_paused'}
for key, value in request.data.iteritems():
if key not in fields:
raise BadParameter('%s is not a valid field. Valid fields are: %s' % (key, fields))
name = manifest_dict['job']['name']
version = manifest_dict['job']['jobVersion']
if name == 'validation':
logger.exception('Unable to create job type named "validation"')
raise BadParameter(unicode('Unable to create job type named "validation"'))
existing_job_type = JobType.objects.filter(name=name, version=version).first()
if not existing_job_type:
try:
# Create the job type
job_type = JobType.objects.create_job_type_v6(icon_code=icon_code,
is_published=is_published,
max_scheduled=max_scheduled,
docker_image=docker_image,
manifest=manifest,
configuration=configuration)
except (InvalidJobField, InvalidSecretsConfiguration, ValueError) as ex:
message = 'Unable to create new job type'
logger.exception(message)
raise BadParameter('%s: %s' % (message, unicode(ex)))
except InvalidSeedManifestDefinition as ex:
message = 'Job type manifest invalid'
logger.exception(message)
raise BadParameter('%s: %s' % (message, unicode(ex)))
except InvalidJobConfiguration as ex:
message = 'Job type configuration invalid'
logger.exception(message)
raise BadParameter('%s: %s' % (message, unicode(ex)))
# Fetch the full job type with details
try:
job_type = JobType.objects.get_details_v6(name=name, version=version)
except JobType.DoesNotExist:
raise Http404
url = reverse('job_type_details_view', args=[job_type.name, job_type.version], request=request)
serializer = JobTypeDetailsSerializerV6(job_type)
return Response(serializer.data, status=status.HTTP_201_CREATED, headers=dict(location=url))
else:
try:
validation = JobType.objects.edit_job_type_v6(job_type_id=existing_job_type.id, manifest=manifest,
docker_image=docker_image, icon_code=icon_code, is_active=is_active,
is_paused=is_paused, max_scheduled=max_scheduled,
is_published=is_published, configuration=configuration,
auto_update=auto_update)
except (InvalidJobField, InvalidSecretsConfiguration, ValueError, InvalidInterfaceDefinition) as ex:
logger.exception('Unable to update job type: %i', existing_job_type.id)
raise BadParameter(unicode(ex))
except InvalidSeedManifestDefinition as ex:
message = 'Job type manifest invalid'
logger.exception(message)
raise BadParameter('%s: %s' % (message, unicode(ex)))
except InvalidJobConfiguration as ex:
message = 'Job type configuration invalid'
logger.exception(message)
raise BadParameter('%s: %s' % (message, unicode(ex)))
resp_dict = {'is_valid': validation.is_valid, 'errors': [e.to_dict() for e in validation.errors],
'warnings': [w.to_dict() for w in validation.warnings]}
return Response(resp_dict)
def create_all_v6(self, request):
"""Creates or edits a dataset - including the dataset members - and returns a link to the detail URL"""
title = rest_util.parse_string(request, 'title', required=False)
description = rest_util.parse_string(request,
'description',
required=False)
definition = rest_util.parse_dict(request, 'definition', required=True)
template = rest_util.parse_dict(request,
'data_template',
required=False)
dry_run = rest_util.parse_bool(request, 'dry_run', default_value=False)
# file filters
data_started = rest_util.parse_timestamp(request,
'data_started',
required=False)
data_ended = rest_util.parse_timestamp(request,
'data_ended',
required=False)
rest_util.check_time_range(data_started, data_ended)
source_started = rest_util.parse_timestamp(request,
'source_started',
required=False)
source_ended = rest_util.parse_timestamp(request,
'source_ended',
required=False)
rest_util.check_time_range(source_started, source_ended)
source_sensor_classes = rest_util.parse_string_list(
request, 'source_sensor_class', required=False)
source_sensors = rest_util.parse_string_list(request,
'source_sensor',
required=False)
source_collections = rest_util.parse_string_list(request,
'source_collection',
required=False)
source_tasks = rest_util.parse_string_list(request,
'source_task',
required=False)
mod_started = rest_util.parse_timestamp(request,
'modified_started',
required=False)
mod_ended = rest_util.parse_timestamp(request,
'modified_ended',
required=False)
rest_util.check_time_range(mod_started, mod_ended)
job_type_ids = rest_util.parse_int_list(request,
'job_type_id',
required=False)
job_type_names = rest_util.parse_string_list(request,
'job_type_name',
required=False)
job_ids = rest_util.parse_int_list(request, 'job_id', required=False)
file_names = rest_util.parse_string_list(request,
'file_name',
required=False)
job_outputs = rest_util.parse_string_list(request,
'job_output',
required=False)
recipe_ids = rest_util.parse_int_list(request,
'recipe_id',
required=False)
recipe_type_ids = rest_util.parse_int_list(request,
'recipe_type_id',
required=False)
recipe_nodes = rest_util.parse_string_list(request,
'recipe_node',
required=False)
batch_ids = rest_util.parse_int_list(request,
'batch_id',
required=False)
order = rest_util.parse_string_list(request, 'order', required=False)
data = rest_util.parse_dict_list(request, 'data', required=False)
data_list = []
# validate the definition & create the dataset
try:
dataset_def = DataSetDefinitionV6(
definition=definition, do_validate=True).get_definition()
except InvalidDataSetDefinition as ex:
message = 'DataSet definition is invalid'
logger.exception(message)
raise BadParameter('%s: %s' % (message, unicode(ex)))
try:
dataset = DataSet.objects.create_dataset_v6(
dataset_def, title=title, description=description)
except Exception as ex:
message = 'Unable to create new dataset'
logger.exception(message)
raise BadParameter('%s: %s' % (message, unicode(ex)))
try:
dataset = DataSet.objects.get_details_v6(dataset.id)
except DataSet.DoesNotExist:
raise Http404
if not data and not template:
url = reverse('dataset_details_view',
args=[dataset.id],
request=request)
serializer = DataSetDetailsSerializerV6(dataset)
return Response(serializer.data,
status=status.HTTP_201_CREATED,
headers=dict(location=url))
# Try and find the data
if data:
try:
for d in data:
data = DataV6(data=d, do_validate=True).get_data()
data_list.append(data)
except InvalidData as ex:
message = 'Data is invalid'
logger.exception(message)
raise BadParameter('%s: %s' % (message, unicode(ex)))
elif template:
try:
data_list = DataSetMember.objects.build_data_list(
template=template,
data_started=data_started,
data_ended=data_ended,
source_started=source_started,
source_ended=source_ended,
source_sensor_classes=source_sensor_classes,
source_sensors=source_sensors,
source_collections=source_collections,
source_tasks=source_tasks,
mod_started=mod_started,
mod_ended=mod_ended,
job_type_ids=job_type_ids,
job_type_names=job_type_names,
job_ids=job_ids,
file_names=file_names,
job_outputs=job_outputs,
recipe_ids=recipe_ids,
recipe_type_ids=recipe_type_ids,
recipe_nodes=recipe_nodes,
batch_ids=batch_ids,
order=order)
except InvalidData as ex:
message = 'Data is invalid'
logger.exception(message)
raise BadParameter('%s: %s' % (message, unicode(ex)))
if not data_list:
resp_dict = {
'No Results':
'No files found from filters and/or no data provided'
}
return Response(resp_dict)
validation = DataSetMember.objects.validate_data_list(
dataset_def=dataset_def, data_list=data_list)
members = []
if validation.is_valid and not dry_run:
members = DataSetMember.objects.create_dataset_members(
dataset=dataset, data_list=data_list)
dataset = DataSet.objects.get(id=dataset.id)
serializer = DataSetDetailsSerializerV6(dataset)
url = reverse('dataset_details_view',
args=[dataset.id],
request=request)
return Response(serializer.data,
status=status.HTTP_201_CREATED,
headers=dict(location=url))
elif not validation.is_valid:
raise BadParameter('%s: %s' %
('Error(s) validating data against dataset',
[e.to_dict() for e in validation.errors]))
resp_dict = []
for dl in data_list:
resp_dict.append(convert_data_to_v6_json(dl).get_dict())
return Response(resp_dict)
def _create_v6(self, request):
"""Creates a new recipe type and returns a link to the detail URL
:param request: the HTTP POST request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
"""
title = rest_util.parse_string(request, 'title', required=True)
description = rest_util.parse_string(request,
'description',
required=False)
definition_dict = rest_util.parse_dict(request,
'definition',
required=True)
basename = title_to_basename(title)
if basename == 'validation':
logger.exception('Unable to create recipe type named "validation"')
raise BadParameter(
unicode('Unable to create recipe type named "validation"'))
existing_recipes = RecipeType.objects.filter(name=basename)
if existing_recipes.count() > 0:
logger.exception(
'Existing recipe types found for %s - will not re-create.',
basename)
raise BadParameter(
unicode(
'Existing recipe types found for %s - will not re-create. Please change the title or patch the existing recipe type.'
% basename))
name = title_to_name(self.queryset, title)
try:
with transaction.atomic():
# Validate the recipe definition
recipe_def = RecipeDefinitionV6(
definition=definition_dict,
do_validate=True).get_definition()
# Create the recipe type
recipe_type = RecipeType.objects.create_recipe_type_v6(
name, title, description, recipe_def)
except InvalidDefinition as ex:
logger.exception('Unable to create new recipe type: %s', name)
raise BadParameter(unicode(ex))
# Fetch the full recipe type with details
try:
recipe_type = RecipeType.objects.get_details_v6(recipe_type.name)
except RecipeType.DoesNotExist:
raise Http404
url = reverse('recipe_type_details_view',
args=[recipe_type.name],
request=request)
serializer = RecipeTypeDetailsSerializerV6(recipe_type)
return Response(serializer.data,
status=status.HTTP_201_CREATED,
headers=dict(location=url))
def _list_v6(self, request):
"""Retrieves a list of files based on filters and returns it in JSON form
:param request: the HTTP GET request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
"""
countries = rest_util.parse_string_list(request,
'countries',
required=False)
data_started = rest_util.parse_timestamp(request,
'data_started',
required=False)
data_ended = rest_util.parse_timestamp(request,
'data_ended',
required=False)
rest_util.check_time_range(data_started, data_ended)
created_started = rest_util.parse_timestamp(request,
'created_started',
required=False)
created_ended = rest_util.parse_timestamp(request,
'created_ended',
required=False)
rest_util.check_time_range(created_started, created_ended)
source_started = rest_util.parse_timestamp(request,
'source_started',
required=False)
source_ended = rest_util.parse_timestamp(request,
'source_ended',
required=False)
rest_util.check_time_range(source_started, source_ended)
source_sensor_classes = rest_util.parse_string_list(
request, 'source_sensor_class', required=False)
file_type = rest_util.parse_string_list(request,
'file_type',
required=False)
source_sensors = rest_util.parse_string_list(request,
'source_sensor',
required=False)
source_collections = rest_util.parse_string_list(request,
'source_collection',
required=False)
source_tasks = rest_util.parse_string_list(request,
'source_task',
required=False)
mod_started = rest_util.parse_timestamp(request,
'modified_started',
required=False)
mod_ended = rest_util.parse_timestamp(request,
'modified_ended',
required=False)
rest_util.check_time_range(mod_started, mod_ended)
job_type_ids = rest_util.parse_int_list(request,
'job_type_id',
required=False)
job_type_names = rest_util.parse_string_list(request,
'job_type_name',
required=False)
job_ids = rest_util.parse_int_list(request, 'job_id', required=False)
file_names = rest_util.parse_string_list(request,
'file_name',
required=False)
job_outputs = rest_util.parse_string_list(request,
'job_output',
required=False)
recipe_ids = rest_util.parse_int_list(request,
'recipe_id',
required=False)
recipe_type_ids = rest_util.parse_int_list(request,
'recipe_type_id',
required=False)
recipe_nodes = rest_util.parse_string_list(request,
'recipe_node',
required=False)
batch_ids = rest_util.parse_int_list(request,
'batch_id',
required=False)
media_type = rest_util.parse_string_list(request,
'media_type',
required=False)
file_name_search = rest_util.parse_string(request,
'file_name_search',
required=False)
order = rest_util.parse_string_list(request, 'order', required=False)
files = ScaleFile.objects.filter_files(
data_started=data_started,
data_ended=data_ended,
created_started=created_started,
created_ended=created_ended,
source_started=source_started,
source_ended=source_ended,
source_sensor_classes=source_sensor_classes,
source_sensors=source_sensors,
source_collections=source_collections,
source_tasks=source_tasks,
mod_started=mod_started,
mod_ended=mod_ended,
job_type_ids=job_type_ids,
job_type_names=job_type_names,
job_ids=job_ids,
file_names=file_names,
file_name_search=file_name_search,
job_outputs=job_outputs,
recipe_ids=recipe_ids,
recipe_type_ids=recipe_type_ids,
recipe_nodes=recipe_nodes,
batch_ids=batch_ids,
order=order,
countries=countries,
file_type=file_type,
media_type=media_type)
page = self.paginate_queryset(files)
serializer = self.get_serializer(page, many=True)
return self.get_paginated_response(serializer.data)
def post(self, request):
"""Validates a new job type and returns any warnings discovered
:param request: the HTTP POST request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
"""
name = rest_util.parse_string(request, 'name')
version = rest_util.parse_string(request, 'version')
# Validate the job interface
interface_dict = rest_util.parse_dict(request, 'interface')
interface = None
try:
if interface_dict:
interface = JobInterface(interface_dict)
except InvalidInterfaceDefinition as ex:
raise BadParameter('Job type interface invalid: %s' % unicode(ex))
# Validate the error mapping
error_dict = rest_util.parse_dict(request,
'error_mapping',
required=False)
error_mapping = None
try:
if error_dict:
error_mapping = ErrorInterface(error_dict)
except InvalidInterfaceDefinition as ex:
raise BadParameter('Job type error mapping invalid: %s' %
unicode(ex))
# Check for optional trigger rule parameters
trigger_rule_dict = rest_util.parse_dict(request,
'trigger_rule',
required=False)
if (('type' in trigger_rule_dict
and 'configuration' not in trigger_rule_dict)
or ('type' not in trigger_rule_dict
and 'configuration' in trigger_rule_dict)):
raise BadParameter(
'Trigger type and configuration are required together.')
# Attempt to look up the trigger handler for the type
rule_handler = None
if trigger_rule_dict and 'type' in trigger_rule_dict:
try:
rule_handler = trigger_handler.get_trigger_rule_handler(
trigger_rule_dict['type'])
except InvalidTriggerType as ex:
logger.exception('Invalid trigger type for job validation: %s',
name)
raise BadParameter(unicode(ex))
# Attempt to look up the trigger rule configuration
trigger_config = None
if rule_handler and 'configuration' in trigger_rule_dict:
try:
trigger_config = rule_handler.create_configuration(
trigger_rule_dict['configuration'])
except InvalidTriggerRule as ex:
logger.exception(
'Invalid trigger rule configuration for job validation: %s',
name)
raise BadParameter(unicode(ex))
try:
from recipe.configuration.definition.exceptions import InvalidDefinition
except:
logger.exception(
'Failed to import higher level recipe application.')
pass
# Validate the job interface
try:
warnings = JobType.objects.validate_job_type(
name, version, interface, error_mapping, trigger_config)
except (InvalidDefinition, InvalidTriggerType,
InvalidTriggerRule) as ex:
logger.exception('Unable to validate new job type: %s', name)
raise BadParameter(unicode(ex))
results = [{'id': w.key, 'details': w.details} for w in warnings]
return Response({'warnings': results})
def list_impl(self, request, source_id=None):
"""Retrieves the products for a given source file ID and returns them in JSON form
:param request: the HTTP GET request
:type request: :class:`rest_framework.request.Request`
:param source_id: The id of the source
:type source_id: int encoded as a string
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
"""
try:
ScaleFile.objects.get(id=source_id, file_type='SOURCE')
except ScaleFile.DoesNotExist:
raise Http404
started = rest_util.parse_timestamp(request, 'started', required=False)
ended = rest_util.parse_timestamp(request, 'ended', required=False)
rest_util.check_time_range(started, ended)
time_field = rest_util.parse_string(
request,
'time_field',
required=False,
accepted_values=ProductFile.VALID_TIME_FIELDS)
batch_ids = rest_util.parse_int_list(request,
'batch_id',
required=False)
job_type_ids = rest_util.parse_int_list(request,
'job_type_id',
required=False)
job_type_names = rest_util.parse_string_list(request,
'job_type_name',
required=False)
job_type_categories = rest_util.parse_string_list(request,
'job_type_category',
required=False)
job_ids = rest_util.parse_int_list(request, 'job_id', required=False)
is_operational = rest_util.parse_bool(request,
'is_operational',
required=False)
is_published = rest_util.parse_bool(request,
'is_published',
required=False)
file_name = rest_util.parse_string(request,
'file_name',
required=False)
job_output = rest_util.parse_string(request,
'job_output',
required=False)
recipe_ids = rest_util.parse_int_list(request,
'recipe_id',
required=False)
recipe_type_ids = rest_util.parse_int_list(request,
'recipe_type_id',
required=False)
recipe_job = rest_util.parse_string(request,
'recipe_job',
required=False)
order = rest_util.parse_string_list(request, 'order', required=False)
products = SourceFile.objects.get_source_products(
source_id,
started=started,
ended=ended,
time_field=time_field,
batch_ids=batch_ids,
job_type_ids=job_type_ids,
job_type_names=job_type_names,
job_type_categories=job_type_categories,
job_ids=job_ids,
is_operational=is_operational,
is_published=is_published,
file_name=file_name,
job_output=job_output,
recipe_ids=recipe_ids,
recipe_type_ids=recipe_type_ids,
recipe_job=recipe_job,
order=order)
page = self.paginate_queryset(products)
serializer = self.get_serializer(page, many=True)
return self.get_paginated_response(serializer.data)
def get(self, request, recipe_id):
"""Retrieve detailed information about the input files for a recipe
-*-*-
parameters:
- name: recipe_id
in: path
description: The ID of the recipe the file is associated with
required: true
example: 113
- name: started
in: query
description: The start time of a start/end time range
required: false
example: 2016-01-01T00:00:00Z
- name: ended
in: query
description: The end time of a start/end time range
required: false
example: 2016-01-02T00:00:00Z
- name: time_field
in: query
description: 'The database time field to apply `started` and `ended` time filters
[Valid fields: `source`, `data`, `last_modified`]'
required: false
example: source
- name: file_name
in: query
description: The name of a specific file in Scale
required: false
example: some_file_i_need_to_find.zip
- name: recipe_input
in: query
description: The name of the input the file is passed to in a recipe
required: false
example: input_1
responses:
'200':
description: A JSON list of files with metadata
-*-*-
:param request: the HTTP GET request
:type request: :class:`rest_framework.request.Request`
:param recipe_id: The ID for the recipe.
:type recipe_id: int encoded as a str
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
"""
started = rest_util.parse_timestamp(request, 'started', required=False)
ended = rest_util.parse_timestamp(request, 'ended', required=False)
rest_util.check_time_range(started, ended)
time_field = rest_util.parse_string(request, 'time_field', required=False,
accepted_values=ScaleFile.VALID_TIME_FIELDS)
file_name = rest_util.parse_string(request, 'file_name', required=False)
recipe_input = rest_util.parse_string(request, 'recipe_input', required=False)
files = RecipeInputFile.objects.get_recipe_input_files(recipe_id, started=started, ended=ended,
time_field=time_field, file_name=file_name,
recipe_input=recipe_input)
page = self.paginate_queryset(files)
serializer = self.get_serializer(page, many=True)
return self.get_paginated_response(serializer.data)
def list(self, request):
"""Retrieves the product for a given file name and returns it in JSON form
:param request: the HTTP GET request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
"""
started = rest_util.parse_timestamp(request, 'started', required=False)
ended = rest_util.parse_timestamp(request, 'ended', required=False)
rest_util.check_time_range(started, ended)
time_field = rest_util.parse_string(
request,
'time_field',
required=False,
accepted_values=ProductFile.VALID_TIME_FIELDS)
job_type_ids = rest_util.parse_int_list(request,
'job_type_id',
required=False)
job_type_names = rest_util.parse_string_list(request,
'job_type_name',
required=False)
job_type_categories = rest_util.parse_string_list(request,
'job_type_category',
required=False)
job_ids = rest_util.parse_int_list(request, 'job_id', required=False)
is_operational = rest_util.parse_bool(request,
'is_operational',
required=False)
is_published = rest_util.parse_bool(request,
'is_published',
default_value=True)
file_name = rest_util.parse_string(request,
'file_name',
required=False)
job_output = rest_util.parse_string(request,
'job_output',
required=False)
recipe_ids = rest_util.parse_int_list(request,
'recipe_id',
required=False)
recipe_type_ids = rest_util.parse_int_list(request,
'recipe_type_id',
required=False)
recipe_job = rest_util.parse_string(request,
'recipe_job',
required=False)
batch_ids = rest_util.parse_int_list(request,
'batch_id',
required=False)
order = rest_util.parse_string_list(request, 'order', required=False)
products = ProductFile.objects.get_products(
started=started,
ended=ended,
time_field=time_field,
job_type_ids=job_type_ids,
job_type_names=job_type_names,
job_type_categories=job_type_categories,
job_ids=job_ids,
is_operational=is_operational,
is_published=is_published,
file_name=file_name,
job_output=job_output,
recipe_ids=recipe_ids,
recipe_type_ids=recipe_type_ids,
recipe_job=recipe_job,
batch_ids=batch_ids,
order=order,
)
page = self.paginate_queryset(products)
serializer = self.get_serializer(page, many=True)
return self.get_paginated_response(serializer.data)
