def create(self, request): """Creates a new Strike process and returns a link to the detail URL :param request: the HTTP POST request :type request: :class:`rest_framework.request.Request` :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user """ name = rest_util.parse_string(request, 'name') title = rest_util.parse_string(request, 'title', required=False) description = rest_util.parse_string(request, 'description', required=False) configuration = rest_util.parse_dict(request, 'configuration') try: strike = Strike.objects.create_strike(name, title, description, configuration) except InvalidStrikeConfiguration as ex: raise BadParameter('Strike configuration invalid: %s' % unicode(ex)) # Fetch the full strike process with details try: strike = Strike.objects.get_details(strike.id) except Strike.DoesNotExist: raise Http404 serializer = StrikeDetailsSerializer(strike) strike_url = urlresolvers.reverse('strike_details_view', args=[strike.id]) return Response(serializer.data, status=status.HTTP_201_CREATED, headers=dict(location=strike_url))
def post(self, request): """Validates a new Strike process and returns any warnings discovered :param request: the HTTP POST request :type request: :class:`rest_framework.request.Request` :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user """ name = rest_util.parse_string(request, 'name') configuration = rest_util.parse_dict(request, 'configuration') rest_util.parse_string(request, 'title', required=False) rest_util.parse_string(request, 'description', required=False) # Validate the Strike configuration try: config = StrikeConfiguration(configuration) warnings = config.validate() except InvalidStrikeConfiguration as ex: logger.exception('Unable to validate new Strike process: %s', name) raise BadParameter(unicode(ex)) results = [{'id': w.key, 'details': w.details} for w in warnings] return Response({'warnings': results})
def patch(self, request, strike_id): """Edits an existing Strike process and returns the updated details :param request: the HTTP GET request :type request: :class:`rest_framework.request.Request` :param strike_id: The ID of the Strike process :type strike_id: int encoded as a str :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user """ title = rest_util.parse_string(request, 'title', required=False) description = rest_util.parse_string(request, 'description', required=False) configuration = rest_util.parse_dict(request, 'configuration', required=False) try: Strike.objects.edit_strike(strike_id, title, description, configuration) strike = Strike.objects.get_details(strike_id) except Strike.DoesNotExist: raise Http404 except InvalidStrikeConfiguration as ex: logger.exception('Unable to edit Strike process: %s', strike_id) raise BadParameter(unicode(ex)) serializer = self.get_serializer(strike) return Response(serializer.data)
def patch(self, request, workspace_id): """Edits an existing workspace and returns the updated details :param request: the HTTP GET request :type request: :class:`rest_framework.request.Request` :param workspace_id: The ID for the workspace. :type workspace_id: int encoded as a str :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user """ title = rest_util.parse_string(request, "title", required=False) description = rest_util.parse_string(request, "description", required=False) json_config = rest_util.parse_dict(request, "json_config", required=False) base_url = rest_util.parse_string(request, "base_url", required=False) is_active = rest_util.parse_string(request, "is_active", required=False) try: Workspace.objects.edit_workspace(workspace_id, title, description, json_config, base_url, is_active) workspace = Workspace.objects.get_details(workspace_id) except Workspace.DoesNotExist: raise Http404 except InvalidWorkspaceConfiguration as ex: logger.exception("Unable to edit workspace: %s", workspace_id) raise BadParameter(unicode(ex)) serializer = self.get_serializer(workspace) return Response(serializer.data)
def patch(self, request, error_id): """Edits an existing error and returns the updated model :param request: the HTTP PATCH request :type request: :class:`rest_framework.request.Request` :param error_id: The id of the error :type error_id: int encoded as a str :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user """ try: error = Error.objects.get(pk=error_id) except Error.DoesNotExist: raise Http404 # Do not allow editing of SYSTEM level errors if error.category == 'SYSTEM': raise BadParameter('System level errors cannot be edited.') title = rest_util.parse_string(request, 'title', required=False) description = rest_util.parse_string(request, 'description', required=False) category = rest_util.parse_string(request, 'category', required=False, accepted_values=[c for c, _ in Error.CATEGORIES]) # Do not allow editing of SYSTEM level errors if category == 'SYSTEM': raise BadParameter('Errors cannot be changed to system level.') error.title = title or error.title error.description = description or error.description error.category = category or error.category error.save() serializer = self.get_serializer(error) return Response(serializer.data)
def create(self, request): """Creates a new Workspace and returns it in JSON form :param request: the HTTP POST request :type request: :class:`rest_framework.request.Request` :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user """ name = rest_util.parse_string(request, "name") title = rest_util.parse_string(request, "title", required=False) description = rest_util.parse_string(request, "description", required=False) json_config = rest_util.parse_dict(request, "json_config") base_url = rest_util.parse_string(request, "base_url", required=False) is_active = rest_util.parse_bool(request, "is_active", default_value=True, required=False) try: workspace = Workspace.objects.create_workspace(name, title, description, json_config, base_url, is_active) except InvalidWorkspaceConfiguration as ex: logger.exception("Unable to create new workspace: %s", name) raise BadParameter(unicode(ex)) # Fetch the full workspace with details try: workspace = Workspace.objects.get_details(workspace.id) except Workspace.DoesNotExist: raise Http404 serializer = WorkspaceDetailsSerializer(workspace) workspace_url = reverse("workspace_details_view", args=[workspace.id], request=request) return Response(serializer.data, status=status.HTTP_201_CREATED, headers=dict(location=workspace_url))
def post(self, request): """Creates a new recipe type and returns a link to the detail URL :param request: the HTTP POST request :type request: :class:`rest_framework.request.Request` :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user """ name = rest_util.parse_string(request, 'name') version = rest_util.parse_string(request, 'version') title = rest_util.parse_string(request, 'title', default_value=name) description = rest_util.parse_string(request, 'description', required=False) definition_dict = rest_util.parse_dict(request, 'definition') # Check for optional trigger rule parameters trigger_rule_dict = rest_util.parse_dict(request, 'trigger_rule', required=False) if (('type' in trigger_rule_dict and 'configuration' not in trigger_rule_dict) or ('type' not in trigger_rule_dict and 'configuration' in trigger_rule_dict)): raise BadParameter('Trigger type and configuration are required together.') is_active = trigger_rule_dict['is_active'] if 'is_active' in trigger_rule_dict else True # Attempt to look up the trigger handler for the type rule_handler = None if trigger_rule_dict and 'type' in trigger_rule_dict: try: rule_handler = trigger_handler.get_trigger_rule_handler(trigger_rule_dict['type']) except InvalidTriggerType as ex: logger.exception('Invalid trigger type for new recipe type: %s', name) raise BadParameter(unicode(ex)) try: with transaction.atomic(): # Validate the recipe definition recipe_def = RecipeDefinition(definition_dict) # Attempt to create the trigger rule trigger_rule = None if rule_handler and 'configuration' in trigger_rule_dict: trigger_rule = rule_handler.create_trigger_rule(trigger_rule_dict['configuration'], name, is_active) # Create the recipe type recipe_type = RecipeType.objects.create_recipe_type(name, version, title, description, recipe_def, trigger_rule) except (InvalidDefinition, InvalidTriggerType, InvalidTriggerRule, InvalidRecipeConnection) as ex: logger.exception('Unable to create new recipe type: %s', name) raise BadParameter(unicode(ex)) # Fetch the full recipe type with details try: recipe_type = RecipeType.objects.get_details(recipe_type.id) except RecipeType.DoesNotExist: raise Http404 url = urlresolvers.reverse('recipe_type_details_view', args=[recipe_type.id]) serializer = RecipeTypeDetailsSerializer(recipe_type) return Response(serializer.data, status=status.HTTP_201_CREATED, headers=dict(location=url))
def post(self, request): """Validates a new workspace and returns any warnings discovered :param request: the HTTP POST request :type request: :class:`rest_framework.request.Request` :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user """ name = rest_util.parse_string(request, "name") json_config = rest_util.parse_dict(request, "json_config") rest_util.parse_string(request, "title", required=False) rest_util.parse_string(request, "description", required=False) rest_util.parse_string(request, "base_url", required=False) rest_util.parse_string(request, "is_active", required=False) # Validate the workspace configuration try: warnings = Workspace.objects.validate_workspace(name, json_config) except InvalidWorkspaceConfiguration as ex: logger.exception("Unable to validate new workspace: %s", name) raise BadParameter(unicode(ex)) results = [{"id": w.key, "details": w.details} for w in warnings] return Response({"warnings": results})
def post(self, request): """Validates a new recipe type and returns any warnings discovered :param request: the HTTP POST request :type request: :class:`rest_framework.request.Request` :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user """ name = rest_util.parse_string(request, 'name') version = rest_util.parse_string(request, 'version') title = rest_util.parse_string(request, 'title', default_value=name) description = rest_util.parse_string(request, 'description', required=False) definition_dict = rest_util.parse_dict(request, 'definition') # Check for optional trigger rule parameters trigger_rule_dict = rest_util.parse_dict(request, 'trigger_rule', required=False) if (('type' in trigger_rule_dict and 'configuration' not in trigger_rule_dict) or ('type' not in trigger_rule_dict and 'configuration' in trigger_rule_dict)): raise BadParameter('Trigger type and configuration are required together.') # Attempt to look up the trigger handler for the type rule_handler = None if trigger_rule_dict and 'type' in trigger_rule_dict: try: rule_handler = trigger_handler.get_trigger_rule_handler(trigger_rule_dict['type']) except InvalidTriggerType as ex: logger.exception('Invalid trigger type for recipe validation: %s', name) raise BadParameter(unicode(ex)) # Attempt to look up the trigger rule configuration trigger_config = None if rule_handler and 'configuration' in trigger_rule_dict: try: trigger_config = rule_handler.create_configuration(trigger_rule_dict['configuration']) except InvalidTriggerRule as ex: logger.exception('Invalid trigger rule configuration for recipe validation: %s', name) raise BadParameter(unicode(ex)) # Validate the recipe definition try: recipe_def = RecipeDefinition(definition_dict) warnings = RecipeType.objects.validate_recipe_type(name, title, version, description, recipe_def, trigger_config) except (InvalidDefinition, InvalidTriggerType, InvalidTriggerRule, InvalidRecipeConnection) as ex: logger.exception('Unable to validate new recipe type: %s', name) raise BadParameter(unicode(ex)) results = [{'id': w.key, 'details': w.details} for w in warnings] return Response({'warnings': results})
def get(self, request): '''Gets job executions and their associated job_type id, name, and version :param request: the HTTP GET request :type request: :class:`rest_framework.request.Request` :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user ''' started = rest_util.parse_timestamp(request, u'started', required=False) ended = rest_util.parse_timestamp(request, u'ended', required=False) rest_util.check_time_range(started, ended) job_status = rest_util.parse_string(request, u'status', required=False) job_type_ids = rest_util.parse_int_list(request, u'job_type_id', required=False) job_type_names = rest_util.parse_string_list(request, u'job_type_name', required=False) job_type_categories = rest_util.parse_string_list(request, u'job_type_category', required=False) node_ids = rest_util.parse_int_list(request, u'node_id', required=False) order = rest_util.parse_string_list(request, u'order', required=False) job_exes = JobExecution.objects.get_exes(started, ended, job_status, job_type_ids, job_type_names, job_type_categories, node_ids, order) page = rest_util.perform_paging(request, job_exes) serializer = JobExecutionListSerializer(page, context={u'request': request}) return Response(serializer.data, status=status.HTTP_200_OK)
def get(self, request): '''Gets jobs and their associated latest execution :param request: the HTTP GET request :type request: :class:`rest_framework.request.Request` :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user ''' started = rest_util.parse_timestamp(request, u'started', required=False) ended = rest_util.parse_timestamp(request, u'ended', required=False) rest_util.check_time_range(started, ended) job_status = rest_util.parse_string(request, u'status', required=False) job_type_ids = rest_util.parse_int_list(request, u'job_type_id', required=False) job_type_names = rest_util.parse_string_list(request, u'job_type_name', required=False) job_type_categories = rest_util.parse_string_list(request, u'job_type_category', required=False) order = rest_util.parse_string_list(request, u'order', required=False) jobs = Job.objects.get_jobs(started, ended, job_status, job_type_ids, job_type_names, job_type_categories, order) page = rest_util.perform_paging(request, jobs) # Add the latest execution for each matching job paged_jobs = list(page.object_list) job_exes_dict = JobExecution.objects.get_latest(page.object_list) for job in paged_jobs: job.latest_job_exe = job_exes_dict[job.id] if job.id in job_exes_dict else None page.object_list = paged_jobs serializer = JobWithExecutionListSerializer(page, context={u'request': request}) return Response(serializer.data, status=status.HTTP_200_OK)
def get(self, request): '''Retrieves the job updates for a given time range and returns it in JSON form :param request: the HTTP GET request :type request: :class:`rest_framework.request.Request` :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user ''' started = rest_util.parse_timestamp(request, u'started', required=False) ended = rest_util.parse_timestamp(request, u'ended', required=False) rest_util.check_time_range(started, ended) job_status = rest_util.parse_string(request, u'status', required=False) job_type_ids = rest_util.parse_int_list(request, u'job_type_id', required=False) job_type_names = rest_util.parse_string_list(request, u'job_type_name', required=False) job_type_categories = rest_util.parse_string_list(request, u'job_type_category', required=False) order = rest_util.parse_string_list(request, u'order', required=False) jobs = Job.objects.get_job_updates(started, ended, job_status, job_type_ids, job_type_names, job_type_categories, order) page = rest_util.perform_paging(request, jobs) Job.objects.populate_input_files(page) serializer = JobUpdateListSerializer(page, context={'request': request}) return Response(serializer.data, status=status.HTTP_200_OK)
def list(self, request): """Gets jobs and their associated latest execution :param request: the HTTP GET request :type request: :class:`rest_framework.request.Request` :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user """ started = rest_util.parse_timestamp(request, 'started', required=False) ended = rest_util.parse_timestamp(request, 'ended', required=False) rest_util.check_time_range(started, ended) job_status = rest_util.parse_string(request, 'status', required=False) job_ids = rest_util.parse_int_list(request, 'job_id', required=False) job_type_ids = rest_util.parse_int_list(request, 'job_type_id', required=False) job_type_names = rest_util.parse_string_list(request, 'job_type_name', required=False) job_type_categories = rest_util.parse_string_list(request, 'job_type_category', required=False) order = rest_util.parse_string_list(request, 'order', required=False) jobs = Job.objects.get_jobs(started, ended, job_status, job_ids, job_type_ids, job_type_names, job_type_categories, order) # Add the latest execution for each matching job page = self.paginate_queryset(jobs) job_exes_dict = JobExecution.objects.get_latest(page) for job in page: job.latest_job_exe = job_exes_dict[job.id] if job.id in job_exes_dict else None serializer = self.get_serializer(page, many=True) return self.get_paginated_response(serializer.data)
def list(self, request): """Gets job executions and their associated job_type id, name, and version :param request: the HTTP GET request :type request: :class:`rest_framework.request.Request` :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user """ started = rest_util.parse_timestamp(request, 'started', required=False) ended = rest_util.parse_timestamp(request, 'ended', required=False) rest_util.check_time_range(started, ended) job_status = rest_util.parse_string(request, 'status', required=False) job_type_ids = rest_util.parse_int_list(request, 'job_type_id', required=False) job_type_names = rest_util.parse_string_list(request, 'job_type_name', required=False) job_type_categories = rest_util.parse_string_list(request, 'job_type_category', required=False) node_ids = rest_util.parse_int_list(request, 'node_id', required=False) order = rest_util.parse_string_list(request, 'order', required=False) job_exes = JobExecution.objects.get_exes(started, ended, job_status, job_type_ids, job_type_names, job_type_categories, node_ids, order) page = self.paginate_queryset(job_exes) serializer = self.get_serializer(page, many=True) return self.get_paginated_response(serializer.data)
def list(self, request): """Retrieves the product updates for a given time range and returns it in JSON form :param request: the HTTP GET request :type request: :class:`rest_framework.request.Request` :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user """ started = rest_util.parse_timestamp(request, u'started', required=False) ended = rest_util.parse_timestamp(request, u'ended', required=False) rest_util.check_time_range(started, ended) job_type_ids = rest_util.parse_int_list(request, u'job_type_id', required=False) job_type_names = rest_util.parse_string_list(request, u'job_type_name', required=False) job_type_categories = rest_util.parse_string_list(request, u'job_type_category', required=False) is_operational = rest_util.parse_bool(request, u'is_operational', required=False) file_name = rest_util.parse_string(request, u'file_name', required=False) order = rest_util.parse_string_list(request, u'order', required=False) products = ProductFile.objects.get_products(started, ended, job_type_ids, job_type_names, job_type_categories, is_operational, file_name, order) page = self.paginate_queryset(products) ProductFile.objects.populate_source_ancestors(page) serializer = self.get_serializer(page, many=True) return self.get_paginated_response(serializer.data)
def get(self, request): """Retrieves the job updates for a given time range and returns it in JSON form :param request: the HTTP GET request :type request: :class:`rest_framework.request.Request` :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user """ started = rest_util.parse_timestamp(request, 'started', required=False) ended = rest_util.parse_timestamp(request, 'ended', required=False) rest_util.check_time_range(started, ended) job_status = rest_util.parse_string(request, 'status', required=False) job_type_ids = rest_util.parse_int_list(request, 'job_type_id', required=False) job_type_names = rest_util.parse_string_list(request, 'job_type_name', required=False) job_type_categories = rest_util.parse_string_list(request, 'job_type_category', required=False) order = rest_util.parse_string_list(request, 'order', required=False) jobs = Job.objects.get_job_updates(started, ended, job_status, job_type_ids, job_type_names, job_type_categories, order) page = self.paginate_queryset(jobs) Job.objects.populate_input_files(page) serializer = self.get_serializer(page, many=True) return self.get_paginated_response(serializer.data)
def get(self, request): '''Retrieves the product updates for a given time range and returns it in JSON form :param request: the HTTP GET request :type request: :class:`rest_framework.request.Request` :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user ''' started = rest_util.parse_timestamp(request, u'started', required=False) ended = rest_util.parse_timestamp(request, u'ended', required=False) rest_util.check_time_range(started, ended) job_type_ids = rest_util.parse_int_list(request, u'job_type_id', required=False) job_type_names = rest_util.parse_string_list(request, u'job_type_name', required=False) job_type_categories = rest_util.parse_string_list(request, u'job_type_category', required=False) is_operational = rest_util.parse_bool(request, u'is_operational', required=False) file_name = rest_util.parse_string(request, u'file_name', required=False) order = rest_util.parse_string_list(request, u'order', required=False) products = ProductFile.objects.get_products(started, ended, job_type_ids, job_type_names, job_type_categories, is_operational, file_name, order) page = rest_util.perform_paging(request, products) ProductFile.objects.populate_source_ancestors(page) serializer = ProductFileUpdateListSerializer(page, context={'request': request}) return Response(serializer.data, status=status.HTTP_200_OK)
def patch(self, request, job_id): """Modify job info with a subset of fields :param request: the HTTP GET request :type request: :class:`rest_framework.request.Request` :param job_id: The ID for the job. :type job_id: int encoded as a str :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user """ # Validate that no extra fields are included rest_util.check_update(request, ['status']) # Validate JSON status_code = rest_util.parse_string(request, 'status') if status_code != 'CANCELED': raise rest_util.BadParameter('Invalid or read-only status. Allowed values: CANCELED') try: Queue.objects.handle_job_cancellation(job_id, datetime.utcnow()) job = Job.objects.get_details(job_id) except (Job.DoesNotExist, JobExecution.DoesNotExist): raise Http404 serializer = self.get_serializer(job) return Response(serializer.data)
def test_parse_string_optional(self): '''Tests parsing an optional string parameter that is missing.''' request = MagicMock(Request) request.QUERY_PARAMS = QueryDict('', mutable=True) request.QUERY_PARAMS.update({ 'test': 'value1', }) self.assertIsNone(rest_util.parse_string(request, 'test2', required=False))
def test_parse_string_post(self): '''Tests parsing a required string parameter that is provided via POST.''' request = MagicMock(Request) request.DATA = QueryDict('', mutable=True) request.DATA.update({ 'test': 'value1', }) self.assertEqual(rest_util.parse_string(request, 'test'), 'value1')
def test_parse_string(self): """Tests parsing a required string parameter that is provided via GET.""" request = MagicMock(Request) request.query_params = QueryDict('', mutable=True) request.query_params.update({ 'test': 'value1', }) self.assertEqual(rest_util.parse_string(request, 'test'), 'value1')
def test_parse_string_default(self): '''Tests parsing an optional string parameter that is provided via default value.''' request = MagicMock(Request) request.QUERY_PARAMS = QueryDict('', mutable=True) request.QUERY_PARAMS.update({ 'test': 'value1', }) self.assertEqual(rest_util.parse_string(request, 'test2', 'value2'), 'value2')
def test_parse_string_accepted_all(self): '''Tests parsing a string parameter where the value is acceptable.''' request = MagicMock(Request) request.QUERY_PARAMS = QueryDict('', mutable=True) request.QUERY_PARAMS.update({ 'test': 'value1', }) self.assertEqual(rest_util.parse_string(request, 'test', accepted_values=['value1']), 'value1')
def post(self, request): '''Creates a new Strike process and returns its ID in JSON form :param request: the HTTP POST request :type request: :class:`rest_framework.request.Request` :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user ''' name = rest_util.parse_string(request, 'name') title = rest_util.parse_string(request, 'title', required=False) description = rest_util.parse_string(request, 'description', required=False) configuration = rest_util.parse_dict(request, 'configuration') try: strike = Strike.objects.create_strike_process(name, title, description, configuration) except InvalidStrikeConfiguration: raise BadParameter('Configuration failed to validate.') return Response({'strike_id': strike.id})
def list(self, request): """Retrieves the list of all ingests and returns it in JSON form :param request: the HTTP GET request :type request: :class:`rest_framework.request.Request` :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user """ started = rest_util.parse_timestamp(request, 'started', required=False) ended = rest_util.parse_timestamp(request, 'ended', required=False) rest_util.check_time_range(started, ended) ingest_status = rest_util.parse_string(request, 'status', required=False) file_name = rest_util.parse_string(request, 'file_name', required=False) order = rest_util.parse_string_list(request, 'order', required=False) ingests = Ingest.objects.get_ingests(started, ended, ingest_status, file_name, order) page = self.paginate_queryset(ingests) serializer = self.get_serializer(page, many=True) return self.get_paginated_response(serializer.data)
def create(self, request): """Creates a new batch and returns a link to the detail URL :param request: the HTTP POST request :type request: :class:`rest_framework.request.Request` :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user """ recipe_type_id = rest_util.parse_int(request, 'recipe_type_id') title = rest_util.parse_string(request, 'title', required=False) description = rest_util.parse_string(request, 'description', required=False) # Make sure the recipe type exists try: recipe_type = RecipeType.objects.get(pk=recipe_type_id) except RecipeType.DoesNotExist: raise BadParameter('Unknown recipe type: %i' % recipe_type_id) # Validate the batch definition definition_dict = rest_util.parse_dict(request, 'definition') definition = None try: if definition_dict: definition = BatchDefinition(definition_dict) except InvalidDefinition as ex: raise BadParameter('Batch definition invalid: %s' % unicode(ex)) # Create the batch batch = Batch.objects.create_batch(recipe_type, definition, title=title, description=description) # Fetch the full batch with details try: batch = Batch.objects.get_details(batch.id) except Batch.DoesNotExist: raise Http404 url = reverse('batch_details_view', args=[batch.id], request=request) serializer = BatchDetailsSerializer(batch) return Response(serializer.data, status=status.HTTP_201_CREATED, headers=dict(location=url))
def post(self, request): '''Validates a new recipe type and returns any warnings discovered :param request: the HTTP POST request :type request: :class:`rest_framework.request.Request` :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user ''' name = rest_util.parse_string(request, u'name') version = rest_util.parse_string(request, u'version') description = rest_util.parse_string(request, u'description', '') definition = rest_util.parse_dict(request, u'definition') try: recipe_def = RecipeDefinition(definition) warnings = RecipeType.objects.validate_recipe_type(name, version, description, recipe_def) except InvalidDefinition as ex: logger.exception('Unable to validate new recipe type: %s', name) raise BadParameter(unicode(ex)) results = [{'id': w.key, 'details': w.details} for w in warnings] return Response({u'warnings': results}, status=status.HTTP_200_OK)
def post(self, request): """Creates a new error and returns a link to the info URL :param request: the HTTP POST request :type request: :class:`rest_framework.request.Request` :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user """ name = rest_util.parse_string(request, 'name') title = rest_util.parse_string(request, 'title') description = rest_util.parse_string(request, 'description') category = rest_util.parse_string(request, 'category', accepted_values=[c for c, _ in Error.CATEGORIES]) # Do not allow the creation of SYSTEM level errors if category == 'SYSTEM': raise BadParameter('System level errors cannot be created.') error = Error.objects.create_error(name, title, description, category) serializer = ErrorDetailsSerializer(error) error_url = urlresolvers.reverse('error_details_view', args=[error.id]) return Response(serializer.data, status=status.HTTP_201_CREATED, headers=dict(location=error_url))
def post(self, request): '''Creates a new recipe type and returns a link to the detail URL :param request: the HTTP POST request :type request: :class:`rest_framework.request.Request` :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user ''' name = rest_util.parse_string(request, u'name') version = rest_util.parse_string(request, u'version') title = rest_util.parse_string(request, u'title') description = rest_util.parse_string(request, u'description', '') definition = rest_util.parse_dict(request, u'definition') try: recipe_def = RecipeDefinition(definition) recipe_type = RecipeType.objects.create_recipe_type(name, version, title, description, recipe_def, None) except InvalidDefinition: logger.exception('Unable to create new recipe type: %s', name) raise BadParameter('Invalid recipe type definition') url = urlresolvers.reverse('recipe_type_details_view', args=[recipe_type.id]) return Response({u'id': recipe_type.id}, status=status.HTTP_201_CREATED, headers=dict(location=url))
def post(self, request): '''Creates a new error and returns a link to the info URL :param request: the HTTP POST request :type request: :class:`rest_framework.request.Request` :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user ''' name = rest_util.parse_string(request, u'name') title = rest_util.parse_string(request, u'title') description = rest_util.parse_string(request, u'description') category = rest_util.parse_string(request, u'category') if category not in map(lambda x: x[1], Error.CATEGORIES): return Response(u'category is not valid', status=status.HTTP_400_BAD_REQUEST) try: error_id = Error.objects.create_error(name, title, description, category).id except InvalidData: return Response('Unable to create error.', status=status.HTTP_400_BAD_REQUEST) error_url = urlresolvers.reverse('error_details_view', args=[error_id]) return Response({u'error_id': error_id}, status=status.HTTP_201_CREATED, headers=dict(location=error_url))
def patch(self, request, name, version): """Edits an existing seed job type and returns the updated details :param request: the HTTP PATCH request :type request: :class:`rest_framework.request.Request` :param job_type_id: The ID for the job type. :type job_type_id: int encoded as a str :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user """ auto_update = rest_util.parse_bool(request, 'auto_update', required=False, default_value=True) icon_code = rest_util.parse_string(request, 'icon_code', required=False) is_published = rest_util.parse_string(request, 'is_published', required=False) is_active = rest_util.parse_bool(request, 'is_active', required=False) is_paused = rest_util.parse_bool(request, 'is_paused', required=False) max_scheduled = rest_util.parse_int(request, 'max_scheduled', required=False) docker_image = rest_util.parse_string(request, 'docker_image', required=False) # Validate the manifest manifest_dict = rest_util.parse_dict(request, 'manifest', required=False) manifest = None if manifest_dict: try: manifest = SeedManifest(manifest_dict, do_validate=True) except InvalidSeedManifestDefinition as ex: message = 'Seed Manifest invalid' logger.exception(message) raise BadParameter('%s: %s' % (message, unicode(ex))) # validate manifest name/version matches job type manifest_name = manifest.get_name() if name != manifest_name: raise BadParameter('Manifest name %s does not match current Job Type name %s.' % (manifest_name, name)) manifest_version = manifest.get_job_version() if manifest_version != version: raise BadParameter('Manifest version %s does not match current Job Type version %s.' % (manifest_version, version)) # Validate the job configuration and pull out secrets configuration_dict = rest_util.parse_dict(request, 'configuration', required=False) configuration = None try: if configuration_dict: configuration = JobConfigurationV6(configuration_dict).get_configuration() except InvalidJobConfiguration as ex: raise BadParameter('Job type configuration invalid: %s' % unicode(ex)) # Fetch the current job type model try: job_type = JobType.objects.get(name=name, version=version) except JobType.DoesNotExist: raise Http404 # Check for invalid fields fields = {'icon_code', 'is_published', 'is_active', 'is_paused', 'max_scheduled', 'configuration', 'manifest', 'docker_image', 'auto_update'} for key, value in request.data.iteritems(): if key not in fields: raise BadParameter('%s is not a valid field. Valid fields are: %s' % (key, fields)) try: with transaction.atomic(): # Edit the job type validation = JobType.objects.edit_job_type_v6(job_type_id=job_type.id, manifest=manifest, is_published=is_published, docker_image=docker_image, icon_code=icon_code, is_active=is_active, is_paused=is_paused, max_scheduled=max_scheduled, configuration=configuration, auto_update=auto_update) except (InvalidJobField, InvalidSecretsConfiguration, ValueError, InvalidJobConfiguration, InvalidInterfaceDefinition) as ex: logger.exception('Unable to update job type: %i', job_type.id) raise BadParameter(unicode(ex)) resp_dict = {'is_valid': validation.is_valid, 'errors': [e.to_dict() for e in validation.errors], 'warnings': [w.to_dict() for w in validation.warnings]} return Response(resp_dict)
def create(self, request): """Creates a new job type and returns a link to the detail URL :param request: the HTTP POST request :type request: :class:`rest_framework.request.Request` :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user """ name = rest_util.parse_string(request, 'name') version = rest_util.parse_string(request, 'version') # Validate the job interface interface_dict = rest_util.parse_dict(request, 'interface') interface = None try: if interface_dict: interface = JobInterface(interface_dict) except InvalidInterfaceDefinition as ex: raise BadParameter('Job type interface invalid: %s' % unicode(ex)) # Validate the error mapping error_dict = rest_util.parse_dict(request, 'error_mapping', required=False) error_mapping = None try: if error_dict: error_mapping = ErrorInterface(error_dict) error_mapping.validate() except InvalidInterfaceDefinition as ex: raise BadParameter('Job type error mapping invalid: %s' % unicode(ex)) # Check for optional trigger rule parameters trigger_rule_dict = rest_util.parse_dict(request, 'trigger_rule', required=False) if (('type' in trigger_rule_dict and 'configuration' not in trigger_rule_dict) or ('type' not in trigger_rule_dict and 'configuration' in trigger_rule_dict)): raise BadParameter( 'Trigger type and configuration are required together.') is_active = trigger_rule_dict[ 'is_active'] if 'is_active' in trigger_rule_dict else True # Attempt to look up the trigger handler for the type rule_handler = None if trigger_rule_dict and 'type' in trigger_rule_dict: try: rule_handler = trigger_handler.get_trigger_rule_handler( trigger_rule_dict['type']) except InvalidTriggerType as ex: logger.exception('Invalid trigger type for new job type: %s', name) raise BadParameter(unicode(ex)) # Extract the fields that should be updated as keyword arguments extra_fields = {} base_fields = { 'name', 'version', 'interface', 'trigger_rule', 'error_mapping' } for key, value in request.data.iteritems(): if key not in base_fields and key not in JobType.UNEDITABLE_FIELDS: extra_fields[key] = value try: with transaction.atomic(): # Attempt to create the trigger rule trigger_rule = None if rule_handler and 'configuration' in trigger_rule_dict: trigger_rule = rule_handler.create_trigger_rule( trigger_rule_dict['configuration'], name, is_active) # Create the job type job_type = JobType.objects.create_job_type( name, version, interface, trigger_rule, error_mapping, **extra_fields) except (InvalidJobField, InvalidTriggerType, InvalidTriggerRule, InvalidConnection, ValueError) as ex: logger.exception('Unable to create new job type: %s', name) raise BadParameter(unicode(ex)) # Fetch the full job type with details try: job_type = JobType.objects.get_details(job_type.id) except JobType.DoesNotExist: raise Http404 url = urlresolvers.reverse('job_type_details_view', args=[job_type.id]) serializer = JobTypeDetailsSerializer(job_type) return Response(serializer.data, status=status.HTTP_201_CREATED, headers=dict(location=url))
def patch(self, request, recipe_type_id): """Edits an existing recipe type and returns the updated details :param request: the HTTP PATCH request :type request: :class:`rest_framework.request.Request` :param recipe_type_id: The ID for the recipe type. :type recipe_type_id: int encoded as a str :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user """ title = rest_util.parse_string(request, 'title', required=False) description = rest_util.parse_string(request, 'description', required=False) definition_dict = rest_util.parse_dict(request, 'definition', required=False) # Check for optional trigger rule parameters trigger_rule_dict = rest_util.parse_dict(request, 'trigger_rule', required=False) if (('type' in trigger_rule_dict and 'configuration' not in trigger_rule_dict) or ('type' not in trigger_rule_dict and 'configuration' in trigger_rule_dict)): raise BadParameter( 'Trigger type and configuration are required together.') is_active = trigger_rule_dict[ 'is_active'] if 'is_active' in trigger_rule_dict else True remove_trigger_rule = rest_util.has_params( request, 'trigger_rule') and not trigger_rule_dict # Fetch the current recipe type model try: recipe_type = RecipeType.objects.select_related( 'trigger_rule').get(pk=recipe_type_id) except RecipeType.DoesNotExist: raise Http404 # Attempt to look up the trigger handler for the type rule_handler = None if trigger_rule_dict and 'type' in trigger_rule_dict: try: rule_handler = trigger_handler.get_trigger_rule_handler( trigger_rule_dict['type']) except InvalidTriggerType as ex: logger.exception('Invalid trigger type for recipe type: %i', recipe_type_id) raise BadParameter(unicode(ex)) try: with transaction.atomic(): # Validate the recipe definition recipe_def = None if definition_dict: recipe_def = RecipeDefinitionSunset.create(definition_dict) # Attempt to create the trigger rule trigger_rule = None if rule_handler and 'configuration' in trigger_rule_dict: trigger_rule = rule_handler.create_trigger_rule( trigger_rule_dict['configuration'], recipe_type.name, is_active) # Update the active state separately if that is only given trigger field if not trigger_rule and recipe_type.trigger_rule and 'is_active' in trigger_rule_dict: recipe_type.trigger_rule.is_active = is_active recipe_type.trigger_rule.save() # Edit the recipe type RecipeType.objects.edit_recipe_type(recipe_type_id, title, description, recipe_def, trigger_rule, remove_trigger_rule) except (InvalidDefinition, InvalidTriggerType, InvalidTriggerRule, InvalidRecipeConnection) as ex: logger.exception('Unable to update recipe type: %i', recipe_type_id) raise BadParameter(unicode(ex)) # Fetch the full recipe type with details try: recipe_type = RecipeType.objects.get_details(recipe_type_id) except RecipeType.DoesNotExist: raise Http404 serializer = self.get_serializer(recipe_type) return Response(serializer.data)
def post(self, request): """Validates a new recipe type and returns any warnings discovered :param request: the HTTP POST request :type request: :class:`rest_framework.request.Request` :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user """ name = rest_util.parse_string(request, 'name') version = rest_util.parse_string(request, 'version') title = rest_util.parse_string(request, 'title', default_value=name) description = rest_util.parse_string(request, 'description', required=False) definition_dict = rest_util.parse_dict(request, 'definition') # Check for optional trigger rule parameters trigger_rule_dict = rest_util.parse_dict(request, 'trigger_rule', required=False) if (('type' in trigger_rule_dict and 'configuration' not in trigger_rule_dict) or ('type' not in trigger_rule_dict and 'configuration' in trigger_rule_dict)): raise BadParameter( 'Trigger type and configuration are required together.') # Attempt to look up the trigger handler for the type rule_handler = None if trigger_rule_dict and 'type' in trigger_rule_dict: try: rule_handler = trigger_handler.get_trigger_rule_handler( trigger_rule_dict['type']) except InvalidTriggerType as ex: logger.exception( 'Invalid trigger type for recipe validation: %s', name) raise BadParameter(unicode(ex)) # Attempt to look up the trigger rule configuration trigger_config = None if rule_handler and 'configuration' in trigger_rule_dict: try: trigger_config = rule_handler.create_configuration( trigger_rule_dict['configuration']) except InvalidTriggerRule as ex: logger.exception( 'Invalid trigger rule configuration for recipe validation: %s', name) raise BadParameter(unicode(ex)) # Validate the recipe definition try: recipe_def = RecipeDefinition(definition_dict) warnings = RecipeType.objects.validate_recipe_type( name, title, version, description, recipe_def, trigger_config) except (InvalidDefinition, InvalidTriggerType, InvalidTriggerRule, InvalidRecipeConnection) as ex: logger.exception('Unable to validate new recipe type: %s', name) raise BadParameter(unicode(ex)) results = [{'id': w.key, 'details': w.details} for w in warnings] return Response({'warnings': results})
def post(self, request): """Creates a new recipe type and returns a link to the detail URL :param request: the HTTP POST request :type request: :class:`rest_framework.request.Request` :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user """ name = rest_util.parse_string(request, 'name') version = rest_util.parse_string(request, 'version') title = rest_util.parse_string(request, 'title', default_value=name) description = rest_util.parse_string(request, 'description', required=False) definition_dict = rest_util.parse_dict(request, 'definition') # Check for optional trigger rule parameters trigger_rule_dict = rest_util.parse_dict(request, 'trigger_rule', required=False) if (('type' in trigger_rule_dict and 'configuration' not in trigger_rule_dict) or ('type' not in trigger_rule_dict and 'configuration' in trigger_rule_dict)): raise BadParameter( 'Trigger type and configuration are required together.') is_active = trigger_rule_dict[ 'is_active'] if 'is_active' in trigger_rule_dict else True # Attempt to look up the trigger handler for the type rule_handler = None if trigger_rule_dict and 'type' in trigger_rule_dict: try: rule_handler = trigger_handler.get_trigger_rule_handler( trigger_rule_dict['type']) except InvalidTriggerType as ex: logger.exception( 'Invalid trigger type for new recipe type: %s', name) raise BadParameter(unicode(ex)) try: with transaction.atomic(): # Validate the recipe definition recipe_def = RecipeDefinition(definition_dict) # Attempt to create the trigger rule trigger_rule = None if rule_handler and 'configuration' in trigger_rule_dict: trigger_rule = rule_handler.create_trigger_rule( trigger_rule_dict['configuration'], name, is_active) # Create the recipe type recipe_type = RecipeType.objects.create_recipe_type( name, version, title, description, recipe_def, trigger_rule) except (InvalidDefinition, InvalidTriggerType, InvalidTriggerRule, InvalidRecipeConnection) as ex: logger.exception('Unable to create new recipe type: %s', name) raise BadParameter(unicode(ex)) # Fetch the full recipe type with details try: recipe_type = RecipeType.objects.get_details(recipe_type.id) except RecipeType.DoesNotExist: raise Http404 url = reverse('recipe_type_details_view', args=[recipe_type.id], request=request) serializer = RecipeTypeDetailsSerializer(recipe_type) return Response(serializer.data, status=status.HTTP_201_CREATED, headers=dict(location=url))
def create_v6(self, request): """Creates or edits a Seed job type and returns a link to the detail URL :param request: the HTTP POST request :type request: :class:`rest_framework.request.Request` :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user """ # Optional icon code value icon_code = rest_util.parse_string(request, 'icon_code', required=False) # Optional is published value is_published = rest_util.parse_string(request, 'is_published', required=False) # Optional max scheduled value max_scheduled = rest_util.parse_int(request, 'max_scheduled', required=False) # Require docker image value docker_image = rest_util.parse_string(request, 'docker_image', required=True) # Validate the job interface / manifest manifest_dict = rest_util.parse_dict(request, 'manifest', required=True) # If editing an existing job type, automatically update recipes containing said job type auto_update = rest_util.parse_bool(request, 'auto_update', required=False, default_value=True) # Optional setting job type active if editing existing job is_active = rest_util.parse_bool(request, 'is_active', required=False) # Optional setting job type to paused if editing an existing job is_paused = rest_util.parse_bool(request, 'is_paused', required=False) manifest = None try: manifest = SeedManifest(manifest_dict, do_validate=True) except InvalidSeedManifestDefinition as ex: message = 'Seed Manifest invalid' logger.exception(message) raise BadParameter('%s: %s' % (message, unicode(ex))) # Validate the job configuration and pull out secrets configuration_dict = rest_util.parse_dict(request, 'configuration', required=False) configuration = None if configuration_dict: try: configuration = JobConfigurationV6(configuration_dict, do_validate=True).get_configuration() except InvalidJobConfiguration as ex: message = 'Job type configuration invalid' logger.exception(message) raise BadParameter('%s: %s' % (message, unicode(ex))) # Check for invalid fields fields = {'icon_code', 'is_published', 'max_scheduled', 'docker_image', 'configuration', 'manifest', 'auto_update', 'is_active', 'is_paused'} for key, value in request.data.iteritems(): if key not in fields: raise BadParameter('%s is not a valid field. Valid fields are: %s' % (key, fields)) name = manifest_dict['job']['name'] version = manifest_dict['job']['jobVersion'] if name == 'validation': logger.exception('Unable to create job type named "validation"') raise BadParameter(unicode('Unable to create job type named "validation"')) existing_job_type = JobType.objects.filter(name=name, version=version).first() if not existing_job_type: try: # Create the job type job_type = JobType.objects.create_job_type_v6(icon_code=icon_code, is_published=is_published, max_scheduled=max_scheduled, docker_image=docker_image, manifest=manifest, configuration=configuration) except (InvalidJobField, InvalidSecretsConfiguration, ValueError) as ex: message = 'Unable to create new job type' logger.exception(message) raise BadParameter('%s: %s' % (message, unicode(ex))) except InvalidSeedManifestDefinition as ex: message = 'Job type manifest invalid' logger.exception(message) raise BadParameter('%s: %s' % (message, unicode(ex))) except InvalidJobConfiguration as ex: message = 'Job type configuration invalid' logger.exception(message) raise BadParameter('%s: %s' % (message, unicode(ex))) # Fetch the full job type with details try: job_type = JobType.objects.get_details_v6(name=name, version=version) except JobType.DoesNotExist: raise Http404 url = reverse('job_type_details_view', args=[job_type.name, job_type.version], request=request) serializer = JobTypeDetailsSerializerV6(job_type) return Response(serializer.data, status=status.HTTP_201_CREATED, headers=dict(location=url)) else: try: validation = JobType.objects.edit_job_type_v6(job_type_id=existing_job_type.id, manifest=manifest, docker_image=docker_image, icon_code=icon_code, is_active=is_active, is_paused=is_paused, max_scheduled=max_scheduled, is_published=is_published, configuration=configuration, auto_update=auto_update) except (InvalidJobField, InvalidSecretsConfiguration, ValueError, InvalidInterfaceDefinition) as ex: logger.exception('Unable to update job type: %i', existing_job_type.id) raise BadParameter(unicode(ex)) except InvalidSeedManifestDefinition as ex: message = 'Job type manifest invalid' logger.exception(message) raise BadParameter('%s: %s' % (message, unicode(ex))) except InvalidJobConfiguration as ex: message = 'Job type configuration invalid' logger.exception(message) raise BadParameter('%s: %s' % (message, unicode(ex))) resp_dict = {'is_valid': validation.is_valid, 'errors': [e.to_dict() for e in validation.errors], 'warnings': [w.to_dict() for w in validation.warnings]} return Response(resp_dict)
def create_all_v6(self, request): """Creates or edits a dataset - including the dataset members - and returns a link to the detail URL""" title = rest_util.parse_string(request, 'title', required=False) description = rest_util.parse_string(request, 'description', required=False) definition = rest_util.parse_dict(request, 'definition', required=True) template = rest_util.parse_dict(request, 'data_template', required=False) dry_run = rest_util.parse_bool(request, 'dry_run', default_value=False) # file filters data_started = rest_util.parse_timestamp(request, 'data_started', required=False) data_ended = rest_util.parse_timestamp(request, 'data_ended', required=False) rest_util.check_time_range(data_started, data_ended) source_started = rest_util.parse_timestamp(request, 'source_started', required=False) source_ended = rest_util.parse_timestamp(request, 'source_ended', required=False) rest_util.check_time_range(source_started, source_ended) source_sensor_classes = rest_util.parse_string_list( request, 'source_sensor_class', required=False) source_sensors = rest_util.parse_string_list(request, 'source_sensor', required=False) source_collections = rest_util.parse_string_list(request, 'source_collection', required=False) source_tasks = rest_util.parse_string_list(request, 'source_task', required=False) mod_started = rest_util.parse_timestamp(request, 'modified_started', required=False) mod_ended = rest_util.parse_timestamp(request, 'modified_ended', required=False) rest_util.check_time_range(mod_started, mod_ended) job_type_ids = rest_util.parse_int_list(request, 'job_type_id', required=False) job_type_names = rest_util.parse_string_list(request, 'job_type_name', required=False) job_ids = rest_util.parse_int_list(request, 'job_id', required=False) file_names = rest_util.parse_string_list(request, 'file_name', required=False) job_outputs = rest_util.parse_string_list(request, 'job_output', required=False) recipe_ids = rest_util.parse_int_list(request, 'recipe_id', required=False) recipe_type_ids = rest_util.parse_int_list(request, 'recipe_type_id', required=False) recipe_nodes = rest_util.parse_string_list(request, 'recipe_node', required=False) batch_ids = rest_util.parse_int_list(request, 'batch_id', required=False) order = rest_util.parse_string_list(request, 'order', required=False) data = rest_util.parse_dict_list(request, 'data', required=False) data_list = [] # validate the definition & create the dataset try: dataset_def = DataSetDefinitionV6( definition=definition, do_validate=True).get_definition() except InvalidDataSetDefinition as ex: message = 'DataSet definition is invalid' logger.exception(message) raise BadParameter('%s: %s' % (message, unicode(ex))) try: dataset = DataSet.objects.create_dataset_v6( dataset_def, title=title, description=description) except Exception as ex: message = 'Unable to create new dataset' logger.exception(message) raise BadParameter('%s: %s' % (message, unicode(ex))) try: dataset = DataSet.objects.get_details_v6(dataset.id) except DataSet.DoesNotExist: raise Http404 if not data and not template: url = reverse('dataset_details_view', args=[dataset.id], request=request) serializer = DataSetDetailsSerializerV6(dataset) return Response(serializer.data, status=status.HTTP_201_CREATED, headers=dict(location=url)) # Try and find the data if data: try: for d in data: data = DataV6(data=d, do_validate=True).get_data() data_list.append(data) except InvalidData as ex: message = 'Data is invalid' logger.exception(message) raise BadParameter('%s: %s' % (message, unicode(ex))) elif template: try: data_list = DataSetMember.objects.build_data_list( template=template, data_started=data_started, data_ended=data_ended, source_started=source_started, source_ended=source_ended, source_sensor_classes=source_sensor_classes, source_sensors=source_sensors, source_collections=source_collections, source_tasks=source_tasks, mod_started=mod_started, mod_ended=mod_ended, job_type_ids=job_type_ids, job_type_names=job_type_names, job_ids=job_ids, file_names=file_names, job_outputs=job_outputs, recipe_ids=recipe_ids, recipe_type_ids=recipe_type_ids, recipe_nodes=recipe_nodes, batch_ids=batch_ids, order=order) except InvalidData as ex: message = 'Data is invalid' logger.exception(message) raise BadParameter('%s: %s' % (message, unicode(ex))) if not data_list: resp_dict = { 'No Results': 'No files found from filters and/or no data provided' } return Response(resp_dict) validation = DataSetMember.objects.validate_data_list( dataset_def=dataset_def, data_list=data_list) members = [] if validation.is_valid and not dry_run: members = DataSetMember.objects.create_dataset_members( dataset=dataset, data_list=data_list) dataset = DataSet.objects.get(id=dataset.id) serializer = DataSetDetailsSerializerV6(dataset) url = reverse('dataset_details_view', args=[dataset.id], request=request) return Response(serializer.data, status=status.HTTP_201_CREATED, headers=dict(location=url)) elif not validation.is_valid: raise BadParameter('%s: %s' % ('Error(s) validating data against dataset', [e.to_dict() for e in validation.errors])) resp_dict = [] for dl in data_list: resp_dict.append(convert_data_to_v6_json(dl).get_dict()) return Response(resp_dict)
def _create_v6(self, request): """Creates a new recipe type and returns a link to the detail URL :param request: the HTTP POST request :type request: :class:`rest_framework.request.Request` :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user """ title = rest_util.parse_string(request, 'title', required=True) description = rest_util.parse_string(request, 'description', required=False) definition_dict = rest_util.parse_dict(request, 'definition', required=True) basename = title_to_basename(title) if basename == 'validation': logger.exception('Unable to create recipe type named "validation"') raise BadParameter( unicode('Unable to create recipe type named "validation"')) existing_recipes = RecipeType.objects.filter(name=basename) if existing_recipes.count() > 0: logger.exception( 'Existing recipe types found for %s - will not re-create.', basename) raise BadParameter( unicode( 'Existing recipe types found for %s - will not re-create. Please change the title or patch the existing recipe type.' % basename)) name = title_to_name(self.queryset, title) try: with transaction.atomic(): # Validate the recipe definition recipe_def = RecipeDefinitionV6( definition=definition_dict, do_validate=True).get_definition() # Create the recipe type recipe_type = RecipeType.objects.create_recipe_type_v6( name, title, description, recipe_def) except InvalidDefinition as ex: logger.exception('Unable to create new recipe type: %s', name) raise BadParameter(unicode(ex)) # Fetch the full recipe type with details try: recipe_type = RecipeType.objects.get_details_v6(recipe_type.name) except RecipeType.DoesNotExist: raise Http404 url = reverse('recipe_type_details_view', args=[recipe_type.name], request=request) serializer = RecipeTypeDetailsSerializerV6(recipe_type) return Response(serializer.data, status=status.HTTP_201_CREATED, headers=dict(location=url))
def _list_v6(self, request): """Retrieves a list of files based on filters and returns it in JSON form :param request: the HTTP GET request :type request: :class:`rest_framework.request.Request` :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user """ countries = rest_util.parse_string_list(request, 'countries', required=False) data_started = rest_util.parse_timestamp(request, 'data_started', required=False) data_ended = rest_util.parse_timestamp(request, 'data_ended', required=False) rest_util.check_time_range(data_started, data_ended) created_started = rest_util.parse_timestamp(request, 'created_started', required=False) created_ended = rest_util.parse_timestamp(request, 'created_ended', required=False) rest_util.check_time_range(created_started, created_ended) source_started = rest_util.parse_timestamp(request, 'source_started', required=False) source_ended = rest_util.parse_timestamp(request, 'source_ended', required=False) rest_util.check_time_range(source_started, source_ended) source_sensor_classes = rest_util.parse_string_list( request, 'source_sensor_class', required=False) file_type = rest_util.parse_string_list(request, 'file_type', required=False) source_sensors = rest_util.parse_string_list(request, 'source_sensor', required=False) source_collections = rest_util.parse_string_list(request, 'source_collection', required=False) source_tasks = rest_util.parse_string_list(request, 'source_task', required=False) mod_started = rest_util.parse_timestamp(request, 'modified_started', required=False) mod_ended = rest_util.parse_timestamp(request, 'modified_ended', required=False) rest_util.check_time_range(mod_started, mod_ended) job_type_ids = rest_util.parse_int_list(request, 'job_type_id', required=False) job_type_names = rest_util.parse_string_list(request, 'job_type_name', required=False) job_ids = rest_util.parse_int_list(request, 'job_id', required=False) file_names = rest_util.parse_string_list(request, 'file_name', required=False) job_outputs = rest_util.parse_string_list(request, 'job_output', required=False) recipe_ids = rest_util.parse_int_list(request, 'recipe_id', required=False) recipe_type_ids = rest_util.parse_int_list(request, 'recipe_type_id', required=False) recipe_nodes = rest_util.parse_string_list(request, 'recipe_node', required=False) batch_ids = rest_util.parse_int_list(request, 'batch_id', required=False) media_type = rest_util.parse_string_list(request, 'media_type', required=False) file_name_search = rest_util.parse_string(request, 'file_name_search', required=False) order = rest_util.parse_string_list(request, 'order', required=False) files = ScaleFile.objects.filter_files( data_started=data_started, data_ended=data_ended, created_started=created_started, created_ended=created_ended, source_started=source_started, source_ended=source_ended, source_sensor_classes=source_sensor_classes, source_sensors=source_sensors, source_collections=source_collections, source_tasks=source_tasks, mod_started=mod_started, mod_ended=mod_ended, job_type_ids=job_type_ids, job_type_names=job_type_names, job_ids=job_ids, file_names=file_names, file_name_search=file_name_search, job_outputs=job_outputs, recipe_ids=recipe_ids, recipe_type_ids=recipe_type_ids, recipe_nodes=recipe_nodes, batch_ids=batch_ids, order=order, countries=countries, file_type=file_type, media_type=media_type) page = self.paginate_queryset(files) serializer = self.get_serializer(page, many=True) return self.get_paginated_response(serializer.data)
def post(self, request): """Validates a new job type and returns any warnings discovered :param request: the HTTP POST request :type request: :class:`rest_framework.request.Request` :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user """ name = rest_util.parse_string(request, 'name') version = rest_util.parse_string(request, 'version') # Validate the job interface interface_dict = rest_util.parse_dict(request, 'interface') interface = None try: if interface_dict: interface = JobInterface(interface_dict) except InvalidInterfaceDefinition as ex: raise BadParameter('Job type interface invalid: %s' % unicode(ex)) # Validate the error mapping error_dict = rest_util.parse_dict(request, 'error_mapping', required=False) error_mapping = None try: if error_dict: error_mapping = ErrorInterface(error_dict) except InvalidInterfaceDefinition as ex: raise BadParameter('Job type error mapping invalid: %s' % unicode(ex)) # Check for optional trigger rule parameters trigger_rule_dict = rest_util.parse_dict(request, 'trigger_rule', required=False) if (('type' in trigger_rule_dict and 'configuration' not in trigger_rule_dict) or ('type' not in trigger_rule_dict and 'configuration' in trigger_rule_dict)): raise BadParameter( 'Trigger type and configuration are required together.') # Attempt to look up the trigger handler for the type rule_handler = None if trigger_rule_dict and 'type' in trigger_rule_dict: try: rule_handler = trigger_handler.get_trigger_rule_handler( trigger_rule_dict['type']) except InvalidTriggerType as ex: logger.exception('Invalid trigger type for job validation: %s', name) raise BadParameter(unicode(ex)) # Attempt to look up the trigger rule configuration trigger_config = None if rule_handler and 'configuration' in trigger_rule_dict: try: trigger_config = rule_handler.create_configuration( trigger_rule_dict['configuration']) except InvalidTriggerRule as ex: logger.exception( 'Invalid trigger rule configuration for job validation: %s', name) raise BadParameter(unicode(ex)) try: from recipe.configuration.definition.exceptions import InvalidDefinition except: logger.exception( 'Failed to import higher level recipe application.') pass # Validate the job interface try: warnings = JobType.objects.validate_job_type( name, version, interface, error_mapping, trigger_config) except (InvalidDefinition, InvalidTriggerType, InvalidTriggerRule) as ex: logger.exception('Unable to validate new job type: %s', name) raise BadParameter(unicode(ex)) results = [{'id': w.key, 'details': w.details} for w in warnings] return Response({'warnings': results})
def list_impl(self, request, source_id=None): """Retrieves the products for a given source file ID and returns them in JSON form :param request: the HTTP GET request :type request: :class:`rest_framework.request.Request` :param source_id: The id of the source :type source_id: int encoded as a string :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user """ try: ScaleFile.objects.get(id=source_id, file_type='SOURCE') except ScaleFile.DoesNotExist: raise Http404 started = rest_util.parse_timestamp(request, 'started', required=False) ended = rest_util.parse_timestamp(request, 'ended', required=False) rest_util.check_time_range(started, ended) time_field = rest_util.parse_string( request, 'time_field', required=False, accepted_values=ProductFile.VALID_TIME_FIELDS) batch_ids = rest_util.parse_int_list(request, 'batch_id', required=False) job_type_ids = rest_util.parse_int_list(request, 'job_type_id', required=False) job_type_names = rest_util.parse_string_list(request, 'job_type_name', required=False) job_type_categories = rest_util.parse_string_list(request, 'job_type_category', required=False) job_ids = rest_util.parse_int_list(request, 'job_id', required=False) is_operational = rest_util.parse_bool(request, 'is_operational', required=False) is_published = rest_util.parse_bool(request, 'is_published', required=False) file_name = rest_util.parse_string(request, 'file_name', required=False) job_output = rest_util.parse_string(request, 'job_output', required=False) recipe_ids = rest_util.parse_int_list(request, 'recipe_id', required=False) recipe_type_ids = rest_util.parse_int_list(request, 'recipe_type_id', required=False) recipe_job = rest_util.parse_string(request, 'recipe_job', required=False) order = rest_util.parse_string_list(request, 'order', required=False) products = SourceFile.objects.get_source_products( source_id, started=started, ended=ended, time_field=time_field, batch_ids=batch_ids, job_type_ids=job_type_ids, job_type_names=job_type_names, job_type_categories=job_type_categories, job_ids=job_ids, is_operational=is_operational, is_published=is_published, file_name=file_name, job_output=job_output, recipe_ids=recipe_ids, recipe_type_ids=recipe_type_ids, recipe_job=recipe_job, order=order) page = self.paginate_queryset(products) serializer = self.get_serializer(page, many=True) return self.get_paginated_response(serializer.data)
def get(self, request, recipe_id): """Retrieve detailed information about the input files for a recipe -*-*- parameters: - name: recipe_id in: path description: The ID of the recipe the file is associated with required: true example: 113 - name: started in: query description: The start time of a start/end time range required: false example: 2016-01-01T00:00:00Z - name: ended in: query description: The end time of a start/end time range required: false example: 2016-01-02T00:00:00Z - name: time_field in: query description: 'The database time field to apply `started` and `ended` time filters [Valid fields: `source`, `data`, `last_modified`]' required: false example: source - name: file_name in: query description: The name of a specific file in Scale required: false example: some_file_i_need_to_find.zip - name: recipe_input in: query description: The name of the input the file is passed to in a recipe required: false example: input_1 responses: '200': description: A JSON list of files with metadata -*-*- :param request: the HTTP GET request :type request: :class:`rest_framework.request.Request` :param recipe_id: The ID for the recipe. :type recipe_id: int encoded as a str :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user """ started = rest_util.parse_timestamp(request, 'started', required=False) ended = rest_util.parse_timestamp(request, 'ended', required=False) rest_util.check_time_range(started, ended) time_field = rest_util.parse_string(request, 'time_field', required=False, accepted_values=ScaleFile.VALID_TIME_FIELDS) file_name = rest_util.parse_string(request, 'file_name', required=False) recipe_input = rest_util.parse_string(request, 'recipe_input', required=False) files = RecipeInputFile.objects.get_recipe_input_files(recipe_id, started=started, ended=ended, time_field=time_field, file_name=file_name, recipe_input=recipe_input) page = self.paginate_queryset(files) serializer = self.get_serializer(page, many=True) return self.get_paginated_response(serializer.data)
def list(self, request): """Retrieves the product for a given file name and returns it in JSON form :param request: the HTTP GET request :type request: :class:`rest_framework.request.Request` :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user """ started = rest_util.parse_timestamp(request, 'started', required=False) ended = rest_util.parse_timestamp(request, 'ended', required=False) rest_util.check_time_range(started, ended) time_field = rest_util.parse_string( request, 'time_field', required=False, accepted_values=ProductFile.VALID_TIME_FIELDS) job_type_ids = rest_util.parse_int_list(request, 'job_type_id', required=False) job_type_names = rest_util.parse_string_list(request, 'job_type_name', required=False) job_type_categories = rest_util.parse_string_list(request, 'job_type_category', required=False) job_ids = rest_util.parse_int_list(request, 'job_id', required=False) is_operational = rest_util.parse_bool(request, 'is_operational', required=False) is_published = rest_util.parse_bool(request, 'is_published', default_value=True) file_name = rest_util.parse_string(request, 'file_name', required=False) job_output = rest_util.parse_string(request, 'job_output', required=False) recipe_ids = rest_util.parse_int_list(request, 'recipe_id', required=False) recipe_type_ids = rest_util.parse_int_list(request, 'recipe_type_id', required=False) recipe_job = rest_util.parse_string(request, 'recipe_job', required=False) batch_ids = rest_util.parse_int_list(request, 'batch_id', required=False) order = rest_util.parse_string_list(request, 'order', required=False) products = ProductFile.objects.get_products( started=started, ended=ended, time_field=time_field, job_type_ids=job_type_ids, job_type_names=job_type_names, job_type_categories=job_type_categories, job_ids=job_ids, is_operational=is_operational, is_published=is_published, file_name=file_name, job_output=job_output, recipe_ids=recipe_ids, recipe_type_ids=recipe_type_ids, recipe_job=recipe_job, batch_ids=batch_ids, order=order, ) page = self.paginate_queryset(products) serializer = self.get_serializer(page, many=True) return self.get_paginated_response(serializer.data)