def post(self, request, recipe_id): """Schedules a recipe for reprocessing and returns it in JSON form :param request: the HTTP GET request :type request: :class:`rest_framework.request.Request` :param recipe_id: The id of the recipe :type recipe_id: int encoded as a str :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user """ job_names = rest_util.parse_string_list(request, 'job_names', required=False) all_jobs = rest_util.parse_bool(request, 'all_jobs', required=False) priority = rest_util.parse_int(request, 'priority', required=False) try: handler = Recipe.objects.reprocess_recipe(recipe_id, job_names, all_jobs, priority) except Recipe.DoesNotExist: raise Http404 except ReprocessError as err: raise BadParameter(unicode(err)) try: new_recipe = Recipe.objects.get_details(handler.recipe.id) except Recipe.DoesNotExist: raise Http404 url = reverse('recipe_details_view', args=[new_recipe.id], request=request) serializer = self.get_serializer(new_recipe) return Response(serializer.data, status=status.HTTP_201_CREATED, headers=dict(location=url))
def post(self, request): """Queue a recipe and returns the new job information in JSON form :param request: the HTTP POST request :type request: :class:`rest_framework.request.Request` :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user """ recipe_type_id = rest_util.parse_int(request, 'recipe_type_id') recipe_data = rest_util.parse_dict(request, 'recipe_data', {}) try: recipe_type = RecipeType.objects.get(pk=recipe_type_id) except RecipeType.DoesNotExist: raise Http404 try: handler = Queue.objects.queue_new_recipe_for_user(recipe_type, RecipeData(recipe_data)) except InvalidRecipeData as err: return Response('Invalid recipe data: ' + unicode(err), status=status.HTTP_400_BAD_REQUEST) try: recipe = Recipe.objects.get_details(handler.recipe.id) except Recipe.DoesNotExist: raise Http404 serializer = self.get_serializer(recipe) recipe_url = reverse('recipe_details_view', args=[recipe.id], request=request) return Response(serializer.data, status=status.HTTP_201_CREATED, headers=dict(location=recipe_url))
def queue_bake_jobs(self, request): """Creates and queues the specified number of Scale Bake jobs :param request: the HTTP POST request :type request: :class:`rest_framework.request.Request` :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user """ num = rest_util.parse_int(request, 'num') if num < 1: raise BadParameter('num must be at least 1') # TODO: in the future, send command message to do this asynchronously try: recipe_type = RecipeType.objects.get(name='scale-bake', revision_num='1') for _ in xrange(num): Queue.objects.queue_new_recipe_for_user_v6(recipe_type, Data()) except (InvalidData, InvalidRecipeData, InactiveRecipeType) as ex: message = 'Unable to create new recipe' logger.exception(message) raise BadParameter('%s: %s' % (message, unicode(ex))) return Response(status=status.HTTP_202_ACCEPTED)
def post(self, request): """Queue a recipe and returns the new job information in JSON form :param request: the HTTP POST request :type request: :class:`rest_framework.request.Request` :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user """ recipe_type_id = rest_util.parse_int(request, 'recipe_type_id') recipe_data = rest_util.parse_dict(request, 'recipe_data', {}) try: recipe_type = RecipeType.objects.get(pk=recipe_type_id) except RecipeType.DoesNotExist: raise Http404 try: handler = Queue.objects.queue_new_recipe_for_user(recipe_type, LegacyRecipeData(recipe_data)) except InvalidRecipeData as err: return Response('Invalid recipe data: ' + unicode(err), status=status.HTTP_400_BAD_REQUEST) try: # TODO: remove this check when REST API v5 is removed if request.version == 'v6': recipe = Recipe.objects.get_details(handler.recipe.id) else: recipe = Recipe.objects.get_details_v5(handler.recipe.id) except Recipe.DoesNotExist: raise Http404 serializer = self.get_serializer(recipe) recipe_url = reverse('recipe_details_view', args=[recipe.id], request=request) return Response(serializer.data, status=status.HTTP_201_CREATED, headers=dict(location=recipe_url))
def post(self, request): """Queue a recipe and returns the new job information in JSON form :param request: the HTTP POST request :type request: :class:`rest_framework.request.Request` :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user """ recipe_type_id = rest_util.parse_int(request, 'recipe_type_id') recipe_data = rest_util.parse_dict(request, 'recipe_data', {}) try: recipe_type = RecipeType.objects.get(pk=recipe_type_id) except RecipeType.DoesNotExist: raise Http404 try: recipe_id = Queue.objects.queue_new_recipe_for_user( recipe_type, recipe_data) except InvalidData: return Response('Invalid recipe information.', status=status.HTTP_400_BAD_REQUEST) recipe_details = Recipe.objects.get_details(recipe_id) serializer = self.get_serializer(recipe_details) recipe_url = urlresolvers.reverse('recipe_details_view', args=[recipe_id]) return Response(serializer.data, status=status.HTTP_201_CREATED, headers=dict(location=recipe_url))
def post(self, request): """Creates a new job, places it on the queue, and returns the new job information in JSON form :param request: the HTTP POST request :type request: :class:`rest_framework.request.Request` :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user """ job_type_id = rest_util.parse_int(request, 'job_type_id') job_data = rest_util.parse_dict(request, 'job_data', {}) try: job_type = JobType.objects.get(pk=job_type_id) except JobType.DoesNotExist: raise Http404 try: job_id, job_exe_id = Queue.objects.queue_new_job_for_user(job_type, job_data) except InvalidData: return Response('Invalid job information.', status=status.HTTP_400_BAD_REQUEST) job_details = Job.objects.get_details(job_id) serializer = JobDetailsSerializer(job_details, context={'request': request}) job_exe_url = urlresolvers.reverse('job_execution_details_view', args=[job_exe_id]) return Response(serializer.data, status=status.HTTP_201_CREATED, headers=dict(location=job_exe_url))
def post(self, request): """Creates a new job, places it on the queue, and returns the new job information in JSON form :param request: the HTTP POST request :type request: :class:`rest_framework.request.Request` :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user """ job_type_id = rest_util.parse_int(request, 'job_type_id') job_data = rest_util.parse_dict(request, 'job_data', {}) try: job_type = JobType.objects.get(pk=job_type_id) except JobType.DoesNotExist: raise Http404 try: job_id = Queue.objects.queue_new_job_for_user(job_type, job_data) except InvalidData as err: logger.exception('Invalid job data.') return Response('Invalid job data: ' + unicode(err), status=status.HTTP_400_BAD_REQUEST) try: # TODO: remove this check when REST API v5 is removed. if request.version == 'v6': job_details = Job.objects.get_details(job_id) else: job_details = Job.objects.get_details_v5(job_id) except Job.DoesNotExist: raise Http404 serializer = self.get_serializer(job_details) job_url = reverse('job_details_view', args=[job_id], request=request) return Response(serializer.data, status=status.HTTP_201_CREATED, headers=dict(location=job_url))
def post(self, request): """Queue a recipe and returns the new job information in JSON form :param request: the HTTP POST request :type request: :class:`rest_framework.request.Request` :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user """ recipe_type_id = rest_util.parse_int(request, 'recipe_type_id') recipe_data = rest_util.parse_dict(request, 'recipe_data', {}) try: recipe_type = RecipeType.objects.get(pk=recipe_type_id) except RecipeType.DoesNotExist: raise Http404 try: recipe_id = Queue.objects.queue_new_recipe_for_user(recipe_type, recipe_data) except InvalidData: return Response('Invalid recipe information.', status=status.HTTP_400_BAD_REQUEST) recipe_details = Recipe.objects.get_details(recipe_id) serializer = RecipeDetailsSerializer(recipe_details, context={'request': request}) recipe_url = urlresolvers.reverse('recipe_details_view', args=[recipe_id]) return Response(serializer.data, status=status.HTTP_201_CREATED, headers=dict(location=recipe_url))
def post(self, request): """Creates a new job, places it on the queue, and returns the new job information in JSON form :param request: the HTTP POST request :type request: :class:`rest_framework.request.Request` :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user """ job_type_id = rest_util.parse_int(request, 'job_type_id') job_data = rest_util.parse_dict(request, 'job_data', {}) try: job_type = JobType.objects.get(pk=job_type_id) except JobType.DoesNotExist: raise Http404 try: job_id, job_exe_id = Queue.objects.queue_new_job_for_user( job_type, job_data) except InvalidData: return Response('Invalid job information.', status=status.HTTP_400_BAD_REQUEST) job_details = Job.objects.get_details(job_id) serializer = self.get_serializer(job_details) job_exe_url = urlresolvers.reverse('job_execution_details_view', args=[job_exe_id]) return Response(serializer.data, status=status.HTTP_201_CREATED, headers=dict(location=job_exe_url))
def post(self, request): """Validates a new batch and returns any warnings discovered :param request: the HTTP POST request :type request: :class:`rest_framework.request.Request` :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user """ recipe_type_id = rest_util.parse_int(request, 'recipe_type_id') # Make sure the recipe type exists try: recipe_type = RecipeType.objects.get(pk=recipe_type_id) except RecipeType.DoesNotExist: raise BadParameter('Unknown recipe type: %i' % recipe_type_id) # Validate the batch definition definition_dict = rest_util.parse_dict(request, 'definition') definition = None try: if definition_dict: definition = BatchDefinition(definition_dict) except InvalidDefinition as ex: raise BadParameter('Batch definition invalid: %s' % unicode(ex)) # Get a rough estimate of how many recipes will be affected old_recipes = Batch.objects.get_matched_recipes(recipe_type, definition) return Response({ 'recipe_count': old_recipes.count(), 'warnings': [], })
def test_parse_int_default(self): """Tests parsing a required int parameter that is provided via default value.""" request = MagicMock(Request) request.query_params = QueryDict('', mutable=True) request.query_params.update({ 'test': '10', }) self.assertEqual(rest_util.parse_int(request, 'test2', 20), 20)
def test_parse_int_zero(self): """Tests parsing an optional int parameter zero instead of using the default value.""" request = MagicMock(Request) request.query_params = QueryDict('', mutable=True) request.query_params.update({ 'test': '0', }) self.assertEqual(rest_util.parse_int(request, 'test', 10), 0)
def test_parse_int_post(self): """Tests parsing a required int parameter that is provided via POST.""" request = MagicMock(Request) request.data = QueryDict('', mutable=True) request.data.update({ 'test': '10', }) self.assertEqual(rest_util.parse_int(request, 'test'), 10)
def test_parse_int_post(self): '''Tests parsing a required int parameter that is provided via POST.''' request = MagicMock(Request) request.DATA = QueryDict('', mutable=True) request.DATA.update({ 'test': '10', }) self.assertEqual(rest_util.parse_int(request, 'test'), 10)
def test_parse_int_default(self): '''Tests parsing a required int parameter that is provided via default value.''' request = MagicMock(Request) request.QUERY_PARAMS = QueryDict('', mutable=True) request.QUERY_PARAMS.update({ 'test': '10', }) self.assertEqual(rest_util.parse_int(request, 'test2', 20), 20)
def test_parse_int(self): """Tests parsing a required int parameter that is provided via GET.""" request = MagicMock(Request) request.query_params = QueryDict('', mutable=True) request.query_params.update({ 'test': '10', }) self.assertEqual(rest_util.parse_int(request, 'test'), 10)
def test_parse_int(self): '''Tests parsing a required int parameter that is provided via GET.''' request = MagicMock(Request) request.query_params = QueryDict('', mutable=True) request.query_params.update({ 'test': '10', }) self.assertEqual(rest_util.parse_int(request, 'test'), 10)
def test_parse_int_optional(self): '''Tests parsing an optional int parameter that is missing.''' request = MagicMock(Request) request.QUERY_PARAMS = QueryDict('', mutable=True) request.QUERY_PARAMS.update({ 'test': 'value1', }) self.assertIsNone(rest_util.parse_int(request, 'test2', required=False))
def test_parse_int_zero(self): '''Tests parsing an optional int parameter zero instead of using the default value.''' request = MagicMock(Request) request.QUERY_PARAMS = QueryDict('', mutable=True) request.QUERY_PARAMS.update({ 'test': '0', }) self.assertEqual(rest_util.parse_int(request, 'test', 10), 0)
def post(self, request): """Queue a recipe and returns the new job information in JSON form :param request: the HTTP POST request :type request: :class:`rest_framework.request.Request` :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user """ if request.version != 'v6': raise Http404 recipe_type_id = rest_util.parse_int(request, 'recipe_type_id') recipe_data = rest_util.parse_dict(request, 'input', {}) configuration_dict = rest_util.parse_dict(request, 'configuration', required=False) configuration = None try: recipeData = DataV6(recipe_data, do_validate=True) except InvalidData as ex: logger.exception('Unable to queue new recipe. Invalid input: %s', recipe_data) raise BadParameter(unicode(ex)) try: recipe_type = RecipeType.objects.get(pk=recipe_type_id) except RecipeType.DoesNotExist: raise Http404 if configuration_dict: try: configuration = RecipeConfigurationV6( configuration_dict, do_validate=True).get_configuration() except InvalidRecipeConfiguration as ex: message = 'Recipe configuration invalid' logger.exception(message) raise BadParameter('%s: %s' % (message, unicode(ex))) try: recipe = Queue.objects.queue_new_recipe_for_user_v6( recipe_type, recipeData.get_data(), recipe_config=configuration) except (InvalidData, InvalidRecipeData) as err: return Response('Invalid recipe data: ' + unicode(err), status=status.HTTP_400_BAD_REQUEST) except InactiveRecipeType as err: return Response('Inactive recipe type: ' + unicode(err), status=status.HTTP_400_BAD_REQUEST) serializer = RecipeSerializerV6(recipe) recipe_url = reverse('recipe_details_view', args=[recipe.id], request=request) return Response(serializer.data, status=status.HTTP_201_CREATED, headers=dict(location=recipe_url))
def _post_v6(self, request): """The v6 version for validating a new batch :param request: the HTTP POST request :type request: :class:`rest_framework.request.Request` :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user """ recipe_type_id = rest_util.parse_int(request, 'recipe_type_id') definition_dict = rest_util.parse_dict(request, 'definition') configuration_dict = rest_util.parse_dict(request, 'configuration', required=False) # Make sure the recipe type exists try: recipe_type = RecipeType.objects.get(pk=recipe_type_id) except RecipeType.DoesNotExist: raise BadParameter('Unknown recipe type: %d' % recipe_type_id) try: definition = BatchDefinitionV6(definition=definition_dict, do_validate=True).get_definition() configuration = BatchConfigurationV6( configuration=configuration_dict, do_validate=True).get_configuration() except InvalidDefinition as ex: raise BadParameter(unicode(ex)) except InvalidConfiguration as ex: raise BadParameter(unicode(ex)) # Validate the batch validation = Batch.objects.validate_batch_v6( recipe_type, definition, configuration=configuration) batch = validation.batch recipe_type_serializer = RecipeTypeBaseSerializerV6(batch.recipe_type) resp_dict = { 'is_valid': validation.is_valid, 'errors': [e.to_dict() for e in validation.errors], 'warnings': [w.to_dict() for w in validation.warnings], 'recipes_estimated': definition.estimated_recipes, 'recipe_type': recipe_type_serializer.data } if batch.superseded_batch: recipe_type_rev_serializer = RecipeTypeRevisionBaseSerializerV6( batch.superseded_batch.recipe_type_rev) prev_batch_dict = { 'recipe_type_rev': recipe_type_rev_serializer.data } resp_dict['prev_batch'] = prev_batch_dict if definition.prev_batch_diff: diff_v6 = convert_recipe_diff_to_v6_json( definition.prev_batch_diff) diff_dict = rest_util.strip_schema_version(diff_v6.get_dict()) prev_batch_dict['diff'] = diff_dict return Response(resp_dict)
def test_parse_int_optional(self): """Tests parsing an optional int parameter that is missing.""" request = MagicMock(Request) request.query_params = QueryDict('', mutable=True) request.query_params.update({ 'test': 'value1', }) self.assertIsNone(rest_util.parse_int(request, 'test2', required=False))
def test_parse_int_accepted_all(self): '''Tests parsing an int parameter where the value is acceptable.''' request = MagicMock(Request) request.QUERY_PARAMS = QueryDict('', mutable=True) request.QUERY_PARAMS.update({ 'test': '1', }) self.assertEqual(rest_util.parse_int(request, 'test', accepted_values=[1, 2, 3]), 1)
def _create_v6(self, request): """The v6 version for creating batches :param request: the HTTP POST request :type request: :class:`rest_framework.request.Request` :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user """ title = rest_util.parse_string(request, 'title', required=False) description = rest_util.parse_string(request, 'description', required=False) recipe_type_id = rest_util.parse_int(request, 'recipe_type_id') definition_dict = rest_util.parse_dict(request, 'definition') configuration_dict = rest_util.parse_dict(request, 'configuration', required=False) # Make sure the recipe type exists try: recipe_type = RecipeType.objects.get(pk=recipe_type_id) except RecipeType.DoesNotExist: raise BadParameter('Unknown recipe type: %d' % recipe_type_id) # Validate and create the batch try: definition = BatchDefinitionV6(definition=definition_dict, do_validate=True).get_definition() configuration = BatchConfigurationV6( configuration=configuration_dict, do_validate=True).get_configuration() with transaction.atomic(): event = TriggerEvent.objects.create_trigger_event( 'USER', None, {'user': '******'}, now()) batch = Batch.objects.create_batch_v6( title, description, recipe_type, event, definition, configuration=configuration) CommandMessageManager().send_messages( [create_batch_recipes_message(batch.id)]) except InvalidDefinition as ex: raise BadParameter(unicode(ex)) except InvalidConfiguration as ex: raise BadParameter(unicode(ex)) # Fetch the full batch with details batch = Batch.objects.get_details_v6(batch.id) url = reverse('batch_details_view', args=[batch.id], request=request) serializer = BatchDetailsSerializerV6(batch) return Response(serializer.data, status=status.HTTP_201_CREATED, headers={'Location': url})
def test_parse_int_accepted_all(self): """Tests parsing an int parameter where the value is acceptable.""" request = MagicMock(Request) request.query_params = QueryDict('', mutable=True) request.query_params.update({ 'test': '1', }) self.assertEqual( rest_util.parse_int(request, 'test', accepted_values=[1, 2, 3]), 1)
def post(self, request): """Increase max_tries, place it on the queue, and returns the new job information in JSON form :param request: the HTTP GET request :type request: :class:`rest_framework.request.Request` :returns: the HTTP response to send back to the user """ if request.version == 'v6': raise Http404 started = rest_util.parse_timestamp(request, 'started', required=False) ended = rest_util.parse_timestamp(request, 'ended', required=False) rest_util.check_time_range(started, ended) job_status = rest_util.parse_string(request, 'status', required=False) job_ids = rest_util.parse_int_list(request, 'job_ids', required=False) job_type_ids = rest_util.parse_int_list(request, 'job_type_ids', required=False) job_type_names = rest_util.parse_string_list(request, 'job_type_names', required=False) job_type_categories = rest_util.parse_string_list( request, 'job_type_categories', required=False) error_categories = rest_util.parse_string_list(request, 'error_categories', required=False) priority = rest_util.parse_int(request, 'priority', required=False) # Fetch all the jobs matching the filters job_status = [job_status] if job_status else job_status jobs = Job.objects.get_jobs(started=started, ended=ended, statuses=job_status, job_ids=job_ids, job_type_ids=job_type_ids, job_type_names=job_type_names, job_type_categories=job_type_categories, error_categories=error_categories) # Attempt to queue all jobs matching the filters requested_job_ids = {job.id for job in jobs} if requested_job_ids: Queue.objects.requeue_jobs(requested_job_ids, priority) # Refresh models to get the new status information for all originally requested jobs jobs = Job.objects.get_jobs(job_ids=requested_job_ids) page = self.paginate_queryset(jobs) serializer = JobSerializerV5(page, many=True) return self.get_paginated_response(serializer.data)
def patch(self, request, name, version): """Edits an existing seed job type and returns the updated details :param request: the HTTP PATCH request :type request: :class:`rest_framework.request.Request` :param job_type_id: The ID for the job type. :type job_type_id: int encoded as a str :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user """ auto_update = rest_util.parse_bool(request, 'auto_update', required=False) icon_code = rest_util.parse_string(request, 'icon_code', required=False) is_published = rest_util.parse_string(request, 'is_published', required=False) is_active = rest_util.parse_bool(request, 'is_active', required=False) is_paused = rest_util.parse_bool(request, 'is_paused', required=False) max_scheduled = rest_util.parse_int(request, 'max_scheduled', required=False) # Validate the job configuration and pull out secrets configuration_dict = rest_util.parse_dict(request, 'configuration', required=False) configuration = None try: if configuration_dict: configuration = JobConfigurationV6(configuration_dict).get_configuration() except InvalidJobConfiguration as ex: raise BadParameter('Job type configuration invalid: %s' % unicode(ex)) # Fetch the current job type model try: job_type = JobType.objects.get(name=name, version=version) except JobType.DoesNotExist: raise Http404 # Check for invalid fields fields = {'icon_code', 'is_published', 'is_active', 'is_paused', 'max_scheduled', 'configuration'} for key, value in request.data.iteritems(): if key not in fields: raise InvalidJobField try: with transaction.atomic(): # Edit the job type JobType.objects.edit_job_type_v6(job_type_id=job_type.id, manifest=None, is_published=is_published, docker_image=None, icon_code=icon_code, is_active=is_active, is_paused=is_paused, max_scheduled=max_scheduled, configuration=configuration, auto_update=auto_update) except (InvalidJobField, InvalidSecretsConfiguration, ValueError, InvalidJobConfiguration, InvalidInterfaceDefinition) as ex: logger.exception('Unable to update job type: %i', job_type.id) raise BadParameter(unicode(ex)) return HttpResponse(status=204)
def post(self, request): """Creates a new job, places it on the queue, and returns the new job information in JSON form :param request: the HTTP POST request :type request: :class:`rest_framework.request.Request` :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user """ job_type_id = rest_util.parse_int(request, 'job_type_id') job_data = rest_util.parse_dict(request, 'input', {}) configuration_dict = rest_util.parse_dict(request, 'configuration', required=False) configuration = None try: jobData = DataV6(job_data, do_validate=True) except InvalidData as ex: logger.exception('Unable to queue new job. Invalid input: %s', job_data) raise BadParameter(unicode(ex)) try: job_type = JobType.objects.get(pk=job_type_id) except JobType.DoesNotExist: raise Http404 if configuration_dict: try: existing = convert_config_to_v6_json(job_type.get_job_configuration()) configuration = JobConfigurationV6(configuration_dict, existing=existing, do_validate=True).get_configuration() except InvalidJobConfiguration as ex: message = 'Job type configuration invalid' logger.exception(message) raise BadParameter('%s: %s' % (message, unicode(ex))) try: job_id = Queue.objects.queue_new_job_for_user_v6(job_type=job_type, job_data=jobData.get_data(), job_configuration=configuration) CommandMessageManager().send_messages(create_process_job_input_messages([job_id])) except InvalidData as err: logger.exception('Invalid job data.') return Response('Invalid job data: ' + unicode(err), status=status.HTTP_400_BAD_REQUEST) try: job_details = Job.objects.get_details(job_id) except Job.DoesNotExist: raise Http404 serializer = JobDetailsSerializerV6(job_details) job_url = reverse('job_details_view', args=[job_id], request=request) return Response(serializer.data, status=status.HTTP_201_CREATED, headers=dict(location=job_url))
def _create_v5(self, request): """The v5 version for creating batches :param request: the HTTP POST request :type request: :class:`rest_framework.request.Request` :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user """ recipe_type_id = rest_util.parse_int(request, 'recipe_type_id') title = rest_util.parse_string(request, 'title', required=False) description = rest_util.parse_string(request, 'description', required=False) # Make sure the recipe type exists try: recipe_type = RecipeType.objects.get(pk=recipe_type_id) except RecipeType.DoesNotExist: raise BadParameter('Unknown recipe type: %i' % recipe_type_id) # Validate the batch definition definition_dict = rest_util.parse_dict(request, 'definition') definition = None try: if definition_dict: definition = OldBatchDefinition(definition_dict) definition.validate(recipe_type) except InvalidDefinition as ex: raise BadParameter('Batch definition invalid: %s' % unicode(ex)) # Create the batch batch = Batch.objects.create_batch_old(recipe_type, definition, title=title, description=description) # Fetch the full batch with details try: batch = Batch.objects.get_details_v5(batch.id) except Batch.DoesNotExist: raise Http404 url = reverse('batch_details_view', args=[batch.id], request=request) serializer = BatchDetailsSerializerV5(batch) return Response(serializer.data, status=status.HTTP_201_CREATED, headers=dict(location=url))
def post(self, request): """Kicks off the process of purging a given source file from Scale :param request: the HTTP POST request :type request: :class:`rest_framework.request.Request` :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user """ if self.request.version != 'v6' and self.request.version != 'v7': content = 'This endpoint is supported with REST API v6+' return Response(status=status.HTTP_400_BAD_REQUEST, data=content) file_id = rest_util.parse_int(request, 'file_id') try: file_id = int(file_id) except ValueError: content = 'The given file_id is not valid: %i' % (file_id) return Response(status=status.HTTP_400_BAD_REQUEST, data=content) # Attempt to fetch the ScaleFile model try: source_file = ScaleFile.objects.get(id=file_id) except ScaleFile.DoesNotExist: content = 'No file record exists for the given file_id: %i' % ( file_id) return Response(status=status.HTTP_400_BAD_REQUEST, data=content) # Inspect the file to ensure it will purge correctly if source_file.file_type != 'SOURCE': content = 'The given file_id does not correspond to a SOURCE file_type: %i' % ( file_id) return Response(status=status.HTTP_400_BAD_REQUEST, data=content) event = TriggerEvent.objects.create_trigger_event( 'USER', None, {'user': '******'}, now()) PurgeResults.objects.create(source_file_id=file_id, trigger_event=event) CommandMessageManager().send_messages([ create_purge_source_file_message(source_file_id=file_id, trigger_id=event.id) ]) return Response(status=status.HTTP_204_NO_CONTENT)
def post(self, request, recipe_id): """Schedules a recipe for reprocessing and returns it in JSON form :param request: the HTTP GET request :type request: :class:`rest_framework.request.Request` :param recipe_id: The id of the recipe :type recipe_id: int encoded as a str :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user """ job_names = rest_util.parse_string_list(request, 'job_names', required=False) all_jobs = rest_util.parse_bool(request, 'all_jobs', required=False) priority = rest_util.parse_int(request, 'priority', required=False) try: handler = Recipe.objects.reprocess_recipe(recipe_id, job_names=job_names, all_jobs=all_jobs, priority=priority) except Recipe.DoesNotExist: raise Http404 except ReprocessError as err: raise BadParameter(unicode(err)) try: # TODO: remove this check when REST API v5 is removed if request.version == 'v6': new_recipe = Recipe.objects.get_details(handler.recipe.id) else: new_recipe = Recipe.objects.get_details_v5(handler.recipe.id) except Recipe.DoesNotExist: raise Http404 serializer = self.get_serializer(new_recipe) url = reverse('recipe_details_view', args=[new_recipe.id], request=request) return Response(serializer.data, status=status.HTTP_201_CREATED, headers=dict(location=url))
def post(self, request): """Submit command message to re-queue jobs that fit the given filter criteria :param request: the HTTP GET request :type request: :class:`rest_framework.request.Request` :returns: the HTTP response to send back to the user """ started = rest_util.parse_timestamp(request, 'started', required=False) ended = rest_util.parse_timestamp(request, 'ended', required=False) rest_util.check_time_range(started, ended) error_categories = rest_util.parse_string_list(request, 'error_categories', required=False) error_ids = rest_util.parse_int_list(request, 'error_ids', required=False) job_ids = rest_util.parse_int_list(request, 'job_ids', required=False) job_status = rest_util.parse_string(request, 'status', required=False) job_type_ids = rest_util.parse_int_list(request, 'job_type_ids', required=False) priority = rest_util.parse_int(request, 'priority', required=False) job_type_names = rest_util.parse_string_list(request, 'job_type_names', required=False) batch_ids = rest_util.parse_int_list(request, 'batch_ids', required=False) recipe_ids = rest_util.parse_int_list(request, 'recipe_ids', required=False) is_superseded = rest_util.parse_bool(request, 'is_superseded', required=False) job_types = rest_util.parse_dict_list(request, 'job_types', required=False) for jt in job_types: if 'name' not in jt or 'version' not in jt: raise BadParameter('Job types argument invalid: %s' % job_types) existing_job_type = JobType.objects.filter(name=jt['name'], version=jt['version']).first() if not existing_job_type: raise BadParameter( 'Job Type with name: %s and version: %s does not exist' % (jt['name'], jt['version'])) job_type_ids.append(existing_job_type.id) # Create and send message msg = create_requeue_jobs_bulk_message(started=started, ended=ended, error_categories=error_categories, error_ids=error_ids, job_ids=job_ids, job_type_ids=job_type_ids, priority=priority, status=job_status, job_type_names=job_type_names, batch_ids=batch_ids, recipe_ids=recipe_ids, is_superseded=is_superseded) CommandMessageManager().send_messages([msg]) return Response(status=status.HTTP_202_ACCEPTED)
def post(self, request): """Creates and queues the specified number of Scale Roulette jobs :param request: the HTTP POST request :type request: :class:`rest_framework.request.Request` :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user """ num = rest_util.parse_int(request, 'num') if num < 1: raise BadParameter('num must be at least 1') # TODO: in the future, send command message to do this asynchronously job_type = JobType.objects.get(name='scale-roulette', version='1.0') for _ in xrange(num): Queue.objects.queue_new_job_for_user(job_type, {}) return Response(status=status.HTTP_202_ACCEPTED)
def post(self, request): """Increase max_tries, place it on the queue, and returns the new job information in JSON form :param request: the HTTP GET request :type request: :class:`rest_framework.request.Request` :returns: the HTTP response to send back to the user """ started = rest_util.parse_timestamp(request, 'started', required=False) ended = rest_util.parse_timestamp(request, 'ended', required=False) rest_util.check_time_range(started, ended) job_status = rest_util.parse_string(request, 'status', required=False) job_ids = rest_util.parse_int_list(request, 'job_ids', required=False) job_type_ids = rest_util.parse_int_list(request, 'job_type_ids', required=False) job_type_names = rest_util.parse_string_list(request, 'job_type_names', required=False) job_type_categories = rest_util.parse_string_list(request, 'job_type_categories', required=False) error_categories = rest_util.parse_string_list(request, 'error_categories', required=False) priority = rest_util.parse_int(request, 'priority', required=False) # Fetch all the jobs matching the filters job_status = [job_status] if job_status else job_status jobs = Job.objects.get_jobs(started=started, ended=ended, statuses=job_status, job_ids=job_ids, job_type_ids=job_type_ids, job_type_names=job_type_names, job_type_categories=job_type_categories, error_categories=error_categories) if not jobs: raise Http404 # Attempt to queue all jobs matching the filters requested_job_ids = {job.id for job in jobs} Queue.objects.requeue_jobs(requested_job_ids, priority) # Refresh models to get the new status information for all originally requested jobs jobs = Job.objects.get_jobs(job_ids=requested_job_ids) page = self.paginate_queryset(jobs) serializer = self.get_serializer(page, many=True) return self.get_paginated_response(serializer.data)
def queue_casino_recipes(self, request): """Creates and queues the specified number of Scale Casino recipes :param request: the HTTP POST request :type request: :class:`rest_framework.request.Request` :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user """ num = rest_util.parse_int(request, 'num') if num < 1: raise BadParameter('num must be at least 1') # TODO: in the future, send command message to do this asynchronously recipe_type = RecipeType.objects.get(name='scale-casino', version='1.0') for _ in xrange(num): Queue.objects.queue_new_recipe_for_user(recipe_type, LegacyRecipeData()) return Response(status=status.HTTP_202_ACCEPTED)
def create(self, request): """Creates a new batch and returns a link to the detail URL :param request: the HTTP POST request :type request: :class:`rest_framework.request.Request` :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user """ recipe_type_id = rest_util.parse_int(request, 'recipe_type_id') title = rest_util.parse_string(request, 'title', required=False) description = rest_util.parse_string(request, 'description', required=False) # Make sure the recipe type exists try: recipe_type = RecipeType.objects.get(pk=recipe_type_id) except RecipeType.DoesNotExist: raise BadParameter('Unknown recipe type: %i' % recipe_type_id) # Validate the batch definition definition_dict = rest_util.parse_dict(request, 'definition') definition = None try: if definition_dict: definition = BatchDefinition(definition_dict) except InvalidDefinition as ex: raise BadParameter('Batch definition invalid: %s' % unicode(ex)) # Create the batch batch = Batch.objects.create_batch(recipe_type, definition, title=title, description=description) # Fetch the full batch with details try: batch = Batch.objects.get_details(batch.id) except Batch.DoesNotExist: raise Http404 url = reverse('batch_details_view', args=[batch.id], request=request) serializer = BatchDetailsSerializer(batch) return Response(serializer.data, status=status.HTTP_201_CREATED, headers=dict(location=url))
def _post_v5(self, request): """The v5 version for validating a new batch :param request: the HTTP POST request :type request: :class:`rest_framework.request.Request` :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user """ recipe_type_id = rest_util.parse_int(request, 'recipe_type_id') # Make sure the recipe type exists try: recipe_type = RecipeType.objects.get(pk=recipe_type_id) except RecipeType.DoesNotExist: raise BadParameter('Unknown recipe type: %i' % recipe_type_id) # Validate the batch definition definition_dict = rest_util.parse_dict(request, 'definition') definition = None warnings = [] try: if definition_dict: definition = OldBatchDefinition(definition_dict) warnings = definition.validate(recipe_type) except InvalidDefinition as ex: raise BadParameter('Batch definition invalid: %s' % unicode(ex)) # Get a rough estimate of how many recipes/files will be affected old_recipes = Batch.objects.get_matched_recipes( recipe_type, definition) old_files = Batch.objects.get_matched_files(recipe_type, definition) return Response({ 'recipe_count': old_recipes.count(), 'file_count': old_files.count(), 'warnings': warnings, })
def post(self, request): '''Creates a new JobType and returns its ID in JSON form :param request: the HTTP POST request :type request: :class:`rest_framework.request.Request` :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user ''' name = rest_util.parse_string(request, u'name') version = rest_util.parse_string(request, u'version') title = rest_util.parse_string(request, u'title', default_value=u'Unknown Job Type') description = rest_util.parse_string(request, u'description') category = rest_util.parse_string(request, u'category', default_value=u'unknown') author_name = rest_util.parse_string(request, u'author_name', required=False) author_url = rest_util.parse_string(request, u'author_url', required=False) is_system = False is_long_running = False is_active = rest_util.parse_bool(request, u'is_active', default_value=True) is_operational = rest_util.parse_bool(request, u'is_operational', default_value=True) is_paused = rest_util.parse_bool(request, u'is_paused', default_value=False) requires_cleanup = True uses_docker = True docker_privileged = rest_util.parse_bool(request, u'docker_privileged', default_value=False) docker_image = rest_util.parse_string(request, u'docker_image') interface = rest_util.parse_dict(request, u'interface') error_mapping = rest_util.parse_dict(request, u'error_mapping', default_value={}) priority = rest_util.parse_int(request, u'priority', default_value=260) timeout = rest_util.parse_int(request, u'timeout', default_value=1800) max_tries = rest_util.parse_int(request, u'max_tries', default_value=3) cpus_required = rest_util.parse_float(request, u'cpus_required', default_value=1) mem_required = rest_util.parse_float(request, u'mem_required', default_value=5120) disk_out_const_required = rest_util.parse_float(request, u'disk_out_const_required', default_value=0) disk_out_mult_required = rest_util.parse_float(request, u'disk_out_mult_required', default_value=0) icon_code = rest_util.parse_string(request, u'icon_code', default_value=u'f013') try: try: job_type = JobType.objects.get(name=name, version=version) job_type.description = description job_type.docker_image = docker_image job_type.interface = interface job_type.priority = priority job_type.timeout = timeout job_type.max_tries = max_tries job_type.cpus_required = cpus_required job_type.mem_required = mem_required job_type.disk_out_const_required = disk_out_const_required except JobType.DoesNotExist: job_type = JobType.objects.create_job_type(name, version, description, docker_image, interface, priority, timeout, max_tries, cpus_required, mem_required, disk_out_const_required, None) job_type.title = title job_type.category = category job_type.author_name = author_name job_type.author_url = author_url job_type.is_system = is_system job_type.is_long_running = is_long_running job_type.is_active = is_active job_type.is_operational = is_operational job_type.is_paused = is_paused job_type.requires_cleanup = requires_cleanup job_type.uses_docker = uses_docker job_type.docker_privileged = docker_privileged job_type.error_mapping = error_mapping job_type.icon_code = icon_code job_type.disk_out_mult_required = disk_out_mult_required job_type.save() except InvalidInterfaceDefinition: raise rest_util.BadParameter('Interface definition failed to validate.') return Response({'job_type_id': job_type.id}, status=status.HTTP_200_OK)
def patch(self, request, name, version): """Edits an existing seed job type and returns the updated details :param request: the HTTP PATCH request :type request: :class:`rest_framework.request.Request` :param job_type_id: The ID for the job type. :type job_type_id: int encoded as a str :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user """ auto_update = rest_util.parse_bool(request, 'auto_update', required=False, default_value=True) icon_code = rest_util.parse_string(request, 'icon_code', required=False) is_published = rest_util.parse_string(request, 'is_published', required=False) is_active = rest_util.parse_bool(request, 'is_active', required=False) is_paused = rest_util.parse_bool(request, 'is_paused', required=False) max_scheduled = rest_util.parse_int(request, 'max_scheduled', required=False) docker_image = rest_util.parse_string(request, 'docker_image', required=False) # Validate the manifest manifest_dict = rest_util.parse_dict(request, 'manifest', required=False) manifest = None if manifest_dict: try: manifest = SeedManifest(manifest_dict, do_validate=True) except InvalidSeedManifestDefinition as ex: message = 'Seed Manifest invalid' logger.exception(message) raise BadParameter('%s: %s' % (message, unicode(ex))) # validate manifest name/version matches job type manifest_name = manifest.get_name() if name != manifest_name: raise BadParameter('Manifest name %s does not match current Job Type name %s.' % (manifest_name, name)) manifest_version = manifest.get_job_version() if manifest_version != version: raise BadParameter('Manifest version %s does not match current Job Type version %s.' % (manifest_version, version)) # Validate the job configuration and pull out secrets configuration_dict = rest_util.parse_dict(request, 'configuration', required=False) configuration = None try: if configuration_dict: configuration = JobConfigurationV6(configuration_dict).get_configuration() except InvalidJobConfiguration as ex: raise BadParameter('Job type configuration invalid: %s' % unicode(ex)) # Fetch the current job type model try: job_type = JobType.objects.get(name=name, version=version) except JobType.DoesNotExist: raise Http404 # Check for invalid fields fields = {'icon_code', 'is_published', 'is_active', 'is_paused', 'max_scheduled', 'configuration', 'manifest', 'docker_image', 'auto_update'} for key, value in request.data.iteritems(): if key not in fields: raise BadParameter('%s is not a valid field. Valid fields are: %s' % (key, fields)) try: with transaction.atomic(): # Edit the job type validation = JobType.objects.edit_job_type_v6(job_type_id=job_type.id, manifest=manifest, is_published=is_published, docker_image=docker_image, icon_code=icon_code, is_active=is_active, is_paused=is_paused, max_scheduled=max_scheduled, configuration=configuration, auto_update=auto_update) except (InvalidJobField, InvalidSecretsConfiguration, ValueError, InvalidJobConfiguration, InvalidInterfaceDefinition) as ex: logger.exception('Unable to update job type: %i', job_type.id) raise BadParameter(unicode(ex)) resp_dict = {'is_valid': validation.is_valid, 'errors': [e.to_dict() for e in validation.errors], 'warnings': [w.to_dict() for w in validation.warnings]} return Response(resp_dict)
def create_v6(self, request): """Creates or edits a Seed job type and returns a link to the detail URL :param request: the HTTP POST request :type request: :class:`rest_framework.request.Request` :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user """ # Optional icon code value icon_code = rest_util.parse_string(request, 'icon_code', required=False) # Optional is published value is_published = rest_util.parse_string(request, 'is_published', required=False) # Optional max scheduled value max_scheduled = rest_util.parse_int(request, 'max_scheduled', required=False) # Require docker image value docker_image = rest_util.parse_string(request, 'docker_image', required=True) # Validate the job interface / manifest manifest_dict = rest_util.parse_dict(request, 'manifest', required=True) # If editing an existing job type, automatically update recipes containing said job type auto_update = rest_util.parse_bool(request, 'auto_update', required=False, default_value=True) # Optional setting job type active if editing existing job is_active = rest_util.parse_bool(request, 'is_active', required=False) # Optional setting job type to paused if editing an existing job is_paused = rest_util.parse_bool(request, 'is_paused', required=False) manifest = None try: manifest = SeedManifest(manifest_dict, do_validate=True) except InvalidSeedManifestDefinition as ex: message = 'Seed Manifest invalid' logger.exception(message) raise BadParameter('%s: %s' % (message, unicode(ex))) # Validate the job configuration and pull out secrets configuration_dict = rest_util.parse_dict(request, 'configuration', required=False) configuration = None if configuration_dict: try: configuration = JobConfigurationV6(configuration_dict, do_validate=True).get_configuration() except InvalidJobConfiguration as ex: message = 'Job type configuration invalid' logger.exception(message) raise BadParameter('%s: %s' % (message, unicode(ex))) # Check for invalid fields fields = {'icon_code', 'is_published', 'max_scheduled', 'docker_image', 'configuration', 'manifest', 'auto_update', 'is_active', 'is_paused'} for key, value in request.data.iteritems(): if key not in fields: raise BadParameter('%s is not a valid field. Valid fields are: %s' % (key, fields)) name = manifest_dict['job']['name'] version = manifest_dict['job']['jobVersion'] if name == 'validation': logger.exception('Unable to create job type named "validation"') raise BadParameter(unicode('Unable to create job type named "validation"')) existing_job_type = JobType.objects.filter(name=name, version=version).first() if not existing_job_type: try: # Create the job type job_type = JobType.objects.create_job_type_v6(icon_code=icon_code, is_published=is_published, max_scheduled=max_scheduled, docker_image=docker_image, manifest=manifest, configuration=configuration) except (InvalidJobField, InvalidSecretsConfiguration, ValueError) as ex: message = 'Unable to create new job type' logger.exception(message) raise BadParameter('%s: %s' % (message, unicode(ex))) except InvalidSeedManifestDefinition as ex: message = 'Job type manifest invalid' logger.exception(message) raise BadParameter('%s: %s' % (message, unicode(ex))) except InvalidJobConfiguration as ex: message = 'Job type configuration invalid' logger.exception(message) raise BadParameter('%s: %s' % (message, unicode(ex))) # Fetch the full job type with details try: job_type = JobType.objects.get_details_v6(name=name, version=version) except JobType.DoesNotExist: raise Http404 url = reverse('job_type_details_view', args=[job_type.name, job_type.version], request=request) serializer = JobTypeDetailsSerializerV6(job_type) return Response(serializer.data, status=status.HTTP_201_CREATED, headers=dict(location=url)) else: try: validation = JobType.objects.edit_job_type_v6(job_type_id=existing_job_type.id, manifest=manifest, docker_image=docker_image, icon_code=icon_code, is_active=is_active, is_paused=is_paused, max_scheduled=max_scheduled, is_published=is_published, configuration=configuration, auto_update=auto_update) except (InvalidJobField, InvalidSecretsConfiguration, ValueError, InvalidInterfaceDefinition) as ex: logger.exception('Unable to update job type: %i', existing_job_type.id) raise BadParameter(unicode(ex)) except InvalidSeedManifestDefinition as ex: message = 'Job type manifest invalid' logger.exception(message) raise BadParameter('%s: %s' % (message, unicode(ex))) except InvalidJobConfiguration as ex: message = 'Job type configuration invalid' logger.exception(message) raise BadParameter('%s: %s' % (message, unicode(ex))) resp_dict = {'is_valid': validation.is_valid, 'errors': [e.to_dict() for e in validation.errors], 'warnings': [w.to_dict() for w in validation.warnings]} return Response(resp_dict)
def post(self, request, recipe_id): """Schedules a recipe for reprocessing and returns it in JSON form :param request: the HTTP GET request :type request: :class:`rest_framework.request.Request` :param recipe_id: The id of the recipe :type recipe_id: int encoded as a str :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user """ job_names = rest_util.parse_string_list(request, 'job_names', required=False) all_jobs = rest_util.parse_bool(request, 'all_jobs', required=False) priority = rest_util.parse_int(request, 'priority', required=False) try: recipe = Recipe.objects.select_related('recipe_type', 'recipe_type_rev').get(id=recipe_id) except Recipe.DoesNotExist: raise Http404 if recipe.is_superseded: raise BadParameter('Cannot reprocess a superseded recipe') event = TriggerEvent.objects.create_trigger_event('USER', None, {'user': '******'}, now()) root_recipe_id = recipe.root_superseded_recipe_id if recipe.root_superseded_recipe_id else recipe.id recipe_type_name = recipe.recipe_type.name revision_num = recipe.recipe_type_rev.revision_num forced_nodes = ForcedNodes() if all_jobs: forced_nodes.set_all_nodes() elif job_names: for job_name in job_names: forced_nodes.add_node(job_name) # Execute all of the messages to perform the reprocess messages = create_reprocess_messages([root_recipe_id], recipe_type_name, revision_num, event.id, forced_nodes=forced_nodes) while messages: msg = messages.pop(0) result = msg.execute() if not result: raise Exception('Reprocess failed on message type \'%s\'' % msg.type) messages.extend(msg.new_messages) # Update job priorities if priority is not None: Job.objects.filter(event_id=event.id).update(priority=priority) from queue.models import Queue Queue.objects.filter(job__event_id=event.id).update(priority=priority) new_recipe = Recipe.objects.get(root_superseded_recipe_id=root_recipe_id, is_superseded=False) try: # TODO: remove this check when REST API v5 is removed if request.version == 'v6': new_recipe = Recipe.objects.get_details(new_recipe.id) else: new_recipe = Recipe.objects.get_details_v5(new_recipe.id) except Recipe.DoesNotExist: raise Http404 serializer = self.get_serializer(new_recipe) url = reverse('recipe_details_view', args=[new_recipe.id], request=request) return Response(serializer.data, status=status.HTTP_201_CREATED, headers=dict(location=url))