def _handle_previous_batch(self, batch, definition): """Handles re-processing all recipes in the previous batch, returning any messages needed for the re-processing :param batch: The batch :type batch: :class:`batch.models.Batch` :param definition: The batch definition :type definition: :class:`batch.definition.definition.BatchDefinition` :return: The messages needed for the re-processing :rtype: list """ messages = [] if batch.superseded_batch_id and definition.root_batch_id is None: self.is_prev_batch_done = True return messages # Re-processing a previous batch recipe_qry = Recipe.objects.filter(batch_id=batch.superseded_batch_id, recipe__isnull=True) # Only handle MAX_RECIPE_NUM at a time if self.current_recipe_id: recipe_qry = recipe_qry.filter(id__lt=self.current_recipe_id) recipe_qry = recipe_qry.order_by('-id') root_recipe_ids = [] last_recipe_id = None for recipe in recipe_qry.defer('input')[:MAX_RECIPE_NUM]: last_recipe_id = recipe.id if recipe.root_superseded_recipe_id is not None: root_recipe_ids.append(recipe.root_superseded_recipe_id) else: root_recipe_ids.append(recipe.id) recipe_count = len(root_recipe_ids) self.current_recipe_id = last_recipe_id if recipe_count > 0: logger.info( 'Found %d recipe(s) from previous batch to reprocess, creating messages', recipe_count) msgs = create_reprocess_messages( root_recipe_ids, batch.recipe_type.name, batch.recipe_type_rev.revision_num, batch.event_id, batch_id=batch.id, forced_nodes=definition.forced_nodes) messages.extend(msgs) if recipe_count < MAX_RECIPE_NUM: # Handled less than the max number of recipes, so recipes from previous batch must be done self.is_prev_batch_done = True return messages
def _post_v6(self, request, recipe_id): """Schedules a recipe for reprocessing and returns it in JSON form :param request: the HTTP GET request :type request: :class:`rest_framework.request.Request` :param recipe_id: The id of the recipe :type recipe_id: int encoded as a str :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user """ forced_nodes_json = rest_util.parse_dict(request, 'forced_nodes', required=True) revision_num = rest_util.parse_dict(request, 'revision_num', required=False) try: forced_nodes = ForcedNodesV6(forced_nodes_json, do_validate=True) except InvalidDiff as ex: logger.exception('Unable to reprocess recipe. Invalid input: %s', forced_nodes_json) raise BadParameter(unicode(ex)) try: recipe = Recipe.objects.select_related('recipe_type').get(id=recipe_id) if revision_num: recipe.recipe_type_rev = RecipeTypeRevision.objects.get_revision(recipe.recipe_type.name, revision_num) else: revision_num = recipe.recipe_type.revision_num recipe.recipe_type_rev = RecipeTypeRevision.objects.get_revision(recipe.recipe_type.name, recipe.recipe_type.revision_num) except Recipe.DoesNotExist: raise Http404 except RecipeTypeRevision.DoesNotExist: raise Http404 if recipe.is_superseded: raise BadParameter('Cannot reprocess a superseded recipe') validation = recipe.recipe_type_rev.validate_forced_nodes(forced_nodes_json) if not validation.is_valid: raise BadParameter('Unable to reprocess recipe. Errors in validating forced_nodes: %s' % validation.errors) if validation.warnings: logger.warning('Warnings encountered when reprocessing: %s' % validation.warnings) event = TriggerEvent.objects.create_trigger_event('USER', None, {'user': '******'}, now()) root_recipe_id = recipe.root_superseded_recipe_id if recipe.root_superseded_recipe_id else recipe.id recipe_type_name = recipe.recipe_type.name # Execute all of the messages to perform the reprocess messages = create_reprocess_messages([root_recipe_id], recipe_type_name, revision_num, event.id, forced_nodes=forced_nodes.get_forced_nodes()) CommandMessageManager().send_messages(messages) return Response(status=status.HTTP_202_ACCEPTED)
def post(self, request, recipe_id): """Schedules a recipe for reprocessing and returns it in JSON form :param request: the HTTP GET request :type request: :class:`rest_framework.request.Request` :param recipe_id: The id of the recipe :type recipe_id: int encoded as a str :rtype: :class:`rest_framework.response.Response` :returns: the HTTP response to send back to the user """ job_names = rest_util.parse_string_list(request, 'job_names', required=False) all_jobs = rest_util.parse_bool(request, 'all_jobs', required=False) priority = rest_util.parse_int(request, 'priority', required=False) try: recipe = Recipe.objects.select_related('recipe_type', 'recipe_type_rev').get(id=recipe_id) except Recipe.DoesNotExist: raise Http404 if recipe.is_superseded: raise BadParameter('Cannot reprocess a superseded recipe') event = TriggerEvent.objects.create_trigger_event('USER', None, {'user': '******'}, now()) root_recipe_id = recipe.root_superseded_recipe_id if recipe.root_superseded_recipe_id else recipe.id recipe_type_name = recipe.recipe_type.name revision_num = recipe.recipe_type_rev.revision_num forced_nodes = ForcedNodes() if all_jobs: forced_nodes.set_all_nodes() elif job_names: for job_name in job_names: forced_nodes.add_node(job_name) # Execute all of the messages to perform the reprocess messages = create_reprocess_messages([root_recipe_id], recipe_type_name, revision_num, event.id, forced_nodes=forced_nodes) while messages: msg = messages.pop(0) result = msg.execute() if not result: raise Exception('Reprocess failed on message type \'%s\'' % msg.type) messages.extend(msg.new_messages) # Update job priorities if priority is not None: Job.objects.filter(event_id=event.id).update(priority=priority) from queue.models import Queue Queue.objects.filter(job__event_id=event.id).update(priority=priority) new_recipe = Recipe.objects.get(root_superseded_recipe_id=root_recipe_id, is_superseded=False) try: # TODO: remove this check when REST API v5 is removed if request.version == 'v6': new_recipe = Recipe.objects.get_details(new_recipe.id) else: new_recipe = Recipe.objects.get_details_v5(new_recipe.id) except Recipe.DoesNotExist: raise Http404 serializer = self.get_serializer(new_recipe) url = reverse('recipe_details_view', args=[new_recipe.id], request=request) return Response(serializer.data, status=status.HTTP_201_CREATED, headers=dict(location=url))
def _handle_new_batch(self, batch, definition): """Handles creating a new batch of recipes with the defined dataset, returning any messages needed for the batch :param batch: The batch :type batch: :class:`batch.models.Batch` :param definition: The batch definition :type definition: :class:`batch.definition.definition.BatchDefinition` :return: The messages needed for the re-processing :rtype: list """ messages = [] dataset = DataSet.objects.get(pk=definition.dataset) recipe_type_rev = RecipeTypeRevision.objects.get_revision( name=batch.recipe_type.name, revision_num=batch.recipe_type_rev.revision_num) # combine the parameters dataset_parameters = Batch.objects.merge_parameter_map(batch, dataset) try: recipe_type_rev.get_definition( ).input_interface.validate_connection(dataset_parameters) except InvalidInterfaceConnection as ex: # No recipe inputs match the dataset logger.info( 'None of the dataset parameters matched the recipe type inputs; No recipes will be created' ) self.is_prev_batch_done = True return messages # Get previous recipes for dataset files: ds_files = DataSetFile.objects.get_dataset_files( dataset.id).values_list('scale_file_id', flat=True) recipe_ids = RecipeInputFile.objects.filter( input_file_id__in=ds_files).values_list('recipe_id', flat=True) recipe_file_ids = RecipeInputFile.objects.filter( input_file_id__in=ds_files, recipe__recipe_type=batch.recipe_type, recipe__recipe_type_rev=batch.recipe_type_rev).values_list( 'input_file_id', flat=True) extra_files_qry = ScaleFile.objects.filter(id__in=ds_files) recipe_count = 0 # Reprocess previous recipes if definition.supersedes: if len(recipe_ids) > 0: # Create re-process messages for all recipes recipe_qry = Recipe.objects.filter( id__in=recipe_ids).order_by('-id') if self.current_recipe_id: recipe_qry = recipe_qry.filter( id__lt=self.current_recipe_id) root_recipe_ids = [] for recipe in recipe_qry.defer('input')[:MAX_RECIPE_NUM]: root_recipe_ids.append(recipe.id) self.current_recipe_id = recipe.id recipe_count = len(root_recipe_ids) if recipe_count > 0: logger.info( 'Found %d recipe(s) from previous batch to reprocess, creating messages', recipe_count) msgs = create_reprocess_messages( root_recipe_ids, batch.recipe_type.name, batch.recipe_type_rev.revision_num, batch.event_id, batch_id=batch.id, forced_nodes=definition.forced_nodes) messages.extend(msgs) # Filter down the extra files to exclude those we've already re-processed extra_files_qry = extra_files_qry.exclude(id__in=recipe_file_ids) # If we have data that didn't match any previous recipes if self.current_dataset_file_id: extra_files_qry = extra_files_qry.filter( id__lt=self.current_dataset_file_id) extra_file_ids = list( extra_files_qry.order_by('-id').values_list( 'id', flat=True)[:(MAX_RECIPE_NUM - recipe_count)]) if extra_file_ids: self.current_dataset_file_id = extra_file_ids[-1] if len(extra_file_ids) > 0: logger.info( 'Found %d files that do not have previous recipes to re-process', len(extra_file_ids)) input_data = [] for file in DataSetFile.objects.get_dataset_files( dataset.id).filter(scale_file__id__in=extra_file_ids): data = Data() parameter_name = file.parameter_name # if we needed to map the inputs to parameters: if batch.get_configuration().input_map: for param in batch.get_configuration().input_map: if param['datasetParameter'] == file.parameter_name: parameter_name = param['input'] break data.add_value(FileValue(parameter_name, [file.scale_file_id])) input_data.append(convert_data_to_v6_json(data).get_dict()) msgs = create_batch_recipes_messages( batch.recipe_type.name, batch.recipe_type.revision_num, input_data, batch.event_id, batch_id=batch.id) messages.extend(msgs) recipe_count += len(input_data) if recipe_count < MAX_RECIPE_NUM: # Handled less than the max number of recipes, so recipes from previous batch must be done self.is_prev_batch_done = True return messages