def _handle_previous_batch(self, batch, definition):
"""Handles re-processing all recipes in the previous batch, returning any messages needed for the re-processing
:param batch: The batch
:type batch: :class:`batch.models.Batch`
:param definition: The batch definition
:type definition: :class:`batch.definition.definition.BatchDefinition`
:return: The messages needed for the re-processing
:rtype: list
"""
messages = []
if batch.superseded_batch_id and definition.root_batch_id is None:
self.is_prev_batch_done = True
return messages
# Re-processing a previous batch
recipe_qry = Recipe.objects.filter(batch_id=batch.superseded_batch_id,
recipe__isnull=True)
# Only handle MAX_RECIPE_NUM at a time
if self.current_recipe_id:
recipe_qry = recipe_qry.filter(id__lt=self.current_recipe_id)
recipe_qry = recipe_qry.order_by('-id')
root_recipe_ids = []
last_recipe_id = None
for recipe in recipe_qry.defer('input')[:MAX_RECIPE_NUM]:
last_recipe_id = recipe.id
if recipe.root_superseded_recipe_id is not None:
root_recipe_ids.append(recipe.root_superseded_recipe_id)
else:
root_recipe_ids.append(recipe.id)
recipe_count = len(root_recipe_ids)
self.current_recipe_id = last_recipe_id
if recipe_count > 0:
logger.info(
'Found %d recipe(s) from previous batch to reprocess, creating messages',
recipe_count)
msgs = create_reprocess_messages(
root_recipe_ids,
batch.recipe_type.name,
batch.recipe_type_rev.revision_num,
batch.event_id,
batch_id=batch.id,
forced_nodes=definition.forced_nodes)
messages.extend(msgs)
if recipe_count < MAX_RECIPE_NUM:
# Handled less than the max number of recipes, so recipes from previous batch must be done
self.is_prev_batch_done = True
return messages
def _post_v6(self, request, recipe_id):
"""Schedules a recipe for reprocessing and returns it in JSON form
:param request: the HTTP GET request
:type request: :class:`rest_framework.request.Request`
:param recipe_id: The id of the recipe
:type recipe_id: int encoded as a str
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
"""
forced_nodes_json = rest_util.parse_dict(request, 'forced_nodes', required=True)
revision_num = rest_util.parse_dict(request, 'revision_num', required=False)
try:
forced_nodes = ForcedNodesV6(forced_nodes_json, do_validate=True)
except InvalidDiff as ex:
logger.exception('Unable to reprocess recipe. Invalid input: %s', forced_nodes_json)
raise BadParameter(unicode(ex))
try:
recipe = Recipe.objects.select_related('recipe_type').get(id=recipe_id)
if revision_num:
recipe.recipe_type_rev = RecipeTypeRevision.objects.get_revision(recipe.recipe_type.name, revision_num)
else:
revision_num = recipe.recipe_type.revision_num
recipe.recipe_type_rev = RecipeTypeRevision.objects.get_revision(recipe.recipe_type.name, recipe.recipe_type.revision_num)
except Recipe.DoesNotExist:
raise Http404
except RecipeTypeRevision.DoesNotExist:
raise Http404
if recipe.is_superseded:
raise BadParameter('Cannot reprocess a superseded recipe')
validation = recipe.recipe_type_rev.validate_forced_nodes(forced_nodes_json)
if not validation.is_valid:
raise BadParameter('Unable to reprocess recipe. Errors in validating forced_nodes: %s' % validation.errors)
if validation.warnings:
logger.warning('Warnings encountered when reprocessing: %s' % validation.warnings)
event = TriggerEvent.objects.create_trigger_event('USER', None, {'user': '******'}, now())
root_recipe_id = recipe.root_superseded_recipe_id if recipe.root_superseded_recipe_id else recipe.id
recipe_type_name = recipe.recipe_type.name
# Execute all of the messages to perform the reprocess
messages = create_reprocess_messages([root_recipe_id], recipe_type_name, revision_num, event.id,
forced_nodes=forced_nodes.get_forced_nodes())
CommandMessageManager().send_messages(messages)
return Response(status=status.HTTP_202_ACCEPTED)
def post(self, request, recipe_id):
"""Schedules a recipe for reprocessing and returns it in JSON form
:param request: the HTTP GET request
:type request: :class:`rest_framework.request.Request`
:param recipe_id: The id of the recipe
:type recipe_id: int encoded as a str
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
"""
job_names = rest_util.parse_string_list(request, 'job_names', required=False)
all_jobs = rest_util.parse_bool(request, 'all_jobs', required=False)
priority = rest_util.parse_int(request, 'priority', required=False)
try:
recipe = Recipe.objects.select_related('recipe_type', 'recipe_type_rev').get(id=recipe_id)
except Recipe.DoesNotExist:
raise Http404
if recipe.is_superseded:
raise BadParameter('Cannot reprocess a superseded recipe')
event = TriggerEvent.objects.create_trigger_event('USER', None, {'user': '******'}, now())
root_recipe_id = recipe.root_superseded_recipe_id if recipe.root_superseded_recipe_id else recipe.id
recipe_type_name = recipe.recipe_type.name
revision_num = recipe.recipe_type_rev.revision_num
forced_nodes = ForcedNodes()
if all_jobs:
forced_nodes.set_all_nodes()
elif job_names:
for job_name in job_names:
forced_nodes.add_node(job_name)
# Execute all of the messages to perform the reprocess
messages = create_reprocess_messages([root_recipe_id], recipe_type_name, revision_num, event.id,
forced_nodes=forced_nodes)
while messages:
msg = messages.pop(0)
result = msg.execute()
if not result:
raise Exception('Reprocess failed on message type \'%s\'' % msg.type)
messages.extend(msg.new_messages)
# Update job priorities
if priority is not None:
Job.objects.filter(event_id=event.id).update(priority=priority)
from queue.models import Queue
Queue.objects.filter(job__event_id=event.id).update(priority=priority)
new_recipe = Recipe.objects.get(root_superseded_recipe_id=root_recipe_id, is_superseded=False)
try:
# TODO: remove this check when REST API v5 is removed
if request.version == 'v6':
new_recipe = Recipe.objects.get_details(new_recipe.id)
else:
new_recipe = Recipe.objects.get_details_v5(new_recipe.id)
except Recipe.DoesNotExist:
raise Http404
serializer = self.get_serializer(new_recipe)
url = reverse('recipe_details_view', args=[new_recipe.id], request=request)
return Response(serializer.data, status=status.HTTP_201_CREATED, headers=dict(location=url))
def _handle_new_batch(self, batch, definition):
"""Handles creating a new batch of recipes with the defined dataset, returning any messages needed for the batch
:param batch: The batch
:type batch: :class:`batch.models.Batch`
:param definition: The batch definition
:type definition: :class:`batch.definition.definition.BatchDefinition`
:return: The messages needed for the re-processing
:rtype: list
"""
messages = []
dataset = DataSet.objects.get(pk=definition.dataset)
recipe_type_rev = RecipeTypeRevision.objects.get_revision(
name=batch.recipe_type.name,
revision_num=batch.recipe_type_rev.revision_num)
# combine the parameters
dataset_parameters = Batch.objects.merge_parameter_map(batch, dataset)
try:
recipe_type_rev.get_definition(
).input_interface.validate_connection(dataset_parameters)
except InvalidInterfaceConnection as ex:
# No recipe inputs match the dataset
logger.info(
'None of the dataset parameters matched the recipe type inputs; No recipes will be created'
)
self.is_prev_batch_done = True
return messages
# Get previous recipes for dataset files:
ds_files = DataSetFile.objects.get_dataset_files(
dataset.id).values_list('scale_file_id', flat=True)
recipe_ids = RecipeInputFile.objects.filter(
input_file_id__in=ds_files).values_list('recipe_id', flat=True)
recipe_file_ids = RecipeInputFile.objects.filter(
input_file_id__in=ds_files,
recipe__recipe_type=batch.recipe_type,
recipe__recipe_type_rev=batch.recipe_type_rev).values_list(
'input_file_id', flat=True)
extra_files_qry = ScaleFile.objects.filter(id__in=ds_files)
recipe_count = 0
# Reprocess previous recipes
if definition.supersedes:
if len(recipe_ids) > 0:
# Create re-process messages for all recipes
recipe_qry = Recipe.objects.filter(
id__in=recipe_ids).order_by('-id')
if self.current_recipe_id:
recipe_qry = recipe_qry.filter(
id__lt=self.current_recipe_id)
root_recipe_ids = []
for recipe in recipe_qry.defer('input')[:MAX_RECIPE_NUM]:
root_recipe_ids.append(recipe.id)
self.current_recipe_id = recipe.id
recipe_count = len(root_recipe_ids)
if recipe_count > 0:
logger.info(
'Found %d recipe(s) from previous batch to reprocess, creating messages',
recipe_count)
msgs = create_reprocess_messages(
root_recipe_ids,
batch.recipe_type.name,
batch.recipe_type_rev.revision_num,
batch.event_id,
batch_id=batch.id,
forced_nodes=definition.forced_nodes)
messages.extend(msgs)
# Filter down the extra files to exclude those we've already re-processed
extra_files_qry = extra_files_qry.exclude(id__in=recipe_file_ids)
# If we have data that didn't match any previous recipes
if self.current_dataset_file_id:
extra_files_qry = extra_files_qry.filter(
id__lt=self.current_dataset_file_id)
extra_file_ids = list(
extra_files_qry.order_by('-id').values_list(
'id', flat=True)[:(MAX_RECIPE_NUM - recipe_count)])
if extra_file_ids:
self.current_dataset_file_id = extra_file_ids[-1]
if len(extra_file_ids) > 0:
logger.info(
'Found %d files that do not have previous recipes to re-process',
len(extra_file_ids))
input_data = []
for file in DataSetFile.objects.get_dataset_files(
dataset.id).filter(scale_file__id__in=extra_file_ids):
data = Data()
parameter_name = file.parameter_name
# if we needed to map the inputs to parameters:
if batch.get_configuration().input_map:
for param in batch.get_configuration().input_map:
if param['datasetParameter'] == file.parameter_name:
parameter_name = param['input']
break
data.add_value(FileValue(parameter_name, [file.scale_file_id]))
input_data.append(convert_data_to_v6_json(data).get_dict())
msgs = create_batch_recipes_messages(
batch.recipe_type.name,
batch.recipe_type.revision_num,
input_data,
batch.event_id,
batch_id=batch.id)
messages.extend(msgs)
recipe_count += len(input_data)
if recipe_count < MAX_RECIPE_NUM:
# Handled less than the max number of recipes, so recipes from previous batch must be done
self.is_prev_batch_done = True
return messages