def validate(self, batch):
"""Validates the given batch to make sure it is valid with respect to this batch definition. The given batch
must have all of its related fields populated, though id and root_batch_id may be None. The derived definition
attributes, such as estimated recipe total and previous batch diff, will be populated by this method.
:param batch: The batch model
:type batch: :class:`batch.models.Batch`
:returns: A list of warnings discovered during validation
:rtype: list
:raises :class:`batch.definition.exceptions.InvalidDefinition`: If the definition is invalid
"""
if self.root_batch_id:
if batch.recipe_type_id != batch.superseded_batch.recipe_type_id:
raise InvalidDefinition('MISMATCHED_RECIPE_TYPE',
'New batch and previous batch must have the same recipe type')
if not batch.superseded_batch.is_creation_done:
raise InvalidDefinition('PREV_BATCH_STILL_CREATING',
'Previous batch must have completed creating all of its recipes')
# Generate recipe diff against the previous batch
recipe_def = batch.recipe_type_rev.get_definition()
prev_recipe_def = batch.superseded_batch.recipe_type_rev.get_definition()
self.prev_batch_diff = RecipeDiff(prev_recipe_def, recipe_def)
if self.forced_nodes:
self.prev_batch_diff.set_force_reprocess(self.forced_nodes)
if not self.prev_batch_diff.can_be_reprocessed:
raise InvalidDefinition('PREV_BATCH_NO_REPROCESS', 'Previous batch cannot be reprocessed')
self._estimate_recipe_total(batch)
if not self.estimated_recipes:
raise InvalidDefinition('NO_RECIPES', 'Batch definition must result in creating at least one recipe')
return []
def __init__(self, definition=None, do_validate=False):
"""Creates a v6 batch definition JSON object from the given dictionary
:param definition: The batch definition JSON dict
:type definition: dict
:param do_validate: Whether to perform validation on the JSON schema
:type do_validate: bool
:raises :class:`batch.definition.exceptions.InvalidDefinition`: If the given definition is invalid
"""
if not definition:
definition = {}
self._definition = definition
if 'version' not in self._definition:
self._definition['version'] = SCHEMA_VERSION
if self._definition['version'] not in SCHEMA_VERSIONS:
msg = '%s is an unsupported version number' % self._definition[
'version']
raise InvalidDefinition('INVALID_BATCH_DEFINITION', msg)
try:
if do_validate:
validate(self._definition, BATCH_DEFINITION_SCHEMA)
if 'forced_nodes' in self._definition:
ForcedNodesV6(self._definition['forced_nodes'],
do_validate=True)
except ValidationError as ex:
raise InvalidDefinition(
'INVALID_BATCH_DEFINITION',
'Invalid batch definition: %s' % unicode(ex))
def __init__(self, definition):
"""Creates a batch definition object from the given dictionary. The general format is checked for correctness.
:param definition: The batch definition
:type definition: dict
:raises :class:`batch.configuration.definition.exceptions.InvalidDefinition`:
If the given definition is invalid
"""
self._definition = definition
try:
validate(definition, BATCH_DEFINITION_SCHEMA)
except ValidationError as ex:
raise InvalidDefinition('', 'Invalid batch definition: %s' % unicode(ex))
self._populate_default_values()
if not self._definition['version'] == '1.0':
raise InvalidDefinition('', '%s is an unsupported version number' % self._definition['version'])
date_range = self._definition['date_range'] if 'date_range' in self._definition else None
self.date_range_type = None
if date_range and 'type' in date_range:
self.date_range_type = date_range['type']
self.started = None
if date_range and 'started' in date_range:
try:
self.started = parse.parse_datetime(date_range['started'])
except ValueError:
raise InvalidDefinition('', 'Invalid start date format: %s' % date_range['started'])
self.ended = None
if date_range and 'ended' in date_range:
try:
self.ended = parse.parse_datetime(date_range['ended'])
except ValueError:
raise InvalidDefinition('', 'Invalid end date format: %s' % date_range['ended'])
self.job_names = self._definition['job_names']
self.all_jobs = self._definition['all_jobs']
self.priority = None
if 'priority' in self._definition:
try:
self.priority = self._definition['priority']
except ValueError:
raise InvalidDefinition('', 'Invalid priority: %s' % self._definition['priority'])
self.trigger_rule = False
self.trigger_config = None
if 'trigger_rule' in self._definition:
if isinstance(self._definition['trigger_rule'], bool):
self.trigger_rule = self._definition['trigger_rule']
else:
self.trigger_config = BatchTriggerConfiguration('BATCH', self._definition['trigger_rule'])
def validate(self, batch):
"""Validates the given batch to make sure it is valid with respect to this batch definition. The given batch
must have all of its related fields populated, though id and root_batch_id may be None. The derived definition
attributes, such as estimated recipe total and previous batch diff, will be populated by this method.
:param batch: The batch model
:type batch: :class:`batch.models.Batch`
:returns: A list of warnings discovered during validation
:rtype: :func:`list`
:raises :class:`batch.definition.exceptions.InvalidDefinition`: If the definition is invalid
"""
# Re-processing a previous batch
if self.root_batch_id:
if batch.recipe_type_id != batch.superseded_batch.recipe_type_id:
raise InvalidDefinition('MISMATCHED_RECIPE_TYPE',
'New batch and previous batch must have the same recipe type')
if not batch.superseded_batch.is_creation_done:
raise InvalidDefinition('PREV_BATCH_STILL_CREATING',
'Previous batch must have completed creating all of its recipes')
# Generate recipe diff against the previous batch
recipe_def = batch.recipe_type_rev.get_definition()
prev_recipe_def = batch.superseded_batch.recipe_type_rev.get_definition()
self.prev_batch_diff = RecipeDiff(prev_recipe_def, recipe_def)
if self.forced_nodes:
self.prev_batch_diff.set_force_reprocess(self.forced_nodes)
if not self.prev_batch_diff.can_be_reprocessed:
raise InvalidDefinition('PREV_BATCH_NO_REPROCESS', 'Previous batch cannot be reprocessed')
# New batch - need to validate dataset parameters against recipe revision
elif self.dataset:
from data.interface.exceptions import InvalidInterfaceConnection
from data.models import DataSet
from recipe.models import RecipeTypeRevision
dataset_definition = DataSet.objects.get(pk=self.dataset).get_definition()
recipe_type_rev = RecipeTypeRevision.objects.get_revision(name=batch.recipe_type.name, revision_num=batch.recipe_type_rev.revision_num).recipe_type
# combine the parameters
from batch.models import Batch
dataset_parameters = Batch.objects.merge_parameter_map(batch, DataSet.objects.get(pk=self.dataset))
try:
recipe_type_rev.get_definition().input_interface.validate_connection(dataset_parameters)
except InvalidInterfaceConnection as ex:
raise InvalidDefinition('MISMATCHED_PARAMS', 'No parameters in the dataset match the recipe type inputs. %s' % unicode(ex))
self._estimate_recipe_total(batch)
if not self.estimated_recipes:
raise InvalidDefinition('NO_RECIPES', 'Batch definition must result in creating at least one recipe')
return []
def validate_batch_v6(self, recipe_type, definition, configuration=None):
"""Validates the given recipe type, definition, and configuration for creating a new batch
:param recipe_type: The type of recipes that will be created for this batch
:type recipe_type: :class:`recipe.models.RecipeType`
:param definition: The definition for running the batch
:type definition: :class:`batch.definition.definition.BatchDefinition`
:param configuration: The batch configuration
:type configuration: :class:`batch.configuration.configuration.BatchConfiguration`
:returns: The batch validation
:rtype: :class:`batch.models.BatchValidation`
"""
is_valid = True
errors = []
warnings = []
try:
batch = Batch()
batch.recipe_type = recipe_type
batch.recipe_type_rev = RecipeTypeRevision.objects.get_revision(
recipe_type.name, recipe_type.revision_num)
batch.definition = convert_definition_to_v6(definition).get_dict()
batch.configuration = convert_configuration_to_v6(
configuration).get_dict()
if definition.root_batch_id is not None:
# Find latest batch with the root ID
try:
superseded_batch = Batch.objects.get_batch_from_root(
definition.root_batch_id)
except Batch.DoesNotExist:
raise InvalidDefinition(
'PREV_BATCH_NOT_FOUND',
'No batch with that root ID exists')
batch.root_batch_id = superseded_batch.root_batch_id
batch.superseded_batch = superseded_batch
warnings.extend(definition.validate(batch))
warnings.extend(configuration.validate(batch))
except ValidationException as ex:
is_valid = False
errors.append(ex.error)
batch.recipes_estimated = definition.estimated_recipes
return BatchValidation(is_valid, errors, warnings, batch)
def create_batch_v6(self,
title,
description,
recipe_type,
event,
definition,
configuration=None):
"""Creates a new batch that will contain a collection of recipes to process. The definition and configuration
will be stored in version 6 of their respective schemas. This method will only create the batch, not its
recipes. To create the batch's recipes, a CreateBatchRecipes message needs to be sent to the messaging backend.
:param title: The human-readable name of the batch
:type title: string
:param description: A human-readable description of the batch
:type description: string
:param recipe_type: The type of recipes that will be created for this batch
:type recipe_type: :class:`recipe.models.RecipeType`
:param event: The event that created this batch
:type event: :class:`trigger.models.TriggerEvent`
:param definition: The definition for running the batch
:type definition: :class:`batch.definition.definition.BatchDefinition`
:param configuration: The batch configuration
:type configuration: :class:`batch.configuration.configuration.BatchConfiguration`
:returns: The newly created batch
:rtype: :class:`batch.models.Batch`
:raises :class:`batch.configuration.exceptions.InvalidConfiguration`: If the configuration is invalid
:raises :class:`batch.definition.exceptions.InvalidDefinition`: If the definition is invalid
"""
batch = Batch()
batch.title = title
batch.description = description
batch.recipe_type = recipe_type
batch.recipe_type_rev = RecipeTypeRevision.objects.get_revision(
recipe_type.name, recipe_type.revision_num)
batch.event = event
batch.definition = convert_definition_to_v6(definition).get_dict()
batch.configuration = convert_configuration_to_v6(
configuration).get_dict()
with transaction.atomic():
if definition.root_batch_id is not None:
# Find latest batch with the root ID and supersede it
try:
superseded_batch = Batch.objects.get_locked_batch_from_root(
definition.root_batch_id)
except Batch.DoesNotExist:
raise InvalidDefinition(
'PREV_BATCH_NOT_FOUND',
'No batch with that root ID exists')
batch.root_batch_id = superseded_batch.root_batch_id
batch.superseded_batch = superseded_batch
self.supersede_batch(superseded_batch.id, now())
definition.validate(batch)
configuration.validate(batch)
batch.recipes_estimated = definition.estimated_recipes
batch.save()
if batch.root_batch_id is None: # Batches with no superseded batch are their own root
batch.root_batch_id = batch.id
Batch.objects.filter(id=batch.id).update(
root_batch_id=batch.id)
# Create models for batch metrics
batch_metrics_models = []
for job_name in recipe_type.get_definition().get_topological_order(
):
batch_metrics_model = BatchMetrics()
batch_metrics_model.batch_id = batch.id
batch_metrics_model.job_name = job_name
batch_metrics_models.append(batch_metrics_model)
BatchMetrics.objects.bulk_create(batch_metrics_models)
return batch