def test_convert_configuration_to_v6(self):
"""Tests calling convert_configuration_to_v6()"""
# Try configuration with nothing set
configuration = BatchConfiguration()
json = convert_configuration_to_v6(configuration)
BatchConfigurationV6(configuration=json.get_dict(), do_validate=True) # Revalidate
# Try configuration with priority set
configuration = BatchConfiguration()
configuration.priority = 100
json = convert_configuration_to_v6(configuration)
BatchConfigurationV6(configuration=json.get_dict(), do_validate=True) # Revalidate
self.assertEqual(json.get_configuration().priority, configuration.priority)
def edit_batch_v6(self,
batch,
title=None,
description=None,
configuration=None):
"""Edits the given batch to update any of the given fields. The configuration will be stored in version 6 of its
schemas.
:param batch: The batch to edit
:type batch: :class:`batch.models.Batch`
:param title: The human-readable name of the batch
:type title: string
:param description: A human-readable description of the batch
:type description: string
:param configuration: The batch configuration
:type configuration: :class:`batch.configuration.configuration.BatchConfiguration`
:raises :class:`batch.configuration.exceptions.InvalidConfiguration`: If the configuration is invalid
"""
update_fields = {}
if title is not None:
update_fields['title'] = title
if description is not None:
update_fields['description'] = description
if configuration:
configuration.validate(batch)
configuration_dict = convert_configuration_to_v6(
configuration).get_dict()
update_fields['configuration'] = configuration_dict
if update_fields:
Batch.objects.filter(id=batch.id).update(**update_fields)
def get_v6_configuration_json(self):
"""Returns the batch configuration in v6 of the JSON schema
:returns: The batch configuration in v6 of the JSON schema
:rtype: dict
"""
return rest_utils.strip_schema_version(convert_configuration_to_v6(self.get_configuration()).get_dict())
def validate_batch_v6(self, recipe_type, definition, configuration=None):
"""Validates the given recipe type, definition, and configuration for creating a new batch
:param recipe_type: The type of recipes that will be created for this batch
:type recipe_type: :class:`recipe.models.RecipeType`
:param definition: The definition for running the batch
:type definition: :class:`batch.definition.definition.BatchDefinition`
:param configuration: The batch configuration
:type configuration: :class:`batch.configuration.configuration.BatchConfiguration`
:returns: The batch validation
:rtype: :class:`batch.models.BatchValidation`
"""
is_valid = True
errors = []
warnings = []
try:
batch = Batch()
batch.recipe_type = recipe_type
batch.recipe_type_rev = RecipeTypeRevision.objects.get_revision(
recipe_type.name, recipe_type.revision_num)
batch.definition = convert_definition_to_v6(definition).get_dict()
batch.configuration = convert_configuration_to_v6(
configuration).get_dict()
if definition.root_batch_id is not None:
# Find latest batch with the root ID
try:
superseded_batch = Batch.objects.get_batch_from_root(
definition.root_batch_id)
except Batch.DoesNotExist:
raise InvalidDefinition(
'PREV_BATCH_NOT_FOUND',
'No batch with that root ID exists')
batch.root_batch_id = superseded_batch.root_batch_id
batch.superseded_batch = superseded_batch
warnings.extend(definition.validate(batch))
warnings.extend(configuration.validate(batch))
except ValidationException as ex:
is_valid = False
errors.append(ex.error)
batch.recipes_estimated = definition.estimated_recipes
return BatchValidation(is_valid, errors, warnings, batch)
def create_batch_v6(self,
title,
description,
recipe_type,
event,
definition,
configuration=None):
"""Creates a new batch that will contain a collection of recipes to process. The definition and configuration
will be stored in version 6 of their respective schemas. This method will only create the batch, not its
recipes. To create the batch's recipes, a CreateBatchRecipes message needs to be sent to the messaging backend.
:param title: The human-readable name of the batch
:type title: string
:param description: A human-readable description of the batch
:type description: string
:param recipe_type: The type of recipes that will be created for this batch
:type recipe_type: :class:`recipe.models.RecipeType`
:param event: The event that created this batch
:type event: :class:`trigger.models.TriggerEvent`
:param definition: The definition for running the batch
:type definition: :class:`batch.definition.definition.BatchDefinition`
:param configuration: The batch configuration
:type configuration: :class:`batch.configuration.configuration.BatchConfiguration`
:returns: The newly created batch
:rtype: :class:`batch.models.Batch`
:raises :class:`batch.configuration.exceptions.InvalidConfiguration`: If the configuration is invalid
:raises :class:`batch.definition.exceptions.InvalidDefinition`: If the definition is invalid
"""
batch = Batch()
batch.title = title
batch.description = description
batch.recipe_type = recipe_type
batch.recipe_type_rev = RecipeTypeRevision.objects.get_revision(
recipe_type.name, recipe_type.revision_num)
batch.event = event
batch.definition = convert_definition_to_v6(definition).get_dict()
batch.configuration = convert_configuration_to_v6(
configuration).get_dict()
with transaction.atomic():
if definition.root_batch_id is not None:
# Find latest batch with the root ID and supersede it
try:
superseded_batch = Batch.objects.get_locked_batch_from_root(
definition.root_batch_id)
except Batch.DoesNotExist:
raise InvalidDefinition(
'PREV_BATCH_NOT_FOUND',
'No batch with that root ID exists')
batch.root_batch_id = superseded_batch.root_batch_id
batch.superseded_batch = superseded_batch
self.supersede_batch(superseded_batch.id, now())
definition.validate(batch)
configuration.validate(batch)
batch.recipes_estimated = definition.estimated_recipes
batch.save()
if batch.root_batch_id is None: # Batches with no superseded batch are their own root
batch.root_batch_id = batch.id
Batch.objects.filter(id=batch.id).update(
root_batch_id=batch.id)
# Create models for batch metrics
batch_metrics_models = []
for job_name in recipe_type.get_definition().get_topological_order(
):
batch_metrics_model = BatchMetrics()
batch_metrics_model.batch_id = batch.id
batch_metrics_model.job_name = job_name
batch_metrics_models.append(batch_metrics_model)
BatchMetrics.objects.bulk_create(batch_metrics_models)
return batch
def _perform_batch_field_iteration(self):
"""Performs a single iteration of the setting of batch fields on job and recipe models
"""
# Get batch ID
batch_qry = Batch.objects.all()
if self._current_batch_id:
batch_qry = batch_qry.filter(id__gt=self._current_batch_id)
for batch in batch_qry.order_by('id').only('id')[:1]:
batch_id = batch.id
break
# Populate job.batch_id and recipe.batch_id if they are missing
qry_1 = 'UPDATE job j SET batch_id = bj.batch_id FROM batch_job bj'
qry_1 += ' WHERE j.id = bj.job_id AND bj.batch_id = %s AND j.batch_id IS NULL'
qry_2 = 'UPDATE recipe r SET batch_id = br.batch_id FROM batch_recipe br'
qry_2 += ' WHERE r.id = br.recipe_id AND br.batch_id = %s AND r.batch_id IS NULL'
qry_3 = 'UPDATE batch b SET recipe_type_rev_id = '
qry_3 += '(SELECT recipe_type_rev_id FROM recipe r WHERE r.batch_id = %s LIMIT 1) '
qry_3 += 'WHERE b.id = %s AND b.recipe_type_rev_id = 1 '
qry_3 += 'AND (SELECT count(*) FROM recipe r WHERE r.batch_id = %s) > 0'
qry_4 = 'UPDATE batch b SET recipe_type_rev_id = '
qry_4 += '(SELECT rtv.id FROM recipe_type_revision rtv '
qry_4 += 'JOIN recipe_type rt ON rtv.recipe_type_id = rt.id '
qry_4 += 'JOIN batch b ON rt.id = b.recipe_type_id '
qry_4 += 'WHERE b.id = %s AND b.created > rtv.created ORDER BY rtv.created DESC LIMIT 1) '
qry_4 += 'WHERE b.id = %s AND b.recipe_type_rev_id = 1'
# Populate batch.recipe_type_rev if it hasn't been set yet
with connection.cursor() as cursor:
cursor.execute(qry_1, [str(batch_id)])
count = cursor.rowcount
if count:
logger.info('%d job(s) updated with batch_id %d', count, batch_id)
cursor.execute(qry_2, [str(batch_id)])
count = cursor.rowcount
if count:
logger.info('%d recipe(s) updated with batch_id %d', count, batch_id)
cursor.execute(qry_3, [str(batch_id), str(batch_id), str(batch_id)])
count = cursor.rowcount
if count:
logger.info('Batch with batch_id %d set to correct recipe type revision (based on old batch recipe)',
batch_id)
cursor.execute(qry_4, [str(batch_id), str(batch_id)])
count = cursor.rowcount
if count:
logger.info('Batch with batch_id %d set to correct recipe type revision (based on revision creation)',
batch_id)
batch = Batch.objects.get(id=batch_id)
if not batch.configuration:
definition = batch.get_old_definition()
if definition.priority is not None:
configuration = BatchConfiguration()
configuration.priority = definition.priority
from batch.configuration.json.configuration_v6 import convert_configuration_to_v6
config_dict = convert_configuration_to_v6(configuration).get_dict()
Batch.objects.filter(id=batch_id).update(configuration=config_dict)
logger.info('Batch with batch_id %d updated with new configuration', batch_id)
self._current_batch_id = batch_id
self._updated_batch += 1
if self._updated_batch > self._total_batch:
self._updated_batch = self._total_batch
percent = (float(self._updated_batch) / float(self._total_batch)) * 100.00
logger.info('Completed %s of %s batches (%.1f%%)', self._updated_batch, self._total_batch, percent)
