def test_convert_configuration_to_v6(self): """Tests calling convert_configuration_to_v6()""" # Try configuration with nothing set configuration = BatchConfiguration() json = convert_configuration_to_v6(configuration) BatchConfigurationV6(configuration=json.get_dict(), do_validate=True) # Revalidate # Try configuration with priority set configuration = BatchConfiguration() configuration.priority = 100 json = convert_configuration_to_v6(configuration) BatchConfigurationV6(configuration=json.get_dict(), do_validate=True) # Revalidate self.assertEqual(json.get_configuration().priority, configuration.priority)
def edit_batch_v6(self, batch, title=None, description=None, configuration=None): """Edits the given batch to update any of the given fields. The configuration will be stored in version 6 of its schemas. :param batch: The batch to edit :type batch: :class:`batch.models.Batch` :param title: The human-readable name of the batch :type title: string :param description: A human-readable description of the batch :type description: string :param configuration: The batch configuration :type configuration: :class:`batch.configuration.configuration.BatchConfiguration` :raises :class:`batch.configuration.exceptions.InvalidConfiguration`: If the configuration is invalid """ update_fields = {} if title is not None: update_fields['title'] = title if description is not None: update_fields['description'] = description if configuration: configuration.validate(batch) configuration_dict = convert_configuration_to_v6( configuration).get_dict() update_fields['configuration'] = configuration_dict if update_fields: Batch.objects.filter(id=batch.id).update(**update_fields)
def get_v6_configuration_json(self): """Returns the batch configuration in v6 of the JSON schema :returns: The batch configuration in v6 of the JSON schema :rtype: dict """ return rest_utils.strip_schema_version(convert_configuration_to_v6(self.get_configuration()).get_dict())
def validate_batch_v6(self, recipe_type, definition, configuration=None): """Validates the given recipe type, definition, and configuration for creating a new batch :param recipe_type: The type of recipes that will be created for this batch :type recipe_type: :class:`recipe.models.RecipeType` :param definition: The definition for running the batch :type definition: :class:`batch.definition.definition.BatchDefinition` :param configuration: The batch configuration :type configuration: :class:`batch.configuration.configuration.BatchConfiguration` :returns: The batch validation :rtype: :class:`batch.models.BatchValidation` """ is_valid = True errors = [] warnings = [] try: batch = Batch() batch.recipe_type = recipe_type batch.recipe_type_rev = RecipeTypeRevision.objects.get_revision( recipe_type.name, recipe_type.revision_num) batch.definition = convert_definition_to_v6(definition).get_dict() batch.configuration = convert_configuration_to_v6( configuration).get_dict() if definition.root_batch_id is not None: # Find latest batch with the root ID try: superseded_batch = Batch.objects.get_batch_from_root( definition.root_batch_id) except Batch.DoesNotExist: raise InvalidDefinition( 'PREV_BATCH_NOT_FOUND', 'No batch with that root ID exists') batch.root_batch_id = superseded_batch.root_batch_id batch.superseded_batch = superseded_batch warnings.extend(definition.validate(batch)) warnings.extend(configuration.validate(batch)) except ValidationException as ex: is_valid = False errors.append(ex.error) batch.recipes_estimated = definition.estimated_recipes return BatchValidation(is_valid, errors, warnings, batch)
def create_batch_v6(self, title, description, recipe_type, event, definition, configuration=None): """Creates a new batch that will contain a collection of recipes to process. The definition and configuration will be stored in version 6 of their respective schemas. This method will only create the batch, not its recipes. To create the batch's recipes, a CreateBatchRecipes message needs to be sent to the messaging backend. :param title: The human-readable name of the batch :type title: string :param description: A human-readable description of the batch :type description: string :param recipe_type: The type of recipes that will be created for this batch :type recipe_type: :class:`recipe.models.RecipeType` :param event: The event that created this batch :type event: :class:`trigger.models.TriggerEvent` :param definition: The definition for running the batch :type definition: :class:`batch.definition.definition.BatchDefinition` :param configuration: The batch configuration :type configuration: :class:`batch.configuration.configuration.BatchConfiguration` :returns: The newly created batch :rtype: :class:`batch.models.Batch` :raises :class:`batch.configuration.exceptions.InvalidConfiguration`: If the configuration is invalid :raises :class:`batch.definition.exceptions.InvalidDefinition`: If the definition is invalid """ batch = Batch() batch.title = title batch.description = description batch.recipe_type = recipe_type batch.recipe_type_rev = RecipeTypeRevision.objects.get_revision( recipe_type.name, recipe_type.revision_num) batch.event = event batch.definition = convert_definition_to_v6(definition).get_dict() batch.configuration = convert_configuration_to_v6( configuration).get_dict() with transaction.atomic(): if definition.root_batch_id is not None: # Find latest batch with the root ID and supersede it try: superseded_batch = Batch.objects.get_locked_batch_from_root( definition.root_batch_id) except Batch.DoesNotExist: raise InvalidDefinition( 'PREV_BATCH_NOT_FOUND', 'No batch with that root ID exists') batch.root_batch_id = superseded_batch.root_batch_id batch.superseded_batch = superseded_batch self.supersede_batch(superseded_batch.id, now()) definition.validate(batch) configuration.validate(batch) batch.recipes_estimated = definition.estimated_recipes batch.save() if batch.root_batch_id is None: # Batches with no superseded batch are their own root batch.root_batch_id = batch.id Batch.objects.filter(id=batch.id).update( root_batch_id=batch.id) # Create models for batch metrics batch_metrics_models = [] for job_name in recipe_type.get_definition().get_topological_order( ): batch_metrics_model = BatchMetrics() batch_metrics_model.batch_id = batch.id batch_metrics_model.job_name = job_name batch_metrics_models.append(batch_metrics_model) BatchMetrics.objects.bulk_create(batch_metrics_models) return batch
def _perform_batch_field_iteration(self): """Performs a single iteration of the setting of batch fields on job and recipe models """ # Get batch ID batch_qry = Batch.objects.all() if self._current_batch_id: batch_qry = batch_qry.filter(id__gt=self._current_batch_id) for batch in batch_qry.order_by('id').only('id')[:1]: batch_id = batch.id break # Populate job.batch_id and recipe.batch_id if they are missing qry_1 = 'UPDATE job j SET batch_id = bj.batch_id FROM batch_job bj' qry_1 += ' WHERE j.id = bj.job_id AND bj.batch_id = %s AND j.batch_id IS NULL' qry_2 = 'UPDATE recipe r SET batch_id = br.batch_id FROM batch_recipe br' qry_2 += ' WHERE r.id = br.recipe_id AND br.batch_id = %s AND r.batch_id IS NULL' qry_3 = 'UPDATE batch b SET recipe_type_rev_id = ' qry_3 += '(SELECT recipe_type_rev_id FROM recipe r WHERE r.batch_id = %s LIMIT 1) ' qry_3 += 'WHERE b.id = %s AND b.recipe_type_rev_id = 1 ' qry_3 += 'AND (SELECT count(*) FROM recipe r WHERE r.batch_id = %s) > 0' qry_4 = 'UPDATE batch b SET recipe_type_rev_id = ' qry_4 += '(SELECT rtv.id FROM recipe_type_revision rtv ' qry_4 += 'JOIN recipe_type rt ON rtv.recipe_type_id = rt.id ' qry_4 += 'JOIN batch b ON rt.id = b.recipe_type_id ' qry_4 += 'WHERE b.id = %s AND b.created > rtv.created ORDER BY rtv.created DESC LIMIT 1) ' qry_4 += 'WHERE b.id = %s AND b.recipe_type_rev_id = 1' # Populate batch.recipe_type_rev if it hasn't been set yet with connection.cursor() as cursor: cursor.execute(qry_1, [str(batch_id)]) count = cursor.rowcount if count: logger.info('%d job(s) updated with batch_id %d', count, batch_id) cursor.execute(qry_2, [str(batch_id)]) count = cursor.rowcount if count: logger.info('%d recipe(s) updated with batch_id %d', count, batch_id) cursor.execute(qry_3, [str(batch_id), str(batch_id), str(batch_id)]) count = cursor.rowcount if count: logger.info('Batch with batch_id %d set to correct recipe type revision (based on old batch recipe)', batch_id) cursor.execute(qry_4, [str(batch_id), str(batch_id)]) count = cursor.rowcount if count: logger.info('Batch with batch_id %d set to correct recipe type revision (based on revision creation)', batch_id) batch = Batch.objects.get(id=batch_id) if not batch.configuration: definition = batch.get_old_definition() if definition.priority is not None: configuration = BatchConfiguration() configuration.priority = definition.priority from batch.configuration.json.configuration_v6 import convert_configuration_to_v6 config_dict = convert_configuration_to_v6(configuration).get_dict() Batch.objects.filter(id=batch_id).update(configuration=config_dict) logger.info('Batch with batch_id %d updated with new configuration', batch_id) self._current_batch_id = batch_id self._updated_batch += 1 if self._updated_batch > self._total_batch: self._updated_batch = self._total_batch percent = (float(self._updated_batch) / float(self._total_batch)) * 100.00 logger.info('Completed %s of %s batches (%.1f%%)', self._updated_batch, self._total_batch, percent)