def test_json(self):
"""Tests coverting a CreateBatchRecipes message to and from JSON"""
# Previous batch with three recipes
recipe_type = recipe_test_utils.create_recipe_type()
prev_batch = batch_test_utils.create_batch(recipe_type=recipe_type,
is_creation_done=True,
recipes_total=3)
recipe_1 = recipe_test_utils.create_recipe(batch=prev_batch)
recipe_2 = recipe_test_utils.create_recipe(batch=prev_batch)
recipe_3 = recipe_test_utils.create_recipe(batch=prev_batch)
definition = BatchDefinition()
definition.root_batch_id = prev_batch.root_batch_id
batch = batch_test_utils.create_batch(recipe_type=recipe_type,
definition=definition)
# Create message
message = create_batch_recipes_message(batch.id)
# Convert message to JSON and back, and then execute
message_json_dict = message.to_json()
new_message = CreateBatchRecipes.from_json(message_json_dict)
result = new_message.execute()
self.assertTrue(result)
# Should be one reprocess_recipes message for the three recipes
self.assertEqual(len(new_message.new_messages), 1)
message = new_message.new_messages[0]
self.assertEqual(message.type, 'reprocess_recipes')
self.assertSetEqual(set(message._root_recipe_ids),
{recipe_1.id, recipe_2.id, recipe_3.id})
def _create_v6(self, request):
"""The v6 version for creating batches
:param request: the HTTP POST request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
"""
title = rest_util.parse_string(request, 'title', required=False)
description = rest_util.parse_string(request,
'description',
required=False)
recipe_type_id = rest_util.parse_int(request, 'recipe_type_id')
definition_dict = rest_util.parse_dict(request, 'definition')
configuration_dict = rest_util.parse_dict(request,
'configuration',
required=False)
# Make sure the recipe type exists
try:
recipe_type = RecipeType.objects.get(pk=recipe_type_id)
except RecipeType.DoesNotExist:
raise BadParameter('Unknown recipe type: %d' % recipe_type_id)
# Validate and create the batch
try:
definition = BatchDefinitionV6(definition=definition_dict,
do_validate=True).get_definition()
configuration = BatchConfigurationV6(
configuration=configuration_dict,
do_validate=True).get_configuration()
with transaction.atomic():
event = TriggerEvent.objects.create_trigger_event(
'USER', None, {'user': '******'}, now())
batch = Batch.objects.create_batch_v6(
title,
description,
recipe_type,
event,
definition,
configuration=configuration)
CommandMessageManager().send_messages(
[create_batch_recipes_message(batch.id)])
except InvalidDefinition as ex:
raise BadParameter(unicode(ex))
except InvalidConfiguration as ex:
raise BadParameter(unicode(ex))
# Fetch the full batch with details
batch = Batch.objects.get_details_v6(batch.id)
url = reverse('batch_details_view', args=[batch.id], request=request)
serializer = BatchDetailsSerializerV6(batch)
return Response(serializer.data,
status=status.HTTP_201_CREATED,
headers={'Location': url})
def test_create_successful_v6(self):
"""Tests calling BatchManager.create_batch_v6"""
batch = batch_test_utils.create_batch(recipe_type=self.recipe_type_v6)
batch = Batch.objects.get(pk=batch.id)
self.assertIsNotNone(batch.title)
self.assertIsNotNone(batch.description)
self.assertEqual(batch.recipe_type, self.recipe_type_v6)
# Create message
from batch.messages.create_batch_recipes import create_batch_recipes_message
message = create_batch_recipes_message(batch.id)
result = message.execute()
self.assertTrue(result)
batch = Batch.objects.get(pk=batch.id)
self.assertTrue(batch.is_creation_done)
def handle(self, *args, **options):
"""See :meth:`django.core.management.base.BaseCommand.handle`.
This method starts the Scale batch creation process.
"""
batch_id = options.get('batch_id')
logger.info('Command starting: scale_batch_creator - Batch ID: %i',
batch_id)
# Schedule all the batch recipes
try:
batch = Batch.objects.get(id=batch_id)
CommandMessageManager().send_messages(
[create_batch_recipes_message(batch_id)])
except Batch.DoesNotExist:
logger.exception('Unable to find batch: %i', batch_id)
sys.exit(1)
logger.info('Command completed: scale_batch_creator')
def test_json_new(self):
"""Tests coverting a CreateBatchRecipes message to and from JSON"""
jt_1 = job_test_utils.create_seed_job_type()
jt_2 = job_test_utils.create_seed_job_type()
jt_3 = job_test_utils.create_seed_job_type()
jt_4 = job_test_utils.create_seed_job_type()
recipe_def = {
'version': '7',
'input': {
'files': [{
'name': 'INPUT_IMAGE',
'media_types': ['image/png'],
'required': True,
'multiple': False
}],
'json': []
},
'nodes': {
'node_a': {
'dependencies': [],
'input': {
'input_a': {
'type': 'recipe',
'input': 'INPUT_IMAGE'
}
},
'node_type': {
'node_type': 'job',
'job_type_name': jt_1.name,
'job_type_version': jt_1.version,
'job_type_revision': jt_1.revision_num
}
},
'node_b': {
'dependencies': [],
'input': {
'input_a': {
'type': 'recipe',
'input': 'INPUT_IMAGE'
}
},
'node_type': {
'node_type': 'job',
'job_type_name': jt_2.name,
'job_type_version': jt_2.version,
'job_type_revision': jt_2.revision_num
}
}
}
}
sub_recipe_type = recipe_test_utils.create_recipe_type_v6(
definition=recipe_def)
sub_recipe = recipe_test_utils.create_recipe(
recipe_type=sub_recipe_type)
# Recipe with two jobs and one subrecipe (c -> d -> r)
recipe_def = {
'version': '7',
'input': {
'files': [{
'name': 'INPUT_IMAGE',
'media_types': ['image/png'],
'required': True,
'multiple': False
}],
'json': []
},
'nodes': {
'recipe_node': {
'dependencies': [{
'name': 'node_d',
'acceptance': True
}],
'input': {
'input_a': {
'type': 'dependency',
'node': 'node_d',
'output': 'OUTPUT_IMAGE'
}
},
'node_type': {
'node_type': 'recipe',
'recipe_type_name': sub_recipe_type.name,
'recipe_type_revision': sub_recipe_type.revision_num
}
},
'node_c': {
'dependencies': [],
'input': {
'INPUT_IMAGE': {
'type': 'recipe',
'input': 'INPUT_IMAGE'
}
},
'node_type': {
'node_type': 'job',
'job_type_name': jt_3.name,
'job_type_version': jt_3.version,
'job_type_revision': jt_3.revision_num
}
},
'node_d': {
'dependencies': [{
'name': 'node_c',
'acceptance': True
}],
'input': {
'INPUT_IMAGE': {
'type': 'dependency',
'node': 'node_c',
'output': 'OUTPUT_IMAGE'
}
},
'node_type': {
'node_type': 'job',
'job_type_name': jt_4.name,
'job_type_version': jt_4.version,
'job_type_revision': jt_4.revision_num
}
}
}
}
recipe_type = recipe_test_utils.create_recipe_type_v6(
definition=recipe_def)
# Create a dataset of 6 files
dataset_def = {
'parameters': {
'files': [{
'media_types': ['image/png'],
'required': True,
'multiple': False,
'name': 'INPUT_IMAGE'
}],
'json': []
}
}
the_dataset = data_test_utils.create_dataset(definition=dataset_def)
workspace = storage_test_utils.create_workspace()
src_file_a = storage_test_utils.create_file(file_name='input_a.PNG',
file_type='SOURCE',
media_type='image/png',
file_size=10,
data_type_tags=['type'],
file_path='the_path',
workspace=workspace)
src_file_b = storage_test_utils.create_file(file_name='input_b.PNG',
file_type='SOURCE',
media_type='image/png',
file_size=10,
data_type_tags=['type'],
file_path='the_path',
workspace=workspace)
data_list = []
data_dict = {
'version': '6',
'files': {
'INPUT_IMAGE': [src_file_a.id]
},
'json': {}
}
data_list.append(DataV6(data=data_dict).get_dict())
data_dict = {
'version': '6',
'files': {
'INPUT_IMAGE': [src_file_b.id]
},
'json': {}
}
data_list.append(DataV6(data=data_dict).get_dict())
member_2 = data_test_utils.create_dataset_members(dataset=the_dataset,
data_list=data_list)
# Create the batch
batch_definition = BatchDefinition()
batch_definition.dataset = the_dataset.id
forced_nodes = ForcedNodes()
forced_nodes.all_nodes = True
batch_definition.forced_nodes = forced_nodes
batch = batch_test_utils.create_batch(recipe_type=recipe_type,
definition=batch_definition)
# Create the message
message = create_batch_recipes_message(batch.id)
# Convert message to JSON and back, and then execute
message_json_dict = message.to_json()
new_message = CreateBatchRecipes.from_json(message_json_dict)
result = new_message.execute()
self.assertTrue(result)
# Should be two create_recipes message for the two files in the dataset
self.assertEqual(len(new_message.new_messages), 2)
# Verify each message has a different input
src_ids = [src_file_a.id, src_file_b.id]
for message in new_message.new_messages:
self.assertEqual(message.type, 'create_recipes')
self.assertEqual(message.create_recipes_type, 'new-recipe')
file_id = DataV6(message.recipe_input_data).get_data(
).values['INPUT_IMAGE'].file_ids[0]
self.assertTrue(file_id in src_ids)
src_ids.remove(file_id)
# Test re-processing existing recipes
data_dict = {
'version': '6',
'files': {
'INPUT_IMAGE': [src_file_a.id]
},
'json': {}
}
recipe_1 = recipe_test_utils.create_recipe(recipe_type=recipe_type,
input=data_dict)
data_dict = {
'version': '6',
'files': {
'INPUT_IMAGE': [src_file_b.id]
},
'json': {}
}
recipe_2 = recipe_test_utils.create_recipe(recipe_type=recipe_type,
input=data_dict)
recipe_test_utils.process_recipe_inputs([recipe_1.id, recipe_2.id])
batch_definition_2 = BatchDefinition()
batch_definition_2.dataset = the_dataset.id
forced_nodes = ForcedNodes()
forced_nodes.all_nodes = True
batch_definition_2.forced_nodes = forced_nodes
batch_2 = batch_test_utils.create_batch(recipe_type=recipe_type,
definition=batch_definition_2)
# Create the message
message = create_batch_recipes_message(batch_2.id)
# Convert message to JSON and back, and then execute
message_json_dict_2 = message.to_json()
new_message_2 = CreateBatchRecipes.from_json(message_json_dict_2)
result_2 = new_message_2.execute()
self.assertTrue(result_2)
self.assertEqual(len(new_message_2.new_messages), 1)
message = new_message_2.new_messages[0]
self.assertEqual(message.type, 'create_recipes')
self.assertEqual(message.create_recipes_type, 'reprocess')
self.assertSetEqual(set(message.root_recipe_ids),
{recipe_1.id, recipe_2.id})