def test_validate(self):
"""Tests calling Data.validate()"""
interface = Interface()
data = Data()
interface.add_parameter(FileParameter('input_1', ['application/json']))
interface.add_parameter(JsonParameter('input_2', 'integer'))
data.add_value(FileValue('input_1', [123]))
data.add_value(JsonValue('input_2', 100))
data.add_value(JsonValue('extra_input_1', 'hello'))
data.add_value(JsonValue('extra_input_2', 'there'))
# Valid data
data.validate(interface)
# Ensure extra data values are removed
self.assertSetEqual(set(data.values.keys()), {'input_1', 'input_2'})
# Data is missing required input 3
interface.add_parameter(FileParameter('input_3', ['image/gif'], required=True))
with self.assertRaises(InvalidData) as context:
data.validate(interface)
self.assertEqual(context.exception.error.name, 'PARAM_REQUIRED')
data.add_value(FileValue('input_3', [999])) # Input 3 taken care of now
# Invalid data
interface.add_parameter(JsonParameter('input_4', 'string'))
mock_value = MagicMock()
mock_value.name = 'input_4'
mock_value.validate.side_effect = InvalidData('MOCK', '')
data.add_value(mock_value)
with self.assertRaises(InvalidData) as context:
data.validate(interface)
self.assertEqual(context.exception.error.name, 'MOCK')
def test_validate_connection(self):
"""Tests calling JsonParameter.validate_connection()"""
json_param = JsonParameter('input_1', 'string')
connecting_param = FileParameter('input_1', ['application/json'])
# Invalid parameter type
with self.assertRaises(InvalidInterfaceConnection) as context:
json_param.validate_connection(connecting_param)
self.assertEqual(context.exception.error.name, 'MISMATCHED_PARAM_TYPE')
# Parameter is required
connecting_param = JsonParameter('input_1', 'string', required=False)
with self.assertRaises(InvalidInterfaceConnection) as context:
json_param.validate_connection(connecting_param)
self.assertEqual(context.exception.error.name, 'PARAM_REQUIRED')
# Mismatched JSON type
connecting_param = JsonParameter('input_1', 'integer')
with self.assertRaises(InvalidInterfaceConnection) as context:
json_param.validate_connection(connecting_param)
self.assertEqual(context.exception.error.name, 'MISMATCHED_JSON_TYPE')
# Valid parameter connection
connecting_param = JsonParameter('input_1', 'string')
warnings = json_param.validate_connection(connecting_param)
self.assertListEqual(warnings, [])
def test_convert_definition_to_v6_json(self):
"""Tests calling convert_data_to_v6_json()"""
# Try interface with nothing set
definition = DataSetDefinitionV6()
json = convert_definition_to_v6_json(definition.get_definition())
DataSetDefinitionV6(definition=json.get_dict(),
do_validate=True) # Revalidate
# Try data with a variety of values
definition = DataSetDefinition(definition={})
file_param = FileParameter('input_a', ['application/json'])
json_param = JsonParameter('input_b', 'integer')
file_param2 = FileParameter('input_c', ['application/json'])
json_param2 = JsonParameter('input_d', 'integer')
definition.add_global_parameter(file_param)
definition.add_global_parameter(json_param)
definition.add_global_value(FileValue('input_a', [123]))
definition.add_global_value(JsonValue('input_b', 100))
definition.add_parameter(file_param2)
definition.add_parameter(json_param2)
json = convert_definition_to_v6_json(definition)
DataSetDefinitionV6(definition=json.get_dict(),
do_validate=True) # Revalidate
self.assertSetEqual(set(json.get_definition().get_parameters()),
{'input_a', 'input_b', 'input_c', 'input_d'})
def test_convert_recipe_definition_to_v6_json_full(self):
"""Tests calling convert_recipe_definition_to_v6_json() with a full definition"""
interface = Interface()
interface.add_parameter(FileParameter('file_param_a', ['image/gif']))
interface.add_parameter(JsonParameter('json_param_a', 'object'))
interface.add_parameter(
JsonParameter('json_param_b', 'object', required=False))
definition = RecipeDefinition(interface)
definition.add_job_node('A', 'job_type_1', '1.0', 1)
definition.add_job_node('B', 'job_type_2', '2.0', 1)
definition.add_job_node('C', 'job_type_3', '1.0', 2)
definition.add_recipe_node('D', 'recipe_type_1', 1)
definition.add_job_node('E', 'job_type_4', '1.0', 1)
definition.add_dependency('A', 'B')
definition.add_dependency('A', 'C')
definition.add_dependency('B', 'E')
definition.add_dependency('C', 'D')
definition.add_recipe_input_connection('A', 'input_1', 'file_param_a')
definition.add_dependency_input_connection('B', 'b_input_1', 'A',
'a_output_1')
definition.add_dependency_input_connection('C', 'c_input_1', 'A',
'a_output_2')
definition.add_dependency_input_connection('D', 'd_input_1', 'C',
'c_output_1')
definition.add_recipe_input_connection('D', 'd_input_2',
'json_param_a')
json = convert_recipe_definition_to_v6_json(definition)
RecipeDefinitionV6(definition=json.get_dict(),
do_validate=True) # Revalidate
self.assertSetEqual(set(json.get_dict()['nodes'].keys()),
{'A', 'B', 'C', 'D', 'E'})
def test_convert_recipe_diff_to_v6_json_with_changes(self):
"""Tests calling convert_recipe_diff_to_v6_json() with a diff containing a variety of changes"""
interface_1 = Interface()
interface_1.add_parameter(FileParameter('file_param_1', ['image/gif']))
interface_1.add_parameter(JsonParameter('json_param_1', 'object'))
interface_2 = Interface()
interface_2.add_parameter(FileParameter('file_param_1', ['image/gif']))
interface_2.add_parameter(JsonParameter('json_param_1', 'object'))
interface_2.add_parameter(
JsonParameter('json_param_2', 'object', required=False))
definition_1 = RecipeDefinition(interface_1)
definition_1.add_job_node('A', 'job_type_1', '1.0', 1)
definition_1.add_job_node('B', 'job_type_2', '2.0', 1)
definition_1.add_job_node('C', 'job_type_3', '1.0', 2)
definition_1.add_recipe_node('D', 'recipe_type_1', 1)
definition_1.add_job_node('E', 'job_type_4', '1.0', 1)
definition_1.add_dependency('A', 'B')
definition_1.add_dependency('A', 'C')
definition_1.add_dependency('B', 'E')
definition_1.add_dependency('C', 'D')
definition_1.add_recipe_input_connection('A', 'input_1',
'file_param_1')
definition_1.add_dependency_input_connection('B', 'b_input_1', 'A',
'a_output_1')
definition_1.add_dependency_input_connection('C', 'c_input_1', 'A',
'a_output_2')
definition_1.add_dependency_input_connection('D', 'd_input_1', 'C',
'c_output_1')
definition_1.add_recipe_input_connection('D', 'd_input_2',
'json_param_1')
definition_2 = RecipeDefinition(interface_2)
# Nodes B and E are deleted
definition_2.add_job_node('A', 'job_type_1', '1.0', 1)
definition_2.add_job_node('C', 'job_type_3', '2.1',
1) # Change to job type version and revision
definition_2.add_recipe_node('D', 'recipe_type_1', 1)
definition_2.add_recipe_node('F', 'recipe_type_2', 5) # New node
definition_2.add_dependency('A', 'C')
definition_2.add_dependency('C', 'D')
definition_2.add_dependency('D', 'F')
definition_2.add_recipe_input_connection('A', 'input_1',
'file_param_1')
definition_2.add_dependency_input_connection('C', 'c_input_1', 'A',
'a_output_2')
definition_2.add_dependency_input_connection('D', 'd_input_1', 'C',
'c_output_1')
definition_2.add_recipe_input_connection('D', 'd_input_2',
'json_param_1')
definition_2.add_recipe_input_connection('F', 'f_input_1',
'json_param_2')
diff = RecipeDiff(definition_1, definition_2)
json = convert_recipe_diff_to_v6_json(diff)
RecipeDiffV6(diff=json.get_dict(), do_validate=True) # Revalidate
self.assertTrue(json.get_dict()['can_be_reprocessed'])
def test_convert_recipe_diff_to_v6_json_new_required_input(self):
"""Tests calling convert_recipe_diff_to_v6_json() with a diff where there is a breaking recipe interface
change
"""
interface_1 = Interface()
interface_1.add_parameter(FileParameter('file_param_1', ['image/gif']))
interface_1.add_parameter(JsonParameter('json_param_1', 'object'))
interface_2 = Interface()
interface_2.add_parameter(FileParameter('file_param_1', ['image/gif']))
interface_2.add_parameter(JsonParameter('json_param_1', 'object'))
interface_2.add_parameter(
JsonParameter('json_param_2', 'object', required=True))
definition_1 = RecipeDefinition(interface_1)
definition_1.add_job_node('A', 'job_type_1', '1.0', 1)
definition_1.add_job_node('B', 'job_type_2', '2.0', 1)
definition_1.add_job_node('C', 'job_type_3', '1.0', 2)
definition_1.add_recipe_node('D', 'recipe_type_1', 1)
definition_1.add_dependency('A', 'B')
definition_1.add_dependency('A', 'C')
definition_1.add_dependency('C', 'D')
definition_1.add_recipe_input_connection('A', 'input_1',
'file_param_1')
definition_1.add_dependency_input_connection('B', 'b_input_1', 'A',
'a_output_1')
definition_1.add_dependency_input_connection('C', 'c_input_1', 'A',
'a_output_2')
definition_1.add_dependency_input_connection('D', 'd_input_1', 'C',
'c_output_1')
definition_1.add_recipe_input_connection('D', 'd_input_2',
'json_param_1')
definition_2 = RecipeDefinition(interface_2)
definition_2.add_job_node('A', 'job_type_1', '1.0', 1)
definition_2.add_job_node('B', 'job_type_2', '2.0', 1)
definition_2.add_job_node('C', 'job_type_3', '1.1',
1) # Change to job type version and revision
definition_2.add_recipe_node('D', 'recipe_type_1', 1)
definition_2.add_dependency('A', 'B')
definition_2.add_dependency('A', 'C')
definition_2.add_dependency('C', 'D')
definition_2.add_recipe_input_connection('A', 'input_1',
'file_param_1')
definition_2.add_dependency_input_connection('B', 'b_input_1', 'A',
'a_output_1')
definition_2.add_dependency_input_connection('C', 'c_input_1', 'A',
'a_output_2')
definition_2.add_dependency_input_connection('D', 'd_input_1', 'C',
'c_output_1')
definition_2.add_recipe_input_connection('D', 'd_input_2',
'json_param_1')
diff = RecipeDiff(definition_1, definition_2)
json = convert_recipe_diff_to_v6_json(diff)
RecipeDiffV6(diff=json.get_dict(), do_validate=True) # Revalidate
self.assertFalse(json.get_dict()['can_be_reprocessed'])
def test_init_new_required_input(self):
"""Tests creating a RecipeDiff when the newer definition has a new required input parameter"""
interface_1 = Interface()
interface_1.add_parameter(FileParameter('file_param_1', ['image/gif']))
interface_1.add_parameter(JsonParameter('json_param_1', 'object'))
interface_2 = Interface()
interface_2.add_parameter(FileParameter('file_param_1', ['image/gif']))
interface_2.add_parameter(JsonParameter('json_param_1', 'object'))
interface_2.add_parameter(JsonParameter('json_param_2', 'object', required=True))
definition_1 = RecipeDefinition(interface_1)
definition_1.add_job_node('A', 'job_type_1', '1.0', 1)
definition_1.add_job_node('B', 'job_type_2', '2.0', 1)
definition_1.add_job_node('C', 'job_type_3', '1.0', 2)
definition_1.add_recipe_node('D', 'recipe_type_1', 1)
definition_1.add_dependency('A', 'B')
definition_1.add_dependency('A', 'C')
definition_1.add_dependency('C', 'D')
definition_1.add_recipe_input_connection('A', 'input_1', 'file_param_1')
definition_1.add_dependency_input_connection('B', 'b_input_1', 'A', 'a_output_1')
definition_1.add_dependency_input_connection('C', 'c_input_1', 'A', 'a_output_2')
definition_1.add_dependency_input_connection('D', 'd_input_1', 'C', 'c_output_1')
definition_1.add_recipe_input_connection('D', 'd_input_2', 'json_param_1')
definition_2 = RecipeDefinition(interface_2)
definition_2.add_job_node('A', 'job_type_1', '1.0', 1)
definition_2.add_job_node('B', 'job_type_2', '2.0', 1)
definition_2.add_job_node('C', 'job_type_3', '1.1', 1) # Change to job type version and revision
definition_2.add_recipe_node('D', 'recipe_type_1', 1)
definition_2.add_dependency('A', 'B')
definition_2.add_dependency('A', 'C')
definition_2.add_dependency('C', 'D')
definition_2.add_recipe_input_connection('A', 'input_1', 'file_param_1')
definition_2.add_dependency_input_connection('B', 'b_input_1', 'A', 'a_output_1')
definition_2.add_dependency_input_connection('C', 'c_input_1', 'A', 'a_output_2')
definition_2.add_dependency_input_connection('D', 'd_input_1', 'C', 'c_output_1')
definition_2.add_recipe_input_connection('D', 'd_input_2', 'json_param_1')
diff = RecipeDiff(definition_1, definition_2)
self.assertFalse(diff.can_be_reprocessed)
self.assertEqual(len(diff.reasons), 1)
self.assertEqual(diff.reasons[0].name, 'INPUT_CHANGE')
# Cannot be reprocessed, so no nodes to copy, supersede, or unpublish
self.assertDictEqual(diff.get_nodes_to_copy(), {})
self.assertDictEqual(diff.get_nodes_to_supersede(), {})
self.assertDictEqual(diff.get_nodes_to_unpublish(), {})
# Ensure no nodes have reprocess_new_node set to true
for node_diff in diff.graph.values():
self.assertFalse(node_diff.reprocess_new_node)
def test_validate(self):
"""Tests calling JsonParameter.validate()"""
connecting_param = FileParameter('input_1', ['application/json'])
# Invalid JSON type
json_param = JsonParameter('input_1', 'BAD')
with self.assertRaises(InvalidInterface) as context:
json_param.validate()
self.assertEqual(context.exception.error.name, 'INVALID_JSON_TYPE')
# Valid JSON parameter
json_param = JsonParameter('input_1', 'string')
warnings = json_param.validate()
self.assertListEqual(warnings, [])
def test_validate(self):
"""Tests calling FileValue.validate()"""
file_param = FileParameter('input_1', ['application/json'])
json_param = JsonParameter('input_1', 'string')
file_value = FileValue('input_1', [1234, 1235])
# Invalid parameter type
with self.assertRaises(InvalidData) as context:
file_value.validate(json_param)
self.assertEqual(context.exception.error.name, 'MISMATCHED_PARAM_TYPE')
# Zero files not accepted
file_value = FileValue('input_1', [])
with self.assertRaises(InvalidData) as context:
file_value.validate(file_param)
self.assertEqual(context.exception.error.name, 'NO_FILES')
# Multiple files not accepted
file_value = FileValue('input_1', [1234, 1235])
with self.assertRaises(InvalidData) as context:
file_value.validate(file_param)
self.assertEqual(context.exception.error.name, 'MULTIPLE_FILES')
# Valid data value
file_value = FileValue('input_1', [1234])
warnings = file_value.validate(file_param)
self.assertListEqual(warnings, [])
def test_validate_connection(self):
"""Tests calling Interface.validate_connection()"""
interface = Interface()
connecting_interface = Interface()
file_param = FileParameter('input_1', ['application/json'])
interface.add_parameter(file_param)
connecting_interface.add_parameter(file_param)
json_param = JsonParameter('input_2', 'integer')
interface.add_parameter(json_param)
connecting_interface.add_parameter(json_param)
# Valid connection
interface.validate_connection(connecting_interface)
new_file_param = FileParameter('input_3', ['image/gif'], required=True)
interface.add_parameter(new_file_param)
# Connection is missing required input 3
with self.assertRaises(InvalidInterfaceConnection) as context:
interface.validate_connection(connecting_interface)
self.assertEqual(context.exception.error.name, 'PARAM_REQUIRED')
connecting_interface.add_parameter(new_file_param)
mock_param = MagicMock()
mock_param.name = 'input_4'
mock_param.validate_connection.side_effect = InvalidInterfaceConnection('MOCK', '')
interface.add_parameter(mock_param)
connecting_interface.add_parameter(mock_param)
# Invalid connection
with self.assertRaises(InvalidInterfaceConnection) as context:
interface.validate_connection(connecting_interface)
self.assertEqual(context.exception.error.name, 'MOCK')
def test_convert_recipe_to_v6_json(self):
"""Tests calling convert_recipe_to_v6_json() successfully"""
job_type_1 = job_test_utils.create_seed_job_type()
job_type_2 = job_test_utils.create_seed_job_type()
job_type_3 = job_test_utils.create_seed_job_type()
job_type_4 = job_test_utils.create_seed_job_type()
recipe_type_1 = recipe_test_utils.create_recipe_type_v6()
interface = Interface()
interface.add_parameter(FileParameter('file_param_1', ['image/gif']))
interface.add_parameter(JsonParameter('json_param_1', 'object'))
df1 = DataFilter(filter_list=[{'name': 'file_param_1', 'type': 'media-type', 'condition': '==', 'values': ['image/gif']},
{'name': 'json_param_1', 'type': 'object', 'condition': 'superset of', 'values': [{}]}],
all=False)
definition = RecipeDefinition(interface)
definition.add_job_node('A', job_type_1.name, job_type_1.version, job_type_1.revision_num)
definition.add_job_node('B', job_type_2.name, job_type_2.version, job_type_2.revision_num)
definition.add_job_node('C', job_type_3.name, job_type_3.version, job_type_3.revision_num)
definition.add_recipe_node('D', recipe_type_1.name, recipe_type_1.revision_num)
definition.add_job_node('E', job_type_4.name, job_type_4.version, job_type_4.revision_num)
definition.add_condition_node('F', interface, df1) #False
definition.add_job_node('G', job_type_4.name, job_type_4.version, job_type_4.revision_num)
definition.add_dependency('A', 'B')
definition.add_dependency('A', 'C')
definition.add_dependency('B', 'E')
definition.add_dependency('C', 'D')
definition.add_dependency('A', 'F')
definition.add_dependency('F', 'G')
definition.add_recipe_input_connection('A', 'input_1', 'file_param_1')
definition.add_dependency_input_connection('B', 'b_input_1', 'A', 'a_output_1')
definition.add_dependency_input_connection('C', 'c_input_1', 'A', 'a_output_2')
definition.add_dependency_input_connection('D', 'd_input_1', 'C', 'c_output_1')
definition.add_recipe_input_connection('D', 'd_input_2', 'json_param_1')
recipe = recipe_test_utils.create_recipe()
job_a = job_test_utils.create_job(job_type=job_type_1, status='COMPLETED', save=False)
job_b = job_test_utils.create_job(job_type=job_type_2, status='RUNNING', save=False)
job_c = job_test_utils.create_job(job_type=job_type_3, status='COMPLETED', save=False)
job_e = job_test_utils.create_job(job_type=job_type_4, status='PENDING', num_exes=0, save=False)
Job.objects.bulk_create([job_a, job_b, job_c, job_e])
condition_f = recipe_test_utils.create_recipe_condition(is_processed=True, is_accepted=False, save=True)
recipe_d = recipe_test_utils.create_recipe(recipe_type=recipe_type_1)
recipe_node_a = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='A', job=job_a, save=False)
recipe_node_b = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='B', job=job_b, save=False)
recipe_node_c = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='C', job=job_c, save=False)
recipe_node_d = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='D', sub_recipe=recipe_d,
save=False)
recipe_node_e = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='E', job=job_e, save=False)
recipe_node_f = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='F', condition=condition_f,
save=False)
recipe_nodes = [recipe_node_a, recipe_node_b, recipe_node_c, recipe_node_d, recipe_node_e, recipe_node_f]
recipe_instance = RecipeInstance(definition, recipe, recipe_nodes)
json = convert_recipe_to_v6_json(recipe_instance)
RecipeInstanceV6(json=json.get_dict(), do_validate=True) # Revalidate
self.assertSetEqual(set(json.get_dict()['nodes'].keys()), {'A', 'B', 'C', 'D', 'E', 'F'})
def test_json(self):
"""Tests converting a ProcessCondition message to and from JSON"""
definition = RecipeDefinition(Interface())
cond_interface_1 = Interface()
cond_interface_1.add_parameter(JsonParameter('cond_int', 'integer'))
df1 = DataFilter(filter_list=[{
'name': 'cond_int',
'type': 'integer',
'condition': '==',
'values': [0]
}])
definition = RecipeDefinition(cond_interface_1)
definition.add_condition_node('node_a', cond_interface_1, df1)
definition.add_recipe_input_connection('node_a', 'cond_int',
'cond_int')
definition_dict = convert_recipe_definition_to_v6_json(
definition).get_dict()
recipe_type = recipe_test_utils.create_recipe_type_v6(
definition=definition_dict)
data_1 = Data()
data_1.add_value(JsonValue('cond_int', 0))
data_1_dict = convert_data_to_v6_json(data_1).get_dict()
recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type,
input=data_1_dict)
condition = recipe_test_utils.create_recipe_condition(recipe=recipe,
save=True)
recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='node_a',
condition=condition,
save=True)
# Create message
message = create_process_condition_messages([condition.id])[0]
# Convert message to JSON and back, and then execute
message_json_dict = message.to_json()
new_message = ProcessCondition.from_json(message_json_dict)
result = new_message.execute()
self.assertTrue(result)
condition = RecipeCondition.objects.get(id=condition.id)
self.assertEqual(len(new_message.new_messages), 1)
self.assertEqual(new_message.new_messages[0].type, 'update_recipe')
self.assertEqual(new_message.new_messages[0].root_recipe_id, recipe.id)
self.assertTrue(condition.is_processed)
self.assertIsNotNone(condition.processed)
self.assertTrue(condition.is_accepted)
def test_generate_node_input_data(self):
"""Tests calling RecipeDefinition.generate_node_input_data()"""
input_interface = Interface()
input_interface.add_parameter(
FileParameter('recipe_input_1', ['image/gif'], multiple=True))
input_interface.add_parameter(JsonParameter('recipe_input_2',
'string'))
definition = RecipeDefinition(input_interface)
definition.add_job_node('node_a', 'job_type_1', '1.0', 1)
definition.add_job_node('node_b', 'job_type_2', '1.0', 1)
definition.add_job_node('node_c', 'job_type_3', '1.0', 1)
definition.add_dependency('node_c', 'node_b')
definition.add_dependency('node_c', 'node_a')
definition.add_recipe_input_connection('node_c', 'input_1',
'recipe_input_1')
definition.add_recipe_input_connection('node_c', 'input_2',
'recipe_input_2')
definition.add_dependency_input_connection('node_c', 'input_3',
'node_a', 'output_a_1')
definition.add_dependency_input_connection('node_c', 'input_4',
'node_a', 'output_a_2')
definition.add_dependency_input_connection('node_c', 'input_5',
'node_b', 'output_b_1')
recipe_data = Data()
recipe_data.add_value(FileValue('recipe_input_1', [1, 2, 3, 4, 5]))
recipe_data.add_value(JsonValue('recipe_input_2', 'Scale is awesome!'))
a_output_data = Data()
a_output_data.add_value(FileValue('output_a_1', [1234]))
a_output_data.add_value(JsonValue('output_a_2', {'foo': 'bar'}))
b_output_data = Data()
b_output_data.add_value(JsonValue('output_b_1', 12.34))
node_outputs = {
'node_a': RecipeNodeOutput('node_a', 'job', 1, a_output_data),
'node_b': RecipeNodeOutput('node_b', 'job', 1, b_output_data)
}
node_data = definition.generate_node_input_data(
'node_c', recipe_data, node_outputs)
self.assertSetEqual(
set(node_data.values.keys()),
{'input_1', 'input_2', 'input_3', 'input_4', 'input_5'})
self.assertListEqual(node_data.values['input_1'].file_ids,
[1, 2, 3, 4, 5])
self.assertEqual(node_data.values['input_2'].value,
'Scale is awesome!')
self.assertListEqual(node_data.values['input_3'].file_ids, [1234])
self.assertDictEqual(node_data.values['input_4'].value, {'foo': 'bar'})
self.assertEqual(node_data.values['input_5'].value, 12.34)
def test_validate(self):
"""Tests calling DataFilter.validate()"""
data_filter = DataFilter(all=False)
data_filter.add_filter({'name': 'input_a', 'type': 'media-type', 'condition': '==', 'values': ['application/json']})
data_filter.add_filter({'name': 'input_b', 'type': 'string', 'condition': 'contains', 'values': ['abcde']})
data_filter.add_filter({'name': 'input_c', 'type': 'integer', 'condition': '>', 'values': ['0']})
data_filter.add_filter({'name': 'input_d', 'type': 'integer', 'condition': 'between', 'values': ['0', '100']})
data_filter.add_filter({'name': 'input_f', 'type': 'meta-data', 'condition': 'in', 'values': [['foo','baz']],
'fields': [['a','b']]})
interface = Interface()
interface.add_parameter(FileParameter('input_a', ['application/json']))
warnings = data_filter.validate(interface)
self.assertEqual(len(warnings), 4)
self.assertEqual(warnings[0].name, 'UNMATCHED_FILTER')
interface.add_parameter(JsonParameter('input_e', 'integer'))
warnings = data_filter.validate(interface)
self.assertEqual(len(warnings), 5)
self.assertEqual(warnings[3].name, 'UNMATCHED_FILTER')
self.assertEqual(warnings[4].name, 'UNMATCHED_PARAMETERS')
interface.add_parameter(JsonParameter('input_b', 'integer'))
with self.assertRaises(InvalidDataFilter) as context:
data_filter.validate(interface)
self.assertEqual(context.exception.error.name, 'MISMATCHED_TYPE')
interface2 = Interface()
interface2.add_parameter(FileParameter('input_a', ['application/json']))
interface2.add_parameter(JsonParameter('input_b', 'string'))
interface2.add_parameter(JsonParameter('input_c', 'integer'))
interface2.add_parameter(JsonParameter('input_d', 'integer'))
interface2.add_parameter(FileParameter('input_f', ['integer']))
warnings = data_filter.validate(interface2)
self.assertEqual(len(warnings), 0)
def test_add_parameter(self):
"""Tests calling Interface.add_parameter()"""
interface = Interface()
file_param = FileParameter('input_1', ['application/json'])
interface.add_parameter(file_param)
json_param = JsonParameter('input_2', 'integer')
interface.add_parameter(json_param)
self.assertSetEqual(set(interface.parameters.keys()), {'input_1', 'input_2'})
# Duplicate parameter
dup_param = FileParameter('input_1', [], required=False)
with self.assertRaises(InvalidInterface) as context:
interface.add_parameter(dup_param)
self.assertEqual(context.exception.error.name, 'DUPLICATE_INPUT')
def test_add_parameter_from_output_interface(self):
"""Tests calling Interface.add_parameter_from_output_interface()"""
interface = Interface()
output_interface = Interface()
file_param = FileParameter('input_1', ['application/json'])
output_interface.add_parameter(file_param)
json_param = JsonParameter('input_2', 'integer')
output_interface.add_parameter(json_param)
interface.add_parameter_from_output_interface('input_1', 'input_1', output_interface)
self.assertSetEqual(set(interface.parameters.keys()), {'input_1'})
# Duplicate parameter
with self.assertRaises(InvalidInterfaceConnection) as context:
interface.add_parameter_from_output_interface('input_1', 'input_1', output_interface)
self.assertEqual(context.exception.error.name, 'DUPLICATE_INPUT')
def test_validate_connection(self):
"""Tests calling FileParameter.validate_connection()"""
file_param = FileParameter('input_1', ['application/json'])
connecting_param = JsonParameter('input_1', 'array')
# Invalid parameter type
with self.assertRaises(InvalidInterfaceConnection) as context:
file_param.validate_connection(connecting_param)
self.assertEqual(context.exception.error.name, 'MISMATCHED_PARAM_TYPE')
# Parameter is required
connecting_param = FileParameter('input_1', ['application/json'],
required=False)
with self.assertRaises(InvalidInterfaceConnection) as context:
file_param.validate_connection(connecting_param)
self.assertEqual(context.exception.error.name, 'PARAM_REQUIRED')
# Multiple files not accepted
connecting_param = FileParameter('input_1', ['application/json'],
multiple=True)
with self.assertRaises(InvalidInterfaceConnection) as context:
file_param.validate_connection(connecting_param)
self.assertEqual(context.exception.error.name, 'NO_MULTIPLE_FILES')
# Valid parameter connection
connecting_param = FileParameter('input_1', ['application/json'])
warnings = file_param.validate_connection(connecting_param)
self.assertListEqual(warnings, [])
# Multiple file parameter can accept single file parameter
file_param = FileParameter('input_1', ['application/json'],
multiple=True)
connecting_param = FileParameter('input_1', ['application/json'])
warnings = file_param.validate_connection(connecting_param)
self.assertListEqual(warnings, [])
# Mismatched media types
file_param = FileParameter('input_1', ['application/json'])
connecting_param = FileParameter('input_1',
['application/json', 'image/gif'])
warnings = file_param.validate_connection(connecting_param)
self.assertEqual(len(warnings), 1)
self.assertEqual(warnings[0].name, 'MISMATCHED_MEDIA_TYPES')
def merge_parameter_map(self, batch, dataset):
"""Returns the dataset parameters merged with the batch configuration input map
:param batch: The batch
:type batch: :class:`batch.models.Batch`
:param dataset: The dataset of the batch
:type dataset: :class:`data.models.DataSet`
:returns: The map of datasest parameters
:rtype: :class:`data.interface.Interface`
"""
# combine the parameters
dataset_definition = dataset.get_definition()
dataset_parameters = dataset_definition.global_parameters
for param in dataset_definition.parameters.parameters:
dataset_parameters.add_parameter(
dataset_definition.parameters.parameters[param])
# map dataset param to inputs if applicable
if batch.get_configuration().input_map:
from data.interface.interface import Interface
from data.interface.parameter import FileParameter, JsonParameter
parameters = Interface()
for param_name in dataset_parameters.parameters:
param = dataset_parameters.parameters[param_name]
for map_param in batch.get_configuration().input_map:
if param_name == map_param['datasetParameter']:
if param.PARAM_TYPE == 'file':
parameters.add_parameter(
FileParameter(map_param['input'],
param.media_types,
required=param.required,
multiple=param.multiple))
elif param.PARAM_TYPE == 'json':
parameters.add_parameter(
JsonParameter(map_param['input'],
param.json_type,
required=param.required,
multiple=param.multiple))
else:
parameters.add_parameter(param)
dataset_parameters = parameters
return dataset_parameters
def get_interface(self):
"""Returns the interface represented by this JSON
:returns: The interface
:rtype: :class:`data.interface.interface.Interface`:
"""
interface = Interface()
for file_dict in self._interface['files']:
file_param = FileParameter(file_dict['name'],
file_dict['media_types'],
file_dict['required'],
file_dict['multiple'])
interface.add_parameter(file_param)
for json_dict in self._interface['json']:
json_param = JsonParameter(json_dict['name'], json_dict['type'],
json_dict['required'])
interface.add_parameter(json_param)
return interface
def test_validate(self):
"""Tests calling Interface.validate()"""
interface = Interface()
file_param = FileParameter('input_1', ['application/json'])
interface.add_parameter(file_param)
json_param = JsonParameter('input_2', 'integer')
interface.add_parameter(json_param)
warnings = interface.validate()
self.assertListEqual(warnings, [])
mock_param = MagicMock()
mock_param.name = 'input_3'
mock_param.validate.side_effect = InvalidInterface('MOCK', '')
interface.add_parameter(mock_param)
# Invalid parameter
with self.assertRaises(InvalidInterface) as context:
interface.validate()
self.assertEqual(context.exception.error.name, 'MOCK')
def test_validate_successful(self):
"""Tests calling RecipeDefinition.validate() successfully"""
recipe_interface = Interface()
recipe_interface.add_parameter(
JsonParameter('recipe_input_1', 'integer'))
definition = RecipeDefinition(recipe_interface)
definition.add_job_node('A', 'job_type_1', '1.0', 1)
condition_interface = Interface()
condition_interface.add_parameter(
JsonParameter('cond_param', 'integer'))
definition.add_condition_node('B', condition_interface,
DataFilter(True))
definition.add_recipe_node('C', 'recipe_type_1', 1)
definition.add_dependency('A', 'B')
definition.add_dependency('B', 'C')
definition.add_recipe_input_connection('A', 'a_input_1',
'recipe_input_1')
definition.add_dependency_input_connection('B', 'cond_param', 'A',
'a_output_1')
definition.add_dependency_input_connection('C', 'c_input_1', 'A',
'a_output_1')
definition.add_dependency_input_connection('C', 'c_input_2', 'B',
'cond_param')
job_input_interface = Interface()
job_input_interface.add_parameter(JsonParameter(
'a_input_1', 'integer'))
job_output_interface = Interface()
job_output_interface.add_parameter(
JsonParameter('a_output_1', 'integer'))
recipe_input_interface = Interface()
recipe_input_interface.add_parameter(
JsonParameter('c_input_1', 'integer'))
recipe_input_interface.add_parameter(
JsonParameter('c_input_2', 'integer'))
input_interfaces = {
'A': job_input_interface,
'C': recipe_input_interface
}
output_interfaces = {'A': job_output_interface, 'C': Interface()}
warnings = definition.validate(input_interfaces, output_interfaces)
self.assertListEqual(warnings, [])
def test_has_completed_true(self):
"""Tests calling Recipe.has_completed() when an entire recipe has completed"""
data_dict = convert_data_to_v6_json(Data()).get_dict()
job_type = job_test_utils.create_seed_job_type()
sub_recipe_type = recipe_test_utils.create_recipe_type_v6()
cond_interface_1 = Interface()
cond_interface_1.add_parameter(JsonParameter('cond_int', 'integer'))
df2 = DataFilter(filter_list=[{
'name': 'cond_int',
'type': 'integer',
'condition': '==',
'values': [0]
}, {
'name': 'cond_int',
'type': 'integer',
'condition': '!=',
'values': [0]
}],
all=True) #always False
definition = RecipeDefinition(Interface())
definition.add_job_node('A', job_type.name, job_type.version,
job_type.revision_num)
definition.add_recipe_node('B', sub_recipe_type.name,
sub_recipe_type.revision_num)
definition.add_job_node('C', job_type.name, job_type.version,
job_type.revision_num)
definition.add_job_node('D', job_type.name, job_type.version,
job_type.revision_num)
definition.add_job_node('E', job_type.name, job_type.version,
job_type.revision_num)
definition.add_job_node('F', job_type.name, job_type.version,
job_type.revision_num)
definition.add_recipe_node('G', sub_recipe_type.name,
sub_recipe_type.revision_num)
definition.add_job_node('H', job_type.name, job_type.version,
job_type.revision_num)
definition.add_condition_node('I', cond_interface_1, df2) #False
definition.add_job_node('J', job_type.name, job_type.version,
job_type.revision_num)
definition.add_dependency('A', 'C')
definition.add_dependency('A', 'E')
definition.add_dependency('A', 'H')
definition.add_dependency('C', 'D')
definition.add_dependency('G', 'H')
definition.add_dependency('A', 'I')
definition.add_dependency('I', 'J')
job_a = job_test_utils.create_job(job_type=job_type,
status='COMPLETED',
output=data_dict,
save=False)
job_c = job_test_utils.create_job(job_type=job_type,
status='COMPLETED',
output=data_dict,
save=False)
job_d = job_test_utils.create_job(job_type=job_type,
status='COMPLETED',
output=data_dict,
save=False)
job_e = job_test_utils.create_job(job_type=job_type,
status='COMPLETED',
output=data_dict,
save=False)
job_f = job_test_utils.create_job(job_type=job_type,
status='COMPLETED',
output=data_dict,
save=False)
job_h = job_test_utils.create_job(job_type=job_type,
status='COMPLETED',
output=data_dict,
save=False)
Job.objects.bulk_create([job_a, job_c, job_d, job_e, job_f, job_h])
condition_i = recipe_test_utils.create_recipe_condition(
is_processed=True, is_accepted=False, save=True)
recipe_b = recipe_test_utils.create_recipe(recipe_type=sub_recipe_type,
save=False)
recipe_b.is_completed = True
recipe_g = recipe_test_utils.create_recipe(recipe_type=sub_recipe_type,
save=False)
recipe_g.is_completed = True
Recipe.objects.bulk_create([recipe_b, recipe_g])
definition_json_dict = convert_recipe_definition_to_v6_json(
definition).get_dict()
recipe_type = recipe_test_utils.create_recipe_type_v6(
definition=definition_json_dict)
recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type)
recipe_node_a = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='A',
job=job_a,
save=False,
is_original=False)
recipe_node_c = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='C',
job=job_c,
save=False,
is_original=False)
recipe_node_d = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='D',
job=job_d,
save=False,
is_original=False)
recipe_node_e = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='E',
job=job_e,
save=False)
recipe_node_f = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='F',
job=job_f,
save=False)
recipe_node_h = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='H',
job=job_h,
save=False)
recipe_node_i = recipe_test_utils.create_recipe_node(
recipe=recipe, node_name='I', condition=condition_i, save=False)
recipe_node_g = recipe_test_utils.create_recipe_node(
recipe=recipe,
node_name='G',
sub_recipe=recipe_g,
save=False,
is_original=False)
recipe_node_b = recipe_test_utils.create_recipe_node(
recipe=recipe, node_name='B', sub_recipe=recipe_b, save=False)
RecipeNode.objects.bulk_create([
recipe_node_a, recipe_node_b, recipe_node_c, recipe_node_d,
recipe_node_e, recipe_node_f, recipe_node_g, recipe_node_h,
recipe_node_i
])
recipe_instance = Recipe.objects.get_recipe_instance(recipe.id)
self.assertTrue(recipe_instance.has_completed())
def test_get_nodes_to_process_input(self):
"""Tests calling Recipe.get_nodes_to_process_input()"""
data_dict = convert_data_to_v6_json(Data()).get_dict()
job_type = job_test_utils.create_seed_job_type()
sub_recipe_type = recipe_test_utils.create_recipe_type_v6()
# Create recipe
definition = RecipeDefinition(Interface())
cond_interface_1 = Interface()
cond_interface_1.add_parameter(JsonParameter('cond_int', 'integer'))
definition.add_job_node('A', job_type.name, job_type.version,
job_type.revision_num)
df1 = DataFilter(filter_list=[{
'name': 'cond_int',
'type': 'integer',
'condition': '==',
'values': [0]
}, {
'name': 'cond_int',
'type': 'integer',
'condition': '!=',
'values': [0]
}],
all=False) #always True
df2 = DataFilter(filter_list=[{
'name': 'cond_int',
'type': 'integer',
'condition': '==',
'values': [0]
}, {
'name': 'cond_int',
'type': 'integer',
'condition': '!=',
'values': [0]
}],
all=True) #always False
definition.add_condition_node('B', cond_interface_1, df1) #True
definition.add_condition_node('C', cond_interface_1, df1) #True
definition.add_condition_node('D', cond_interface_1, df2) #False
definition.add_job_node('E', job_type.name, job_type.version,
job_type.revision_num)
definition.add_job_node('F', job_type.name, job_type.version,
job_type.revision_num)
definition.add_recipe_node('G', sub_recipe_type.name,
sub_recipe_type.revision_num)
definition.add_recipe_node('H', sub_recipe_type.name,
sub_recipe_type.revision_num)
definition.add_dependency('A', 'D')
definition.add_dependency('A', 'E')
definition.add_dependency('B', 'E')
definition.add_dependency('B', 'F')
definition.add_dependency('C', 'F')
definition.add_dependency('D', 'G')
definition.add_dependency('E', 'G')
definition.add_dependency('E', 'H')
definition_json_dict = convert_recipe_definition_to_v6_json(
definition).get_dict()
recipe_type = recipe_test_utils.create_recipe_type_v6(
definition=definition_json_dict)
recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type,
input=data_dict)
# Nodes A, B, and D already exist
job_a = job_test_utils.create_job(job_type=job_type,
status='COMPLETED',
input=data_dict,
output=data_dict,
save=True)
condition_b = recipe_test_utils.create_recipe_condition(
is_processed=True, is_accepted=True, save=False)
condition_d = recipe_test_utils.create_recipe_condition(
is_processed=True, is_accepted=False, save=False)
RecipeCondition.objects.bulk_create([condition_b, condition_d])
recipe_node_a = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='A',
job=job_a,
save=False)
recipe_node_b = recipe_test_utils.create_recipe_node(
recipe=recipe, node_name='B', condition=condition_b, save=False)
recipe_node_d = recipe_test_utils.create_recipe_node(
recipe=recipe, node_name='D', condition=condition_d, save=False)
RecipeNode.objects.bulk_create(
[recipe_node_a, recipe_node_b, recipe_node_d])
recipe_instance = Recipe.objects.get_recipe_instance(recipe.id)
nodes_to_process = recipe_instance.get_nodes_to_process_input()
self.assertSetEqual(set(nodes_to_process.keys()), {'C', 'E'})
def test_validate(self):
"""Tests calling JsonValue.validate()"""
file_param = FileParameter('input_1', ['application/json'])
json_param = JsonParameter('input_1', 'string')
json_value = JsonValue('input_1', 'hello')
# Invalid parameter type
with self.assertRaises(InvalidData) as context:
json_value.validate(file_param)
self.assertEqual(context.exception.error.name, 'MISMATCHED_PARAM_TYPE')
# Invalid array
json_param = JsonParameter('input_1', 'array')
json_value = JsonValue('input_1', 123)
with self.assertRaises(InvalidData) as context:
json_value.validate(json_param)
self.assertEqual(context.exception.error.name, 'INVALID_JSON_TYPE')
# Invalid boolean
json_param = JsonParameter('input_1', 'boolean')
json_value = JsonValue('input_1', 123)
with self.assertRaises(InvalidData) as context:
json_value.validate(json_param)
self.assertEqual(context.exception.error.name, 'INVALID_JSON_TYPE')
# Invalid integer
json_param = JsonParameter('input_1', 'integer')
json_value = JsonValue('input_1', 123.5)
with self.assertRaises(InvalidData) as context:
json_value.validate(json_param)
self.assertEqual(context.exception.error.name, 'INVALID_JSON_TYPE')
# Invalid number
json_param = JsonParameter('input_1', 'number')
json_value = JsonValue('input_1', 'foo')
with self.assertRaises(InvalidData) as context:
json_value.validate(json_param)
self.assertEqual(context.exception.error.name, 'INVALID_JSON_TYPE')
# Invalid object
json_param = JsonParameter('input_1', 'object')
json_value = JsonValue('input_1', 123)
with self.assertRaises(InvalidData) as context:
json_value.validate(json_param)
self.assertEqual(context.exception.error.name, 'INVALID_JSON_TYPE')
# Invalid string
json_param = JsonParameter('input_1', 'string')
json_value = JsonValue('input_1', 123)
with self.assertRaises(InvalidData) as context:
json_value.validate(json_param)
self.assertEqual(context.exception.error.name, 'INVALID_JSON_TYPE')
# Valid array value
json_param = JsonParameter('input_1', 'array')
json_value = JsonValue('input_1', [1, 2, 3])
warnings = json_value.validate(json_param)
self.assertListEqual(warnings, [])
# Valid boolean value
json_param = JsonParameter('input_1', 'boolean')
json_value = JsonValue('input_1', True)
warnings = json_value.validate(json_param)
self.assertListEqual(warnings, [])
# Valid integer value
json_param = JsonParameter('input_1', 'integer')
json_value = JsonValue('input_1', 1234)
warnings = json_value.validate(json_param)
self.assertListEqual(warnings, [])
# Valid number value
json_param = JsonParameter('input_1', 'number')
json_value = JsonValue('input_1', 1234.5)
warnings = json_value.validate(json_param)
self.assertListEqual(warnings, [])
# Valid object value
json_param = JsonParameter('input_1', 'object')
json_value = JsonValue('input_1', {'foo': 'bar'})
warnings = json_value.validate(json_param)
self.assertListEqual(warnings, [])
# Valid string value
json_param = JsonParameter('input_1', 'string')
json_value = JsonValue('input_1', 'hello')
warnings = json_value.validate(json_param)
self.assertListEqual(warnings, [])
def test_convert_recipe_to_v6_json(self):
"""Tests calling convert_recipe_to_v6_json() successfully"""
job_type_1 = job_test_utils.create_job_type()
job_type_2 = job_test_utils.create_job_type()
job_type_3 = job_test_utils.create_job_type()
job_type_4 = job_test_utils.create_job_type()
recipe_type_1 = recipe_test_utils.create_recipe_type()
interface = Interface()
interface.add_parameter(FileParameter('file_param_1', ['image/gif']))
interface.add_parameter(JsonParameter('json_param_1', 'object'))
definition = RecipeDefinition(interface)
definition.add_job_node('A', job_type_1.name, job_type_1.version,
job_type_1.revision_num)
definition.add_job_node('B', job_type_2.name, job_type_2.version,
job_type_2.revision_num)
definition.add_job_node('C', job_type_3.name, job_type_3.version,
job_type_3.revision_num)
definition.add_recipe_node('D', recipe_type_1.name,
recipe_type_1.revision_num)
definition.add_job_node('E', job_type_4.name, job_type_4.version,
job_type_4.revision_num)
definition.add_dependency('A', 'B')
definition.add_dependency('A', 'C')
definition.add_dependency('B', 'E')
definition.add_dependency('C', 'D')
definition.add_recipe_input_connection('A', 'input_1', 'file_param_1')
definition.add_dependency_input_connection('B', 'b_input_1', 'A',
'a_output_1')
definition.add_dependency_input_connection('C', 'c_input_1', 'A',
'a_output_2')
definition.add_dependency_input_connection('D', 'd_input_1', 'C',
'c_output_1')
definition.add_recipe_input_connection('D', 'd_input_2',
'json_param_1')
recipe = recipe_test_utils.create_recipe()
job_a = job_test_utils.create_job(job_type=job_type_1,
status='COMPLETED',
save=False)
job_b = job_test_utils.create_job(job_type=job_type_2,
status='RUNNING',
save=False)
job_c = job_test_utils.create_job(job_type=job_type_3,
status='COMPLETED',
save=False)
job_e = job_test_utils.create_job(job_type=job_type_4,
status='PENDING',
num_exes=0,
save=False)
Job.objects.bulk_create([job_a, job_b, job_c, job_e])
recipe_d = recipe_test_utils.create_recipe(recipe_type=recipe_type_1)
recipe_node_a = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='A',
job=job_a,
save=False)
recipe_node_b = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='B',
job=job_b,
save=False)
recipe_node_c = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='C',
job=job_c,
save=False)
recipe_node_d = recipe_test_utils.create_recipe_node(
recipe=recipe, node_name='D', sub_recipe=recipe_d, save=False)
recipe_node_e = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='E',
job=job_e,
save=False)
recipe_nodes = [
recipe_node_a, recipe_node_b, recipe_node_c, recipe_node_d,
recipe_node_e
]
recipe_instance = RecipeInstance(definition, recipe_nodes)
json = convert_recipe_to_v6_json(recipe_instance)
RecipeInstanceV6(json=json.get_dict(), do_validate=True) # Revalidate
self.assertSetEqual(set(json.get_dict()['nodes'].keys()),
{'A', 'B', 'C', 'D', 'E'})
def test_execute_with_recipe(self):
"""Tests calling ProcessRecipeInput.execute() successfully when a sub-recipe has to get its data from its
recipe
"""
workspace = storage_test_utils.create_workspace()
file_1 = storage_test_utils.create_file(workspace=workspace,
file_size=104857600.0)
file_2 = storage_test_utils.create_file(workspace=workspace,
file_size=987654321.0)
file_3 = storage_test_utils.create_file(workspace=workspace,
file_size=65456.0)
file_4 = storage_test_utils.create_file(workspace=workspace,
file_size=24564165456.0)
manifest_a = {
'seedVersion': '1.0.0',
'job': {
'name': 'job-a',
'jobVersion': '1.0.0',
'packageVersion': '1.0.0',
'title': '',
'description': '',
'maintainer': {
'name': 'John Doe',
'email': '*****@*****.**'
},
'timeout': 10,
'interface': {
'command': '',
'inputs': {
'files': [],
'json': []
},
'outputs': {
'files': [{
'name': 'output_a',
'pattern': '*.png'
}]
}
}
}
}
job_type_a = job_test_utils.create_job_type(interface=manifest_a)
output_data_a = Data()
output_data_a.add_value(FileValue('output_a', [file_1.id]))
output_data_a_dict = convert_data_to_v6_json(output_data_a).get_dict()
manifest_b = {
'seedVersion': '1.0.0',
'job': {
'name': 'job-b',
'jobVersion': '1.0.0',
'packageVersion': '1.0.0',
'title': '',
'description': '',
'maintainer': {
'name': 'John Doe',
'email': '*****@*****.**'
},
'timeout': 10,
'interface': {
'command': '',
'inputs': {
'files': [],
'json': []
},
'outputs': {
'files': [{
'name': 'output_b',
'pattern': '*.png',
'multiple': True
}]
}
}
}
}
job_type_b = job_test_utils.create_job_type(interface=manifest_b)
output_data_b = Data()
output_data_b.add_value(
FileValue('output_b', [file_2.id, file_3.id, file_4.id]))
output_data_b_dict = convert_data_to_v6_json(output_data_b).get_dict()
job_a = job_test_utils.create_job(job_type=job_type_a,
num_exes=1,
status='COMPLETED',
output=output_data_a_dict)
job_b = job_test_utils.create_job(job_type=job_type_b,
num_exes=1,
status='COMPLETED',
output=output_data_b_dict)
sub_recipe_interface_c = Interface()
sub_recipe_interface_c.add_parameter(
FileParameter('input_a', ['image/png']))
sub_recipe_interface_c.add_parameter(
FileParameter('input_b', ['image/png'], multiple=True))
sub_recipe_interface_c.add_parameter(JsonParameter(
'input_c', 'string'))
sub_recipe_def_c = RecipeDefinition(sub_recipe_interface_c)
sub_recipe_def_dict_c = convert_recipe_definition_to_v6_json(
sub_recipe_def_c).get_dict()
sub_recipe_type_c = recipe_test_utils.create_recipe_type(
definition=sub_recipe_def_dict_c)
sub_recipe_c = recipe_test_utils.create_recipe(
recipe_type=sub_recipe_type_c)
recipe_interface = Interface()
recipe_interface.add_parameter(JsonParameter('recipe_input', 'string'))
definition = RecipeDefinition(recipe_interface)
definition.add_job_node('node_a', job_type_a.name, job_type_a.version,
job_type_a.revision_num)
definition.add_job_node('node_b', job_type_b.name, job_type_b.version,
job_type_b.revision_num)
definition.add_recipe_node('node_c', sub_recipe_type_c.name,
sub_recipe_type_c.revision_num)
definition.add_recipe_input_connection('node_c', 'input_c',
'recipe_input')
definition.add_dependency('node_c', 'node_a')
definition.add_dependency_input_connection('node_c', 'input_a',
'node_a', 'output_a')
definition.add_dependency('node_c', 'node_b')
definition.add_dependency_input_connection('node_c', 'input_b',
'node_b', 'output_b')
def_dict = convert_recipe_definition_to_v6_json(definition).get_dict()
recipe_type = recipe_test_utils.create_recipe_type(definition=def_dict)
recipe_data = Data()
recipe_data.add_value(JsonValue('recipe_input', 'hello'))
recipe_data_dict = convert_data_to_v6_json(recipe_data).get_dict()
recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type,
input=recipe_data_dict)
recipe_node_a = recipe_test_utils.create_recipe_node(
recipe=recipe, node_name='node_a', job=job_a)
recipe_node_b = recipe_test_utils.create_recipe_node(
recipe=recipe, node_name='node_b', job=job_b)
recipe_node_c = recipe_test_utils.create_recipe_node(
recipe=recipe, node_name='node_c', sub_recipe=sub_recipe_c)
RecipeNode.objects.bulk_create(
[recipe_node_a, recipe_node_b, recipe_node_c])
job_a.recipe = recipe
job_a.save()
job_b.recipe = recipe
job_b.save()
sub_recipe_c.recipe = recipe
sub_recipe_c.save()
# Create message
message = ProcessRecipeInput()
message.recipe_id = sub_recipe_c.id
# Execute message
result = message.execute()
self.assertTrue(result)
sub_recipe_c = Recipe.objects.get(id=sub_recipe_c.id)
# Check for update_recipes message
self.assertEqual(len(message.new_messages), 1)
self.assertEqual(message.new_messages[0].type, 'update_recipes')
# Check sub-recipe for expected input_file_size
self.assertEqual(sub_recipe_c.input_file_size, 24469.0)
# Check sub-recipe for expected input data
self.assertNotEqual(sub_recipe_c.input['version'],
'1.0') # Should not be legacy
self.assertSetEqual(set(sub_recipe_c.get_input_data().values.keys()),
{'input_a', 'input_b', 'input_c'})
self.assertListEqual(
sub_recipe_c.get_input_data().values['input_a'].file_ids,
[file_1.id])
self.assertListEqual(
sub_recipe_c.get_input_data().values['input_b'].file_ids,
[file_2.id, file_3.id, file_4.id])
self.assertEqual(sub_recipe_c.get_input_data().values['input_c'].value,
'hello')
# Make sure sub-recipe input file models are created
input_files = RecipeInputFile.objects.filter(recipe_id=sub_recipe_c.id)
self.assertEqual(len(input_files), 4)
file_ids = {input_file.input_file_id for input_file in input_files}
self.assertSetEqual(file_ids,
{file_1.id, file_2.id, file_3.id, file_4.id})
# Test executing message again
message_json_dict = message.to_json()
message = ProcessRecipeInput.from_json(message_json_dict)
result = message.execute()
self.assertTrue(result)
# Still should have update_recipes message
self.assertEqual(len(message.new_messages), 1)
self.assertEqual(message.new_messages[0].type, 'update_recipes')
# Make sure recipe input file models are unchanged
input_files = RecipeInputFile.objects.filter(recipe_id=sub_recipe_c.id)
self.assertEqual(len(input_files), 4)
def test_set_force_reprocess(self):
"""Tests calling RecipeDiff.set_force_reprocess()"""
interface_1 = Interface()
interface_1.add_parameter(FileParameter('file_param_1', ['image/gif']))
interface_1.add_parameter(JsonParameter('json_param_1', 'object'))
interface_2 = Interface()
interface_2.add_parameter(FileParameter('file_param_1', ['image/gif']))
interface_2.add_parameter(JsonParameter('json_param_1', 'object'))
definition_1 = RecipeDefinition(interface_1)
definition_1.add_job_node('A', 'job_type_1', '1.0', 1)
definition_1.add_job_node('B', 'job_type_2', '2.0', 1)
definition_1.add_job_node('C', 'job_type_3', '1.0', 2)
definition_1.add_recipe_node('D', 'recipe_type_1', 1)
definition_1.add_job_node('E', 'job_type_4', '1.0', 1)
definition_1.add_dependency('A', 'B')
definition_1.add_dependency('A', 'C')
definition_1.add_dependency('C', 'D')
definition_1.add_dependency('C', 'E')
definition_1.add_recipe_input_connection('A', 'input_1',
'file_param_1')
definition_1.add_dependency_input_connection('B', 'b_input_1', 'A',
'a_output_1')
definition_1.add_dependency_input_connection('C', 'c_input_1', 'A',
'a_output_2')
definition_1.add_dependency_input_connection('D', 'd_input_1', 'C',
'c_output_1')
definition_1.add_recipe_input_connection('D', 'd_input_2',
'json_param_1')
definition_1.add_dependency_input_connection('E', 'e_input_1', 'C',
'c_output_1')
# No changes in definition 2
definition_2 = RecipeDefinition(interface_2)
definition_2.add_job_node('A', 'job_type_1', '1.0', 1)
definition_2.add_job_node('B', 'job_type_2', '2.0', 1)
definition_2.add_job_node('C', 'job_type_3', '1.0', 2)
definition_2.add_recipe_node('D', 'recipe_type_1', 1)
definition_2.add_job_node('E', 'job_type_4', '1.0', 1)
definition_2.add_dependency('A', 'B')
definition_2.add_dependency('A', 'C')
definition_2.add_dependency('C', 'D')
definition_2.add_dependency('C', 'E')
definition_2.add_recipe_input_connection('A', 'input_1',
'file_param_1')
definition_2.add_dependency_input_connection('B', 'b_input_1', 'A',
'a_output_1')
definition_2.add_dependency_input_connection('C', 'c_input_1', 'A',
'a_output_2')
definition_2.add_dependency_input_connection('D', 'd_input_1', 'C',
'c_output_1')
definition_2.add_recipe_input_connection('D', 'd_input_2',
'json_param_1')
definition_2.add_dependency_input_connection('E', 'e_input_1', 'C',
'c_output_1')
recipe_d_forced_nodes = ForcedNodes()
recipe_d_forced_nodes.add_node('1')
recipe_d_forced_nodes.add_node('2')
top_forced_nodes = ForcedNodes()
top_forced_nodes.add_node('C')
top_forced_nodes.add_subrecipe('D', recipe_d_forced_nodes)
diff = RecipeDiff(definition_1, definition_2)
diff.set_force_reprocess(top_forced_nodes)
# No recipe input changes so recipe can be reprocessed
self.assertTrue(diff.can_be_reprocessed)
self.assertListEqual(diff.reasons, [])
# Check each node for correct fields
node_a = diff.graph['A']
self.assertEqual(node_a.status, NodeDiff.UNCHANGED)
self.assertFalse(node_a.reprocess_new_node)
self.assertListEqual(node_a.changes, [])
node_b = diff.graph['B']
self.assertEqual(node_b.status, NodeDiff.UNCHANGED)
self.assertFalse(node_b.reprocess_new_node)
self.assertListEqual(node_b.changes, [])
node_c = diff.graph['C']
self.assertEqual(node_c.status, NodeDiff.UNCHANGED)
self.assertTrue(node_c.reprocess_new_node) # Force reprocess
self.assertListEqual(node_c.changes, [])
node_d = diff.graph['D']
self.assertEqual(node_d.status, NodeDiff.UNCHANGED)
self.assertTrue(node_d.reprocess_new_node) # Force reprocess
self.assertListEqual(node_d.changes, [])
# Check forced nodes object that got passed to recipe node D
self.assertEqual(node_d.force_reprocess_nodes, recipe_d_forced_nodes)
node_e = diff.graph['E']
self.assertEqual(node_e.status, NodeDiff.UNCHANGED)
self.assertTrue(
node_e.reprocess_new_node) # Force reprocess due to C being forced
self.assertListEqual(node_e.changes, [])
# Check nodes to copy, supersede, and unpublish
self.assertSetEqual(set(diff.get_nodes_to_copy().keys()), {'A', 'B'})
self.assertSetEqual(set(diff.get_nodes_to_supersede().keys()),
{'C', 'D', 'E'})
self.assertSetEqual(set(diff.get_nodes_to_unpublish().keys()), set())
def test_init_changes_in_middle_of_chains(self):
"""Tests creating a RecipeDiff where nodes are deleted from and inserted into the middle of a chain"""
interface_1 = Interface()
interface_1.add_parameter(FileParameter('file_param_1', ['image/gif']))
interface_1.add_parameter(JsonParameter('json_param_1', 'object'))
interface_2 = Interface()
interface_2.add_parameter(FileParameter('file_param_1', ['image/gif']))
interface_2.add_parameter(JsonParameter('json_param_1', 'object'))
definition_1 = RecipeDefinition(interface_1)
definition_1.add_job_node('A', 'job_type_1', '1.0', 1)
definition_1.add_job_node('B', 'job_type_2', '2.0', 1)
definition_1.add_job_node('C', 'job_type_3', '1.0', 2)
definition_1.add_recipe_node('D', 'recipe_type_1', 1)
definition_1.add_dependency('A', 'B')
definition_1.add_dependency('A', 'C')
definition_1.add_dependency('C', 'D')
definition_1.add_recipe_input_connection('A', 'input_1',
'file_param_1')
definition_1.add_dependency_input_connection('B', 'b_input_1', 'A',
'a_output_1')
definition_1.add_dependency_input_connection('C', 'c_input_1', 'A',
'a_output_2')
definition_1.add_dependency_input_connection('D', 'd_input_1', 'C',
'c_output_1')
definition_1.add_recipe_input_connection('D', 'd_input_2',
'json_param_1')
definition_2 = RecipeDefinition(interface_2)
definition_2.add_job_node('A', 'job_type_1', '1.0', 1)
definition_2.add_job_node('B', 'job_type_2', '2.0', 1)
# Node C is deleted
definition_2.add_recipe_node('D', 'recipe_type_1', 1)
definition_2.add_job_node('E', 'job_type_4', '5.0',
2) # New node inbetween A and B
definition_2.add_dependency('A', 'E')
definition_2.add_dependency('E', 'B')
definition_2.add_dependency('A', 'D')
definition_2.add_recipe_input_connection('A', 'input_1',
'file_param_1')
definition_2.add_dependency_input_connection('E', 'e_input_1', 'A',
'a_output_1')
definition_2.add_dependency_input_connection('B', 'b_input_1', 'A',
'a_output_1')
definition_2.add_dependency_input_connection('D', 'd_input_1', 'A',
'a_output_2')
definition_2.add_recipe_input_connection('D', 'd_input_2',
'json_param_1')
diff = RecipeDiff(definition_1, definition_2)
# No recipe input changes so recipe can be reprocessed
self.assertTrue(diff.can_be_reprocessed)
self.assertListEqual(diff.reasons, [])
# Check each node for correct fields
node_a = diff.graph['A']
self.assertEqual(node_a.status, NodeDiff.UNCHANGED)
self.assertFalse(node_a.reprocess_new_node)
self.assertListEqual(node_a.changes, [])
node_b = diff.graph['B']
self.assertEqual(node_b.status, NodeDiff.CHANGED)
self.assertTrue(node_b.reprocess_new_node)
self.assertEqual(len(node_b.changes), 2)
self.assertEqual(node_b.changes[0].name, 'PARENT_NEW')
self.assertEqual(node_b.changes[1].name, 'PARENT_REMOVED')
node_c = diff.graph['C']
self.assertEqual(node_c.status, NodeDiff.DELETED)
self.assertFalse(node_c.reprocess_new_node)
self.assertListEqual(node_c.changes, [])
node_d = diff.graph['D']
self.assertEqual(node_d.status, NodeDiff.CHANGED)
self.assertTrue(node_d.reprocess_new_node)
self.assertEqual(len(node_d.changes), 3)
self.assertEqual(node_d.changes[0].name, 'PARENT_NEW')
self.assertEqual(node_d.changes[1].name, 'PARENT_REMOVED')
self.assertEqual(node_d.changes[2].name, 'INPUT_CHANGE')
node_e = diff.graph['E']
self.assertEqual(node_e.status, NodeDiff.NEW)
self.assertTrue(node_e.reprocess_new_node)
self.assertListEqual(node_e.changes, [])
# Check nodes to copy, supersede, and unpublish
self.assertSetEqual(set(diff.get_nodes_to_copy().keys()), {'A'})
self.assertSetEqual(set(diff.get_nodes_to_supersede().keys()),
{'B', 'C', 'D'})
self.assertSetEqual(set(diff.get_nodes_to_unpublish().keys()), {'C'})
def test_init_identical(self):
"""Tests creating a RecipeDiff between two identical recipe definitions"""
interface_1 = Interface()
interface_1.add_parameter(FileParameter('file_param_1', ['image/gif']))
interface_1.add_parameter(JsonParameter('json_param_1', 'object'))
interface_2 = Interface()
interface_2.add_parameter(FileParameter('file_param_1', ['image/gif']))
interface_2.add_parameter(JsonParameter('json_param_1', 'object'))
definition_1 = RecipeDefinition(interface_1)
definition_1.add_job_node('A', 'job_type_1', '1.0', 1)
definition_1.add_job_node('B', 'job_type_2', '2.0', 1)
definition_1.add_job_node('C', 'job_type_3', '1.0', 2)
definition_1.add_recipe_node('D', 'recipe_type_1', 1)
definition_1.add_dependency('A', 'B')
definition_1.add_dependency('A', 'C')
definition_1.add_dependency('C', 'D')
definition_1.add_recipe_input_connection('A', 'input_1',
'file_param_1')
definition_1.add_dependency_input_connection('B', 'b_input_1', 'A',
'a_output_1')
definition_1.add_dependency_input_connection('C', 'c_input_1', 'A',
'a_output_2')
definition_1.add_dependency_input_connection('D', 'd_input_1', 'C',
'c_output_1')
definition_1.add_recipe_input_connection('D', 'd_input_2',
'json_param_1')
definition_2 = RecipeDefinition(interface_2)
definition_2.add_job_node('A', 'job_type_1', '1.0', 1)
definition_2.add_job_node('B', 'job_type_2', '2.0', 1)
definition_2.add_job_node('C', 'job_type_3', '1.0', 2)
definition_2.add_recipe_node('D', 'recipe_type_1', 1)
definition_2.add_dependency('A', 'B')
definition_2.add_dependency('A', 'C')
definition_2.add_dependency('C', 'D')
definition_2.add_recipe_input_connection('A', 'input_1',
'file_param_1')
definition_2.add_dependency_input_connection('B', 'b_input_1', 'A',
'a_output_1')
definition_2.add_dependency_input_connection('C', 'c_input_1', 'A',
'a_output_2')
definition_2.add_dependency_input_connection('D', 'd_input_1', 'C',
'c_output_1')
definition_2.add_recipe_input_connection('D', 'd_input_2',
'json_param_1')
diff = RecipeDiff(definition_1, definition_2)
self.assertTrue(diff.can_be_reprocessed)
self.assertListEqual(diff.reasons, [])
# Every node should be unchanged and all should be copied during a reprocess
nodes_to_copy = diff.get_nodes_to_copy()
self.assertSetEqual(set(nodes_to_copy.keys()), {'A', 'B', 'C', 'D'})
for node_diff in nodes_to_copy.values():
self.assertEqual(node_diff.status, NodeDiff.UNCHANGED)
self.assertFalse(node_diff.reprocess_new_node)
self.assertListEqual(node_diff.changes, [])
self.assertDictEqual(diff.get_nodes_to_supersede(), {})
self.assertDictEqual(diff.get_nodes_to_unpublish(), {})
def test_init_changes(self):
"""Tests creating a RecipeDiff when the newer definition has a variety of changes in it"""
interface_1 = Interface()
interface_1.add_parameter(FileParameter('file_param_1', ['image/gif']))
interface_1.add_parameter(JsonParameter('json_param_1', 'object'))
interface_2 = Interface()
interface_2.add_parameter(FileParameter('file_param_1', ['image/gif']))
interface_2.add_parameter(JsonParameter('json_param_1', 'object'))
interface_2.add_parameter(
JsonParameter('json_param_2', 'object', required=False))
definition_1 = RecipeDefinition(interface_1)
definition_1.add_job_node('A', 'job_type_1', '1.0', 1)
definition_1.add_job_node('B', 'job_type_2', '2.0', 1)
definition_1.add_job_node('C', 'job_type_3', '1.0', 2)
definition_1.add_recipe_node('D', 'recipe_type_1', 1)
definition_1.add_job_node('E', 'job_type_4', '1.0', 1)
definition_1.add_dependency('A', 'B')
definition_1.add_dependency('A', 'C')
definition_1.add_dependency('B', 'E')
definition_1.add_dependency('C', 'D')
definition_1.add_recipe_input_connection('A', 'input_1',
'file_param_1')
definition_1.add_dependency_input_connection('B', 'b_input_1', 'A',
'a_output_1')
definition_1.add_dependency_input_connection('C', 'c_input_1', 'A',
'a_output_2')
definition_1.add_dependency_input_connection('D', 'd_input_1', 'C',
'c_output_1')
definition_1.add_recipe_input_connection('D', 'd_input_2',
'json_param_1')
definition_2 = RecipeDefinition(interface_2)
# Nodes B and E are deleted
definition_2.add_job_node('A', 'job_type_1', '1.0', 1)
definition_2.add_job_node('C', 'job_type_3', '2.1',
1) # Change to job type version and revision
definition_2.add_recipe_node('D', 'recipe_type_1', 1)
definition_2.add_recipe_node('F', 'recipe_type_2', 5) # New node
definition_2.add_dependency('A', 'C')
definition_2.add_dependency('C', 'D')
definition_2.add_dependency('D', 'F')
definition_2.add_recipe_input_connection('A', 'input_1',
'file_param_1')
definition_2.add_dependency_input_connection('C', 'c_input_1', 'A',
'a_output_2')
definition_2.add_dependency_input_connection('D', 'd_input_1', 'C',
'c_output_1')
definition_2.add_recipe_input_connection('D', 'd_input_2',
'json_param_1')
definition_2.add_recipe_input_connection('F', 'f_input_1',
'json_param_2')
diff = RecipeDiff(definition_1, definition_2)
# Non-breaking recipe input changes so recipe can be reprocessed
self.assertTrue(diff.can_be_reprocessed)
self.assertListEqual(diff.reasons, [])
# Check each node for correct fields
node_a = diff.graph['A']
self.assertEqual(node_a.status, NodeDiff.UNCHANGED)
self.assertFalse(node_a.reprocess_new_node)
self.assertListEqual(node_a.changes, [])
node_b = diff.graph['B']
self.assertEqual(node_b.status, NodeDiff.DELETED)
self.assertFalse(node_b.reprocess_new_node)
self.assertListEqual(node_b.changes, [])
node_c = diff.graph['C']
self.assertEqual(node_c.status, NodeDiff.CHANGED)
self.assertTrue(node_c.reprocess_new_node)
self.assertEqual(len(node_c.changes), 2)
self.assertEqual(node_c.changes[0].name, 'JOB_TYPE_VERSION_CHANGE')
self.assertEqual(node_c.changes[1].name, 'JOB_TYPE_REVISION_CHANGE')
node_d = diff.graph['D']
self.assertEqual(node_d.status, NodeDiff.CHANGED)
self.assertTrue(node_d.reprocess_new_node)
self.assertEqual(len(node_d.changes), 1)
self.assertEqual(node_d.changes[0].name, 'PARENT_CHANGED')
node_e = diff.graph['E']
self.assertEqual(node_e.status, NodeDiff.DELETED)
self.assertFalse(node_e.reprocess_new_node)
self.assertListEqual(node_e.changes, [])
node_f = diff.graph['F']
self.assertEqual(node_f.status, NodeDiff.NEW)
self.assertTrue(node_f.reprocess_new_node)
self.assertListEqual(node_f.changes, [])
# Check nodes to copy, supersede, and unpublish
self.assertSetEqual(set(diff.get_nodes_to_copy().keys()), {'A'})
self.assertSetEqual(set(diff.get_nodes_to_supersede().keys()),
{'B', 'C', 'D', 'E'})
self.assertSetEqual(set(diff.get_nodes_to_unpublish().keys()),
{'B', 'E'})
