def test_json(self):
"""Tests converting a ProcessCondition message to and from JSON"""
definition = RecipeDefinition(Interface())
# TODO: once DataFilter is implemented, create a DataFilter object here that accepts the inputs
definition.add_condition_node('node_a', Interface(), DataFilter(True))
definition_dict = convert_recipe_definition_to_v6_json(
definition).get_dict()
recipe_type = recipe_test_utils.create_recipe_type(
definition=definition_dict)
recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type)
condition = recipe_test_utils.create_recipe_condition(recipe=recipe,
save=True)
recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='node_a',
condition=condition,
save=True)
# Create message
message = create_process_condition_messages([condition.id])[0]
# Convert message to JSON and back, and then execute
message_json_dict = message.to_json()
new_message = ProcessCondition.from_json(message_json_dict)
result = new_message.execute()
self.assertTrue(result)
condition = RecipeCondition.objects.get(id=condition.id)
self.assertEqual(len(new_message.new_messages), 1)
self.assertEqual(new_message.new_messages[0].type, 'update_recipe')
self.assertEqual(new_message.new_messages[0].root_recipe_id, recipe.id)
self.assertTrue(condition.is_processed)
self.assertIsNotNone(condition.processed)
self.assertTrue(condition.is_accepted)
def test_validate(self):
"""Tests calling Data.validate()"""
interface = Interface()
data = Data()
interface.add_parameter(FileParameter('input_1', ['application/json']))
interface.add_parameter(JsonParameter('input_2', 'integer'))
data.add_value(FileValue('input_1', [123]))
data.add_value(JsonValue('input_2', 100))
data.add_value(JsonValue('extra_input_1', 'hello'))
data.add_value(JsonValue('extra_input_2', 'there'))
# Valid data
data.validate(interface)
# Ensure extra data values are removed
self.assertSetEqual(set(data.values.keys()), {'input_1', 'input_2'})
# Data is missing required input 3
interface.add_parameter(FileParameter('input_3', ['image/gif'], required=True))
with self.assertRaises(InvalidData) as context:
data.validate(interface)
self.assertEqual(context.exception.error.name, 'PARAM_REQUIRED')
data.add_value(FileValue('input_3', [999])) # Input 3 taken care of now
# Invalid data
interface.add_parameter(JsonParameter('input_4', 'string'))
mock_value = MagicMock()
mock_value.name = 'input_4'
mock_value.validate.side_effect = InvalidData('MOCK', '')
data.add_value(mock_value)
with self.assertRaises(InvalidData) as context:
data.validate(interface)
self.assertEqual(context.exception.error.name, 'MOCK')
def test_get_nodes_to_create(self):
"""Tests calling Recipe.get_nodes_to_create()"""
job_type = job_test_utils.create_job_type()
sub_recipe_type = recipe_test_utils.create_recipe_type()
# Create recipe
definition = RecipeDefinition(Interface())
definition.add_job_node('A', job_type.name, job_type.version,
job_type.revision_num)
definition.add_condition_node('B', Interface(), DataFilter(True))
definition.add_condition_node('C', Interface(), DataFilter(True))
definition.add_condition_node('D', Interface(), DataFilter(False))
definition.add_job_node('E', job_type.name, job_type.version,
job_type.revision_num)
definition.add_job_node('F', job_type.name, job_type.version,
job_type.revision_num)
definition.add_recipe_node('G', sub_recipe_type.name,
sub_recipe_type.revision_num)
definition.add_recipe_node('H', sub_recipe_type.name,
sub_recipe_type.revision_num)
definition.add_dependency('A', 'D')
definition.add_dependency('A', 'E')
definition.add_dependency('B', 'E')
definition.add_dependency('B', 'F')
definition.add_dependency('C', 'F')
definition.add_dependency('D', 'G')
definition.add_dependency('E', 'G')
definition.add_dependency('E', 'H')
definition_json_dict = convert_recipe_definition_to_v6_json(
definition).get_dict()
recipe_type = recipe_test_utils.create_recipe_type(
definition=definition_json_dict)
recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type)
# Nodes A, B, and D already exist
job_a = job_test_utils.create_job(job_type=job_type,
status='COMPLETED',
save=True)
condition_b = recipe_test_utils.create_recipe_condition(
is_processed=True, is_accepted=True, save=False)
condition_d = recipe_test_utils.create_recipe_condition(
is_processed=True, is_accepted=False, save=False)
RecipeCondition.objects.bulk_create([condition_b, condition_d])
recipe_node_a = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='A',
job=job_a,
save=False)
recipe_node_b = recipe_test_utils.create_recipe_node(
recipe=recipe, node_name='B', condition=condition_b, save=False)
recipe_node_d = recipe_test_utils.create_recipe_node(
recipe=recipe, node_name='D', condition=condition_d, save=False)
RecipeNode.objects.bulk_create(
[recipe_node_a, recipe_node_b, recipe_node_d])
recipe_instance = Recipe.objects.get_recipe_instance(recipe.id)
nodes_to_create = recipe_instance.get_nodes_to_create()
self.assertSetEqual(set(nodes_to_create.keys()), {'C', 'E', 'H'})
def test_recipe_input_conn_successful(self):
"""Tests calling RecipeDefinition.add_recipe_input_connection() successfully"""
input_interface = Interface()
input_interface.parameters = {'recipe_input_1': MagicMock()}
definition = RecipeDefinition(input_interface)
definition.add_job_node('node_1', 'job_type_1', '1.0', 1)
definition.add_recipe_input_connection('node_1', 'input_1',
'recipe_input_1')
def test_convert_recipe_definition_to_v6_json_full(self):
"""Tests calling convert_recipe_definition_to_v6_json() with a full definition"""
interface = Interface()
interface.add_parameter(FileParameter('file_param_a', ['image/gif']))
interface.add_parameter(JsonParameter('json_param_a', 'object'))
interface.add_parameter(JsonParameter('json_param_b', 'object', required=False))
definition = RecipeDefinition(interface)
definition.add_job_node('A', 'job_type_1', '1.0', 1)
definition.add_job_node('B', 'job_type_2', '2.0', 1)
definition.add_job_node('C', 'job_type_3', '1.0', 2)
definition.add_recipe_node('D', 'recipe_type_1', 1)
definition.add_condition_node('E', Interface(), DataFilter())
definition.add_job_node('F', 'job_type_4', '1.0', 1)
definition.add_dependency('A', 'B')
definition.add_dependency('A', 'C')
definition.add_dependency('B', 'E')
definition.add_dependency('C', 'D')
definition.add_dependency('E', 'F')
definition.add_recipe_input_connection('A', 'input_a', 'file_param_a')
definition.add_dependency_input_connection('B', 'b_input_a', 'A', 'a_output_1')
definition.add_dependency_input_connection('C', 'c_input_a', 'A', 'a_output_2')
definition.add_dependency_input_connection('D', 'd_input_a', 'C', 'c_output_1')
definition.add_recipe_input_connection('D', 'd_input_b', 'json_param_a')
json = convert_recipe_definition_to_v6_json(definition)
RecipeDefinitionV6(definition=json.get_dict(), do_validate=True) # Revalidate
self.assertSetEqual(set(json.get_dict()['nodes'].keys()), {'A', 'B', 'C', 'D', 'E', 'F'})
def test_convert_recipe_diff_to_v6_json_empty(self):
"""Tests calling convert_recipe_diff_to_v6_json() with an empty diff"""
# Try diff with empty recipe definitions
interface_a = Interface()
interface_b = Interface()
definition_a = RecipeDefinition(interface_a)
definition_b = RecipeDefinition(interface_b)
diff = RecipeDiff(definition_a, definition_b)
json = convert_recipe_diff_to_v6_json(diff)
RecipeDiffV6(diff=json.get_dict(), do_validate=True) # Revalidate
self.assertTrue(json.get_dict()['can_be_reprocessed'])
def test_recipe_input_conn_missing_input_node(self):
"""Tests calling RecipeDefinition.add_recipe_input_connection() with an unknown input node"""
input_interface = Interface()
input_interface.parameters = {'recipe_input_1': MagicMock()}
definition = RecipeDefinition(input_interface)
definition.add_job_node('node_1', 'job_type_1', '1.0', 1)
with self.assertRaises(InvalidDefinition) as context:
definition.add_recipe_input_connection('missing_node', 'input_1',
'recipe_input_1')
self.assertEqual(context.exception.error.name, 'UNKNOWN_NODE')
def _get_recipe_interfaces(self, node):
"""Gets the input/output interfaces for a recipe type node
"""
from recipe.models import RecipeTypeRevision
input = Interface()
output = Interface()
rtr = RecipeTypeRevision.objects.get_revision(node.recipe_type_name,
node.revision_num)
if rtr:
input = rtr.get_input_interface() # no output interface
return input, output
def test_json(self):
"""Tests converting a ProcessCondition message to and from JSON"""
definition = RecipeDefinition(Interface())
cond_interface_1 = Interface()
cond_interface_1.add_parameter(JsonParameter('cond_int', 'integer'))
df1 = DataFilter(filter_list=[{
'name': 'cond_int',
'type': 'integer',
'condition': '==',
'values': [0]
}])
definition = RecipeDefinition(cond_interface_1)
definition.add_condition_node('node_a', cond_interface_1, df1)
definition.add_recipe_input_connection('node_a', 'cond_int',
'cond_int')
definition_dict = convert_recipe_definition_to_v6_json(
definition).get_dict()
recipe_type = recipe_test_utils.create_recipe_type_v6(
definition=definition_dict)
data_1 = Data()
data_1.add_value(JsonValue('cond_int', 0))
data_1_dict = convert_data_to_v6_json(data_1).get_dict()
recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type,
input=data_1_dict)
condition = recipe_test_utils.create_recipe_condition(recipe=recipe,
save=True)
recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='node_a',
condition=condition,
save=True)
# Create message
message = create_process_condition_messages([condition.id])[0]
# Convert message to JSON and back, and then execute
message_json_dict = message.to_json()
new_message = ProcessCondition.from_json(message_json_dict)
result = new_message.execute()
self.assertTrue(result)
condition = RecipeCondition.objects.get(id=condition.id)
self.assertEqual(len(new_message.new_messages), 1)
self.assertEqual(new_message.new_messages[0].type, 'update_recipe')
self.assertEqual(new_message.new_messages[0].root_recipe_id, recipe.id)
self.assertTrue(condition.is_processed)
self.assertIsNotNone(condition.processed)
self.assertTrue(condition.is_accepted)
def test_convert_recipe_definition_to_v1_json_full(self):
"""Tests calling convert_recipe_definition_to_v1_json() with a full definition"""
interface = Interface()
interface.add_parameter(FileParameter('file_param_a', ['image/gif']))
interface.add_parameter(JsonParameter('json_param_a', 'object'))
interface.add_parameter(
JsonParameter('json_param_b', 'object', required=False))
definition = RecipeDefinition(interface)
definition.add_job_node('A', 'job_type_1', '1.0', 1)
definition.add_job_node('B', 'job_type_2', '2.0', 1)
definition.add_job_node('C', 'job_type_3', '1.0', 2)
definition.add_recipe_node('D', 'recipe_type_1', 1)
definition.add_job_node('E', 'job_type_4', '1.0', 1)
definition.add_dependency('A', 'B')
definition.add_dependency('A', 'C')
definition.add_dependency('B', 'E')
definition.add_dependency('C', 'D')
definition.add_recipe_input_connection('A', 'input_1', 'file_param_a')
definition.add_dependency_input_connection('B', 'b_input_1', 'A',
'a_output_1')
definition.add_dependency_input_connection('C', 'c_input_1', 'A',
'a_output_2')
definition.add_dependency_input_connection('D', 'd_input_1', 'C',
'c_output_1')
definition.add_recipe_input_connection('D', 'd_input_2',
'json_param_a')
json = convert_recipe_definition_to_v1_json(definition)
RecipeDefinitionV1(definition=json.get_dict(),
do_validate=True) # Revalidate
job_names = {job_dict['name'] for job_dict in json.get_dict()['jobs']}
self.assertSetEqual(job_names,
{'A', 'B', 'C', 'E'}) # D is omitted (recipe node)
def _get_job_interfaces(self, node):
"""Gets the input/output interfaces for a job type node
"""
from job.models import JobTypeRevision
input = Interface()
output = Interface()
jtr = JobTypeRevision.objects.get_details_v6(node.job_type_name,
node.job_type_version,
node.revision_num)
if jtr:
input = jtr.get_input_interface()
output = jtr.get_output_interface()
return input, output
def test_recipe_input_conn_duplicate_input(self):
"""Tests calling RecipeDefinition.add_recipe_input_connection() to connect to a duplicate input"""
input_interface = Interface()
input_interface.parameters = {'recipe_input_1': MagicMock()}
definition = RecipeDefinition(input_interface)
definition.add_job_node('node_1', 'job_type_1', '1.0', 1)
definition.add_job_node('node_2', 'job_type_2', '1.0', 1)
definition.add_recipe_input_connection('node_1', 'input_1',
'recipe_input_1')
with self.assertRaises(InvalidDefinition) as context:
definition.add_recipe_input_connection('node_1', 'input_1',
'recipe_input_1')
self.assertEqual(context.exception.error.name, 'NODE_INTERFACE')
def test_topological_order_successful(self):
"""Tests calling RecipeDefinition.get_topological_order() successfully"""
input_interface = Interface()
definition = RecipeDefinition(input_interface)
definition.add_job_node('A', 'job_type_1', '1.0', 1)
definition.add_job_node('B', 'job_type_2', '1.0', 1)
definition.add_recipe_node('C', 'recipe_type_1', 1)
definition.add_recipe_node('D', 'recipe_type_2', 1)
definition.add_job_node('E', 'job_type_3', '1.0', 1)
definition.add_job_node('F', 'job_type_4', '1.0', 1)
definition.add_dependency('A', 'B')
definition.add_dependency('A', 'C')
definition.add_dependency('A', 'E')
definition.add_dependency('B', 'C')
definition.add_dependency('B', 'D')
definition.add_dependency('C', 'D')
definition.add_dependency('D', 'E')
definition.add_dependency('E', 'F')
order = definition.get_topological_order()
expected_order = [
'A', 'B', 'C', 'D', 'E', 'F'
] # This is the only valid topological order for this graph
self.assertListEqual(order, expected_order)
def test_json(self):
"""Tests converting an UpdateRecipe message to and from JSON"""
data_dict = convert_data_to_v6_json(Data()).get_dict()
job_failed = job_test_utils.create_job(status='FAILED', input=data_dict)
job_pending = job_test_utils.create_job(status='PENDING')
definition = RecipeDefinition(Interface())
definition.add_job_node('job_failed', job_failed.job_type.name, job_failed.job_type.version,
job_failed.job_type_rev.revision_num)
definition.add_job_node('job_pending', job_pending.job_type.name, job_pending.job_type.version,
job_pending.job_type_rev.revision_num)
definition.add_dependency('job_failed', 'job_pending')
definition_dict = convert_recipe_definition_to_v6_json(definition).get_dict()
recipe_type = recipe_test_utils.create_recipe_type_v6(definition=definition_dict)
recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type)
recipe_test_utils.create_recipe_job(recipe=recipe, job_name='job_failed', job=job_failed)
recipe_test_utils.create_recipe_job(recipe=recipe, job_name='job_pending', job=job_pending)
# Create message
message = create_update_recipe_message(recipe.id)
# Convert message to JSON and back, and then execute
message_json_dict = message.to_json()
new_message = UpdateRecipe.from_json(message_json_dict)
result = new_message.execute()
self.assertTrue(result)
# Check for message to set job_pending to BLOCKED
self.assertEqual(len(new_message.new_messages), 1)
msg = new_message.new_messages[0]
self.assertEqual(msg.type, 'blocked_jobs')
self.assertListEqual(msg._blocked_job_ids, [job_pending.id])
def test_convert_recipe_definition_to_v1_json_empty(self):
"""Tests calling convert_recipe_definition_to_v1_json() with an empty definition"""
interface = Interface()
definition = RecipeDefinition(interface)
json = convert_recipe_definition_to_v1_json(definition)
RecipeDefinitionV1(definition=json.get_dict(),
do_validate=True) # Revalidate
def test_convert_recipe_definition_to_v6_json_empty(self):
"""Tests calling convert_recipe_definition_to_v6_json() with an empty definition"""
interface = Interface()
definition = RecipeDefinition(interface)
json = convert_recipe_definition_to_v6_json(definition)
RecipeDefinitionV6(definition=json.get_dict(), do_validate=True) # Revalidate
self.assertDictEqual(json.get_dict()['input'], {'files': [], 'json': []})
def test_convert_recipe_to_v6_json_empty(self):
"""Tests calling convert_recipe_to_v6_json() with an empty recipe instance"""
definition = RecipeDefinition(Interface())
recipe_instance = RecipeInstance(definition, [])
json = convert_recipe_to_v6_json(recipe_instance)
RecipeInstanceV6(json=json.get_dict(), do_validate=True) # Revalidate
self.assertDictEqual(json.get_dict()['nodes'], {})
def validate(self, recipe_input_interface, node_input_interfaces,
node_output_interfaces):
"""Validates this node
:param recipe_input_interface: The interface for the recipe input
:type recipe_input_interface: :class:`data.interface.interface.Interface`
:param node_input_interfaces: The input interface for each node stored by node name
:type node_input_interfaces: dict
:param node_output_interfaces: The output interface for each node stored by node name
:type node_output_interfaces: dict
:returns: A list of warnings discovered during validation
:rtype: :func:`list`
:raises :class:`recipe.definition.exceptions.InvalidDefinition`: If the definition is invalid
"""
warnings = []
input_interface = node_input_interfaces[self.name]
connecting_interface = Interface()
# Generate complete dependency set for this node
all_dependencies = set()
dependency_list = list(self.parents.values())
while dependency_list:
dependency = dependency_list.pop()
if dependency.name not in all_dependencies:
all_dependencies.add(dependency.name)
dependency_list.extend(list(dependency.parents.values()))
try:
for connection in self.connections.values():
# Validate each connection
warnings.extend(connection.validate(all_dependencies))
# Combine all connections into a connecting interface
warnings.extend(
connection.add_parameter_to_interface(
connecting_interface, recipe_input_interface,
node_output_interfaces))
# Validate that connecting interface can be passed to this interface
warnings.extend(
input_interface.validate_connection(connecting_interface))
except InvalidInterfaceConnection as ex:
error_desc = ex.error.description
# Parse error message to try and give the user a better message.
if error_desc.startswith("Parameter") and error_desc.endswith(
"is required"):
msg = 'input \'%s\' is missing for %s' % (
error_desc.split("\'")[1], self.name)
else:
msg = 'Node \'%s\' interface error: %s' % (self.name,
error_desc)
raise InvalidDefinition('NODE_INTERFACE', msg)
return warnings
def test_add_dependency_missing_child(self):
"""Tests calling RecipeDefinition.add_dependency() with a missing child node"""
input_interface = Interface()
definition = RecipeDefinition(input_interface)
definition.add_job_node('node_1', 'job_type_1', '1.0', 1)
with self.assertRaises(InvalidDefinition) as context:
definition.add_dependency('node_1', 'missing_child')
self.assertEqual(context.exception.error.name, 'UNKNOWN_NODE')
def test_dependency_input_conn_successful(self):
"""Tests calling RecipeDefinition.add_dependency_input_connection() successfully"""
input_interface = Interface()
definition = RecipeDefinition(input_interface)
definition.add_recipe_node('node_1', 'recipe_type_1', 1)
definition.add_job_node('node_2', 'job_type_2', '1.0', 1)
definition.add_dependency_input_connection('node_1', 'input_1',
'node_2', 'output_1')
def test_recipe_input_conn_missing_input(self):
"""Tests calling RecipeDefinition.add_recipe_input_connection() with an unknown recipe input"""
input_interface = Interface()
definition = RecipeDefinition(input_interface)
definition.add_job_node('node_1', 'job_type_1', '1.0', 1)
with self.assertRaises(InvalidDefinition) as context:
definition.add_recipe_input_connection('node_1', 'input_1',
'missing_recipe_input')
self.assertEqual(context.exception.error.name, 'UNKNOWN_INPUT')
def test_dependency_input_conn_missing_dependency(self):
"""Tests calling RecipeDefinition.add_dependency_input_connection() with an unknown dependency node"""
input_interface = Interface()
definition = RecipeDefinition(input_interface)
definition.add_job_node('node_1', 'job_type_1', '1.0', 1)
with self.assertRaises(InvalidDefinition) as context:
definition.add_dependency_input_connection('node_1', 'input_1',
'missing_dependency',
'output_1')
self.assertEqual(context.exception.error.name, 'UNKNOWN_NODE')
def test_validate_missing_dependency(self):
"""Tests calling RecipeDefinition.validate() with a connection that has a missing dependency"""
input_interface = Interface()
input_interface.parameters = {'recipe_input_1': MagicMock()}
definition = RecipeDefinition(input_interface)
definition.add_job_node('A', 'job_type_1', '1.0', 1)
definition.add_recipe_node('B', 'recipe_type_1', 1)
definition.add_job_node('C', 'job_type_2', '1.0', 1)
definition.add_dependency('B', 'C')
definition.add_dependency_input_connection('B', 'input_1', 'A',
'output_1')
mocked_interfaces = {
'A': MagicMock(),
'B': MagicMock(),
'C': MagicMock()
}
with self.assertRaises(InvalidDefinition) as context:
definition.validate(mocked_interfaces, mocked_interfaces)
self.assertEqual(context.exception.error.name, 'NODE_INTERFACE')
def test_dependency_input_conn_cannot_connect_to_recipe(self):
"""Tests calling RecipeDefinition.add_dependency_input_connection() to connect to a recipe node (invalid)"""
input_interface = Interface()
definition = RecipeDefinition(input_interface)
definition.add_job_node('node_1', 'job_type_1', '1.0', 1)
definition.add_recipe_node('node_2', 'recipe_type_1', 1)
with self.assertRaises(InvalidDefinition) as context:
definition.add_dependency_input_connection('node_1', 'input_1',
'node_2', 'output_1')
self.assertEqual(context.exception.error.name,
'CONNECTION_INVALID_NODE')
def test_convert_recipe_to_v6_json(self):
"""Tests calling convert_recipe_to_v6_json() successfully"""
job_type_1 = job_test_utils.create_seed_job_type()
job_type_2 = job_test_utils.create_seed_job_type()
job_type_3 = job_test_utils.create_seed_job_type()
job_type_4 = job_test_utils.create_seed_job_type()
recipe_type_1 = recipe_test_utils.create_recipe_type_v6()
interface = Interface()
interface.add_parameter(FileParameter('file_param_1', ['image/gif']))
interface.add_parameter(JsonParameter('json_param_1', 'object'))
df1 = DataFilter(filter_list=[{'name': 'file_param_1', 'type': 'media-type', 'condition': '==', 'values': ['image/gif']},
{'name': 'json_param_1', 'type': 'object', 'condition': 'superset of', 'values': [{}]}],
all=False)
definition = RecipeDefinition(interface)
definition.add_job_node('A', job_type_1.name, job_type_1.version, job_type_1.revision_num)
definition.add_job_node('B', job_type_2.name, job_type_2.version, job_type_2.revision_num)
definition.add_job_node('C', job_type_3.name, job_type_3.version, job_type_3.revision_num)
definition.add_recipe_node('D', recipe_type_1.name, recipe_type_1.revision_num)
definition.add_job_node('E', job_type_4.name, job_type_4.version, job_type_4.revision_num)
definition.add_condition_node('F', interface, df1) #False
definition.add_job_node('G', job_type_4.name, job_type_4.version, job_type_4.revision_num)
definition.add_dependency('A', 'B')
definition.add_dependency('A', 'C')
definition.add_dependency('B', 'E')
definition.add_dependency('C', 'D')
definition.add_dependency('A', 'F')
definition.add_dependency('F', 'G')
definition.add_recipe_input_connection('A', 'input_1', 'file_param_1')
definition.add_dependency_input_connection('B', 'b_input_1', 'A', 'a_output_1')
definition.add_dependency_input_connection('C', 'c_input_1', 'A', 'a_output_2')
definition.add_dependency_input_connection('D', 'd_input_1', 'C', 'c_output_1')
definition.add_recipe_input_connection('D', 'd_input_2', 'json_param_1')
recipe = recipe_test_utils.create_recipe()
job_a = job_test_utils.create_job(job_type=job_type_1, status='COMPLETED', save=False)
job_b = job_test_utils.create_job(job_type=job_type_2, status='RUNNING', save=False)
job_c = job_test_utils.create_job(job_type=job_type_3, status='COMPLETED', save=False)
job_e = job_test_utils.create_job(job_type=job_type_4, status='PENDING', num_exes=0, save=False)
Job.objects.bulk_create([job_a, job_b, job_c, job_e])
condition_f = recipe_test_utils.create_recipe_condition(is_processed=True, is_accepted=False, save=True)
recipe_d = recipe_test_utils.create_recipe(recipe_type=recipe_type_1)
recipe_node_a = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='A', job=job_a, save=False)
recipe_node_b = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='B', job=job_b, save=False)
recipe_node_c = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='C', job=job_c, save=False)
recipe_node_d = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='D', sub_recipe=recipe_d,
save=False)
recipe_node_e = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='E', job=job_e, save=False)
recipe_node_f = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='F', condition=condition_f,
save=False)
recipe_nodes = [recipe_node_a, recipe_node_b, recipe_node_c, recipe_node_d, recipe_node_e, recipe_node_f]
recipe_instance = RecipeInstance(definition, recipe, recipe_nodes)
json = convert_recipe_to_v6_json(recipe_instance)
RecipeInstanceV6(json=json.get_dict(), do_validate=True) # Revalidate
self.assertSetEqual(set(json.get_dict()['nodes'].keys()), {'A', 'B', 'C', 'D', 'E', 'F'})
def test_validate_successful(self):
"""Tests calling RecipeDefinition.validate() successfully"""
input_interface = Interface()
definition = RecipeDefinition(input_interface)
definition.add_job_node('A', 'job_type_1', '1.0', 1)
definition.add_recipe_node('B', 'recipe_type_1', 1)
definition.add_dependency('A', 'B')
definition.add_dependency_input_connection('B', 'input_1', 'A',
'output_1')
mocked_interfaces = {'A': MagicMock(), 'B': MagicMock()}
warnings = definition.validate(mocked_interfaces, mocked_interfaces)
self.assertListEqual(warnings, [])
def test_convert_data_to_v1_json(self):
"""Tests calling convert_data_to_v1_json()"""
# Try interface with nothing set
data = Data()
interface = Interface()
json = convert_data_to_v1_json(data, interface)
DataV1(data=json.get_dict()) # Revalidate
# Try data with a variety of values
data = Data()
data.add_value(FileValue('input_a', [1234]))
data.add_value(FileValue('input_b', [1235, 1236]))
data.add_value(JsonValue('input_c', 'hello'))
data.add_value(JsonValue('input_d', 11.9))
json = convert_data_to_v1_json(data, interface)
self.assertDictEqual(
json.get_dict(), {
u'input_data': [{
u'name': u'input_d',
u'value': 11.9
}, {
u'name': u'input_b',
u'file_ids': [1235, 1236]
}, {
u'name': u'input_c',
u'value': u'hello'
}, {
u'name': u'input_a',
u'file_id': 1234
}],
u'version':
u'1.0'
})
DataV1(data=json.get_dict()) # Revalidate
self.assertSetEqual(
set(DataV6(json.get_dict()).get_data().values.keys()),
{'input_a', 'input_b', 'input_c', 'input_d'})
# Try data with a single file list that should be a directory
data = Data()
data.add_value(FileValue('input_a', [1234]))
interface = Interface()
file_param = FileParameter('input_a', [], True, True)
interface.add_parameter(file_param)
json = convert_data_to_v1_json(data, interface)
self.assertDictEqual(
json.get_dict(), {
u'input_data': [{
u'name': u'input_a',
u'file_ids': [1234]
}],
u'version': u'1.0'
})
def test_add_parameter(self):
"""Tests calling Interface.add_parameter()"""
interface = Interface()
file_param = FileParameter('input_1', ['application/json'])
interface.add_parameter(file_param)
json_param = JsonParameter('input_2', 'integer')
interface.add_parameter(json_param)
self.assertSetEqual(set(interface.parameters.keys()), {'input_1', 'input_2'})
# Duplicate parameter
dup_param = FileParameter('input_1', [], required=False)
with self.assertRaises(InvalidInterface) as context:
interface.add_parameter(dup_param)
self.assertEqual(context.exception.error.name, 'DUPLICATE_INPUT')
def test_topological_order_circular(self):
"""Tests calling RecipeDefinition.get_topological_order() with a circular dependency"""
input_interface = Interface()
definition = RecipeDefinition(input_interface)
definition.add_job_node('A', 'job_type_1', '1.0', 1)
definition.add_job_node('B', 'job_type_2', '1.0', 1)
definition.add_recipe_node('C', 'recipe_type_1', 1)
definition.add_recipe_node('D', 'recipe_type_2', 1)
definition.add_dependency('A', 'B')
definition.add_dependency('B', 'C')
definition.add_dependency('C', 'D')
definition.add_dependency('D', 'B')
with self.assertRaises(InvalidDefinition) as context:
definition.get_topological_order()
self.assertEqual(context.exception.error.name, 'CIRCULAR_DEPENDENCY')
def merge_parameter_map(self, batch, dataset):
"""Returns the dataset parameters merged with the batch configuration input map
:param batch: The batch
:type batch: :class:`batch.models.Batch`
:param dataset: The dataset of the batch
:type dataset: :class:`data.models.DataSet`
:returns: The map of datasest parameters
:rtype: :class:`data.interface.Interface`
"""
# combine the parameters
dataset_definition = dataset.get_definition()
dataset_parameters = dataset_definition.global_parameters
for param in dataset_definition.parameters.parameters:
dataset_parameters.add_parameter(
dataset_definition.parameters.parameters[param])
# map dataset param to inputs if applicable
if batch.get_configuration().input_map:
from data.interface.interface import Interface
from data.interface.parameter import FileParameter, JsonParameter
parameters = Interface()
for param_name in dataset_parameters.parameters:
param = dataset_parameters.parameters[param_name]
for map_param in batch.get_configuration().input_map:
if param_name == map_param['datasetParameter']:
if param.PARAM_TYPE == 'file':
parameters.add_parameter(
FileParameter(map_param['input'],
param.media_types,
required=param.required,
multiple=param.multiple))
elif param.PARAM_TYPE == 'json':
parameters.add_parameter(
JsonParameter(map_param['input'],
param.json_type,
required=param.required,
multiple=param.multiple))
else:
parameters.add_parameter(param)
dataset_parameters = parameters
return dataset_parameters
