def test_get_job_type_keys_unique(self):
"""Tests getting job type keys without duplicates."""
definition = {
'version':
'1.0',
'input_data': [],
'jobs': [{
'name': 'Job 1a',
'job_type': {
'name': self.job_type1.name,
'version': self.job_type1.version,
},
}, {
'name': 'Job 2',
'job_type': {
'name': self.job_type2.name,
'version': self.job_type2.version,
},
}, {
'name': 'Job 1b',
'job_type': {
'name': self.job_type1.name,
'version': self.job_type1.version,
},
}],
}
recipe_definition = RecipeDefinition(definition)
results = recipe_definition.get_job_type_keys()
self.assertSetEqual(
results, {(self.job_type1.name, self.job_type1.version),
(self.job_type2.name, self.job_type2.version)})
def test_successful(self):
'''Tests calling RecipeDefinition.get_unqueued_job_statuses() successfully.'''
recipe_definition = RecipeDefinition(self.definition)
recipe_definition.validate_job_interfaces()
recipe_jobs = {
'job_failed': self.job_failed,
'job_completed': self.job_completed,
'job_running': self.job_running,
'job_queued': self.job_queued,
'job_canceled': self.job_canceled,
'job_fa_co_a': self.job_fa_co_a,
'job_fa_co_b': self.job_fa_co_b,
'job_co_ru_qu_a': self.job_co_ru_qu_a,
'job_co_ru_qu_b': self.job_co_ru_qu_b,
'job_qu_ca_a': self.job_qu_ca_a,
'job_qu_ca_b': self.job_qu_ca_b,
}
results = recipe_definition.get_unqueued_job_statuses(recipe_jobs)
expected_results = {
self.job_fa_co_a.id: 'BLOCKED',
self.job_fa_co_b.id: 'BLOCKED',
self.job_co_ru_qu_a.id: 'PENDING',
self.job_co_ru_qu_b.id: 'PENDING',
self.job_qu_ca_a.id: 'BLOCKED',
self.job_qu_ca_b.id: 'BLOCKED',
}
self.assertDictEqual(results, expected_results)
def test_init_job_name(self):
"""Tests calling RecipeDefinition constructor with good and bad job names."""
definition = {
'version':
'1.0',
'input_data': [{
'name': 'Recipe Input',
'type': 'property',
}],
'jobs': [{
'name': 'Job 1',
'job_type': {
'name': self.job_type1.name,
'version': self.job_type1.version,
},
}, {
'name':
'Job 2',
'job_type': {
'name': self.job_type2.name,
'version': self.job_type2.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input',
'job_input': 'Input 1',
}],
'dependencies': [{
'name':
'Job 1',
'connections': [{
'output': 'Output 1',
'input': 'Input 2',
}],
}],
}],
}
good_names = ['foo', 'bar', 'name with spaces', 'name_with_undersores']
bad_names = [
'Speci@lCharacter',
'dont_use_bang!',
'dont.use.periods',
'names_should_be_less_than_256_characters123456789012345678901234567890123456789012345678901234567890123456'
'7890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012'
'345678901234567890123456789012345678901234567890',
]
for good_name in good_names:
definition['jobs'][0]['name'] = good_name
definition['jobs'][1]['dependencies'][0]['name'] = good_name
# No exception is success
RecipeDefinition(definition)
for bad_name in bad_names:
definition['jobs'][1]['dependencies'][0]['name'] = bad_name
# No exception is success
self.assertRaises(InvalidDefinition, RecipeDefinition, definition)
def test_invalid_job_type(self):
"""Tests calling RecipeDefinition.validate_job_interfaces() with an invalid job type."""
definition = {
'version':
'1.0',
'input_data': [{
'name': 'Recipe Input',
'type': 'property',
}],
'jobs': [{
'name': 'Job 1',
'job_type': {
'name': 'BAD',
'version': 'BAD',
},
}, {
'name': 'Job 2',
'job_type': {
'name': 'invalid-job-type-name',
'version': 'invalid-job-type-version',
},
}],
}
recipe = RecipeDefinition(definition)
self.assertRaises(InvalidDefinition, recipe.validate_job_interfaces)
def test_get_job_types_multi(self):
"""Tests getting job types from the definition."""
definition = {
'version':
'1.0',
'input_data': [],
'jobs': [{
'name': 'Job 1',
'job_type': {
'name': self.job_type1.name,
'version': self.job_type1.version,
},
}, {
'name': 'Job 2',
'job_type': {
'name': self.job_type2.name,
'version': self.job_type2.version,
},
}],
}
recipe_definition = RecipeDefinition(definition)
results = recipe_definition.get_job_types()
self.assertSetEqual(results, {self.job_type1, self.job_type2})
def get_recipe_definition(self):
"""Returns the recipe type definition for this revision
:returns: The recipe type definition for this revision
:rtype: :class:`recipe.configuration.definition.recipe_definition.RecipeDefinition`
"""
return RecipeDefinition(self.definition)
def get_recipe_definition(self):
"""Returns the definition for running recipes of this type
:returns: The recipe definition for this type
:rtype: :class:`recipe.configuration.definition.recipe_definition.RecipeDefinition`
"""
return RecipeDefinition(self.definition)
def post(self, request):
"""Creates a new recipe type and returns a link to the detail URL
:param request: the HTTP POST request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
"""
name = rest_util.parse_string(request, 'name')
version = rest_util.parse_string(request, 'version')
title = rest_util.parse_string(request, 'title', default_value=name)
description = rest_util.parse_string(request, 'description', required=False)
definition_dict = rest_util.parse_dict(request, 'definition')
# Check for optional trigger rule parameters
trigger_rule_dict = rest_util.parse_dict(request, 'trigger_rule', required=False)
if (('type' in trigger_rule_dict and 'configuration' not in trigger_rule_dict) or
('type' not in trigger_rule_dict and 'configuration' in trigger_rule_dict)):
raise BadParameter('Trigger type and configuration are required together.')
is_active = trigger_rule_dict['is_active'] if 'is_active' in trigger_rule_dict else True
# Attempt to look up the trigger handler for the type
rule_handler = None
if trigger_rule_dict and 'type' in trigger_rule_dict:
try:
rule_handler = trigger_handler.get_trigger_rule_handler(trigger_rule_dict['type'])
except InvalidTriggerType as ex:
logger.exception('Invalid trigger type for new recipe type: %s', name)
raise BadParameter(unicode(ex))
try:
with transaction.atomic():
# Validate the recipe definition
recipe_def = RecipeDefinition(definition_dict)
# Attempt to create the trigger rule
trigger_rule = None
if rule_handler and 'configuration' in trigger_rule_dict:
trigger_rule = rule_handler.create_trigger_rule(trigger_rule_dict['configuration'], name, is_active)
# Create the recipe type
recipe_type = RecipeType.objects.create_recipe_type(name, version, title, description, recipe_def,
trigger_rule)
except (InvalidDefinition, InvalidTriggerType, InvalidTriggerRule, InvalidRecipeConnection) as ex:
logger.exception('Unable to create new recipe type: %s', name)
raise BadParameter(unicode(ex))
# Fetch the full recipe type with details
try:
recipe_type = RecipeType.objects.get_details(recipe_type.id)
except RecipeType.DoesNotExist:
raise Http404
url = urlresolvers.reverse('recipe_type_details_view', args=[recipe_type.id])
serializer = RecipeTypeDetailsSerializer(recipe_type)
return Response(serializer.data, status=status.HTTP_201_CREATED, headers=dict(location=url))
def test_bad_input_name(self):
"""Tests calling IngestTriggerRuleConfiguration.validate_trigger_for_recipe() with a bad input name"""
rule_json_str = '{"version": "1.0", "condition": {"media_type": "text/plain", "data_types": ["A", "B"]}, "data": {"input_data_name": "my_input", "workspace_name": "my_workspace"}}'
rule_config = IngestTriggerRuleConfiguration(INGEST_TYPE, json.loads(rule_json_str))
definition_json_str = '{"version": "1.0", "input_data": [{"name": "different_input_name", "type": "file", "media_types": ["text/plain", "application/json"]}], "jobs": [{"name": "my_job", "job_type": {"name": "test_job", "version": "1.0"}}]}'
recipe_definition = RecipeDefinition(json.loads(definition_json_str))
self.assertRaises(InvalidRecipeConnection, rule_config.validate_trigger_for_recipe, recipe_definition)
def test_successful(self, mock_store):
"""Tests calling RecipeDefinition.validate_data() successfully."""
definition = {
'version':
'1.0',
'input_data': [{
'name': 'Recipe Input',
'type': 'file',
'media_types': ['text/plain'],
}],
'jobs': [{
'name':
'Job 1',
'job_type': {
'name': self.job_type_1.name,
'version': self.job_type_1.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input',
'job_input': self.input_name_1,
}],
}, {
'name':
'Job 2',
'job_type': {
'name': self.job_type_2.name,
'version': self.job_type_2.version,
},
'dependencies': [{
'name':
'Job 1',
'connections': [{
'output': self.output_name_1,
'input': self.input_name_2,
}],
}],
}],
}
recipe = RecipeDefinition(definition)
recipe.validate_job_interfaces()
data = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input',
'file_id': self.file_1.id,
}],
'workspace_id': 1,
}
recipe_data = LegacyRecipeData(data)
# No exception is success
recipe.validate_data(recipe_data)
def test_media_type_warning(self):
"""Tests calling IngestTriggerRuleConfiguration.validate_trigger_for_recipe() with a warning for a mis-matched media type"""
rule_json_str = '{"version": "1.0", "condition": {"media_type": "text/plain", "data_types": ["A", "B"]}, "data": {"input_data_name": "my_input", "workspace_name": "my_workspace"}}'
rule_config = IngestTriggerRuleConfiguration(INGEST_TYPE, json.loads(rule_json_str))
definition_json_str = '{"version": "1.0", "input_data": [{"name": "my_input", "type": "file", "media_types": ["application/json"]}], "jobs": [{"name": "my_job", "job_type": {"name": "test_job", "version": "1.0"}}]}'
recipe_definition = RecipeDefinition(json.loads(definition_json_str))
warnings = rule_config.validate_trigger_for_recipe(recipe_definition)
self.assertEqual(len(warnings), 1)
def test_get_job_type_keys_empty(self):
"""Tests getting job type keys when there are no jobs defined."""
definition = {
'version': '1.0',
'input_data': [],
'jobs': [],
}
recipe_definition = RecipeDefinition(definition)
results = recipe_definition.get_job_type_keys()
self.assertSetEqual(results, set())
def test_successful(self):
"""Tests calling ParseTriggerRuleConfiguration.validate_trigger_for_recipe() successfully with no warnings"""
rule_json_str = '{"version": "1.0", "condition": {"media_type": "text/plain", "data_types": ["A", "B"]}, "data": {"input_data_name": "my_input", "workspace_name": "my_workspace"}}'
rule_config = ParseTriggerRuleConfiguration(PARSE_TYPE, json.loads(rule_json_str))
definition_json_str = '{"version": "1.0", "input_data": [{"name": "my_input", "type": "file", "media_types": ["text/plain", "application/json"]}], "jobs": [{"name": "my_job", "job_type": {"name": "test_job", "version": "1.0"}}]}'
recipe_definition = RecipeDefinition(json.loads(definition_json_str))
warnings = rule_config.validate_trigger_for_recipe(recipe_definition)
self.assertListEqual(warnings, [])
def test_missing_workspace(self, mock_store):
"""Tests calling RecipeDefinition.validate_data() with a missing required workspace."""
definition = {
'version':
'1.0',
'input_data': [{
'name': 'Recipe Input',
'type': 'file',
'media_types': ['text/plain'],
}],
'jobs': [{
'name':
'Job 1',
'job_type': {
'name': self.job_type_1.name,
'version': self.job_type_1.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input',
'job_input': self.input_name_1,
}],
}, {
'name':
'Job 2',
'job_type': {
'name': self.job_type_2.name,
'version': self.job_type_2.version,
},
'dependencies': [{
'name':
'Job 1',
'connections': [{
'output': self.output_name_1,
'input': self.input_name_2,
}],
}],
}],
}
recipe = RecipeDefinition(definition)
recipe.validate_job_interfaces()
data = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input',
'file_id': self.file_1.id,
}],
}
recipe_data = LegacyRecipeData(data)
self.assertRaises(InvalidRecipeData, recipe.validate_data, recipe_data)
def post(self, request):
"""Validates a new recipe type and returns any warnings discovered
:param request: the HTTP POST request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
"""
name = rest_util.parse_string(request, 'name')
version = rest_util.parse_string(request, 'version')
title = rest_util.parse_string(request, 'title', default_value=name)
description = rest_util.parse_string(request, 'description', required=False)
definition_dict = rest_util.parse_dict(request, 'definition')
# Check for optional trigger rule parameters
trigger_rule_dict = rest_util.parse_dict(request, 'trigger_rule', required=False)
if (('type' in trigger_rule_dict and 'configuration' not in trigger_rule_dict) or
('type' not in trigger_rule_dict and 'configuration' in trigger_rule_dict)):
raise BadParameter('Trigger type and configuration are required together.')
# Attempt to look up the trigger handler for the type
rule_handler = None
if trigger_rule_dict and 'type' in trigger_rule_dict:
try:
rule_handler = trigger_handler.get_trigger_rule_handler(trigger_rule_dict['type'])
except InvalidTriggerType as ex:
logger.exception('Invalid trigger type for recipe validation: %s', name)
raise BadParameter(unicode(ex))
# Attempt to look up the trigger rule configuration
trigger_config = None
if rule_handler and 'configuration' in trigger_rule_dict:
try:
trigger_config = rule_handler.create_configuration(trigger_rule_dict['configuration'])
except InvalidTriggerRule as ex:
logger.exception('Invalid trigger rule configuration for recipe validation: %s', name)
raise BadParameter(unicode(ex))
# Validate the recipe definition
try:
recipe_def = RecipeDefinition(definition_dict)
warnings = RecipeType.objects.validate_recipe_type(name, title, version, description, recipe_def,
trigger_config)
except (InvalidDefinition, InvalidTriggerType, InvalidTriggerRule, InvalidRecipeConnection) as ex:
logger.exception('Unable to validate new recipe type: %s', name)
raise BadParameter(unicode(ex))
results = [{'id': w.key, 'details': w.details} for w in warnings]
return Response({'warnings': results})
def test_successful(self):
"""Tests calling RecipeDefinition.get_graph() successfully"""
definition = {
'version':
'1.0',
'input_data': [{
'name': 'Recipe Input',
'type': 'file',
'media_types': ['text/plain'],
}],
'jobs': [{
'name':
'Job 1',
'job_type': {
'name': self.job_type_1.name,
'version': self.job_type_1.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input',
'job_input': self.input_name_1,
}]
}, {
'name':
'Job 2',
'job_type': {
'name': self.job_type_2.name,
'version': self.job_type_2.version,
},
'dependencies': [{
'name':
'Job 1',
'connections': [{
'output': self.output_name_1,
'input': self.input_name_2,
}],
}],
}],
}
recipe_definition = RecipeDefinition(definition)
graph = recipe_definition.get_graph()
self.assertEqual(len(graph.inputs), 1)
self.assertEqual(len(graph._root_nodes), 1)
self.assertEqual(len(graph._nodes), 2)
def test_successful_new_recipe(self):
"""Tests calling RecipeDefinition.get_job_type_map() successfully."""
definition = {
'version':
'1.0',
'input_data': [{
'name': 'Recipe Input',
'type': 'file',
'media_types': ['text/plain'],
}],
'jobs': [{
'name':
'Job 1',
'job_type': {
'name': self.job_type1.name,
'version': self.job_type1.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input',
'job_input': 'Input 1',
}]
}, {
'name':
'Job 2',
'job_type': {
'name': self.job_type2.name,
'version': self.job_type2.version,
},
'dependencies': [{
'name':
'Job 1',
'connections': [{
'output': 'Output 1',
'input': 'Input 2',
}]
}]
}]
}
recipe_definition = RecipeDefinition(definition)
results = recipe_definition.get_job_type_map()
self.assertDictEqual(results, {
'Job 1': self.job_type1,
'Job 2': self.job_type2
})
def test_successful(self):
'''Tests calling RecipeDefinition.validate_job_interfaces() successfully.'''
definition = {
'version':
'1.0',
'input_data': [{
'name': 'Recipe Input',
'type': 'file',
'media_types': ['text/plain'],
}],
'jobs': [{
'name':
'Job 1',
'job_type': {
'name': self.job_type_1.name,
'version': self.job_type_1.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input',
'job_input': self.input_name_1,
}],
}, {
'name':
'Job 2',
'job_type': {
'name': self.job_type_2.name,
'version': self.job_type_2.version,
},
'dependencies': [{
'name':
'Job 1',
'connections': [{
'output': self.output_name_1,
'input': self.input_name_2,
}],
}],
}],
}
recipe = RecipeDefinition(definition)
# No exception is success
recipe.validate_job_interfaces()
def test_get_job_types_one(self):
'''Tests getting a job type from the definition.'''
definition = {
'version':
'1.0',
'input_data': [],
'jobs': [{
'name': 'Job 1',
'job_type': {
'name': self.job_type1.name,
'version': self.job_type1.version,
},
}],
}
recipe_definition = RecipeDefinition(definition)
results = recipe_definition.get_job_types()
self.assertSetEqual(results, {self.job_type1})
def test_init_successful(self):
"""Tests calling RecipeDefinition constructor successfully."""
definition = {
'version':
'1.0',
'input_data': [{
'name': 'Recipe Input',
'type': 'property'
}],
'jobs': [{
'name': 'Job 1',
'job_type': {
'name': self.job_type1.name,
'version': self.job_type1.version,
},
}, {
'name':
'Job 2',
'job_type': {
'name': self.job_type2.name,
'version': self.job_type2.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input',
'job_input': 'Input 1',
}],
'dependencies': [{
'name':
'Job 1',
'connections': [{
'output': 'Output 1',
'input': 'Input 2',
}],
}],
}],
}
# No exception is success
RecipeDefinition(definition)
def test_successful_no_workspace(self, mock_store):
"""Tests calling RecipeDefinition.validate_data() successfully with no workspace."""
definition = {
'version':
'1.0',
'input_data': [{
'name': 'Recipe Input',
'type': 'file',
'media_types': ['text/plain'],
}],
'jobs': [{
'name':
'Job 3',
'job_type': {
'name': self.job_type_3.name,
'version': self.job_type_3.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input',
'job_input': self.input_name_3,
}],
}],
}
recipe = RecipeDefinition(definition)
recipe.validate_job_interfaces()
data = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input',
'file_id': self.file_1.id,
}],
}
recipe_data = LegacyRecipeData(data)
# No exception is success
recipe.validate_data(recipe_data)
def test_reprocess_identical_node(self):
"""Tests calling RecipeGraphDelta.reprocess_identical_node() to indicate identical nodes that should be marked
as changed"""
definition_a = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input 1',
'type': 'file',
'media_types': ['text/plain'],
}, {
'name': 'Recipe Input 2',
'type': 'property'
}],
'jobs': [{
'name': 'Job A',
'job_type': {
'name': self.job_a.job_type.name,
'version': self.job_a.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 1',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job B',
'job_type': {
'name': self.job_b.job_type.name,
'version': self.job_b.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job C',
'job_type': {
'name': self.job_c.job_type.name,
'version': self.job_c.job_type.version,
},
'dependencies': [{
'name': 'Job B',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}]
}, {
'name': 'Job D',
'job_type': {
'name': self.job_d.job_type.name,
'version': self.job_d.job_type.version,
},
'dependencies': [{
'name': 'Job B',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}]
}]
}
graph_a = RecipeDefinition(definition_a).get_graph()
definition_b = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input 1',
'type': 'file',
'media_types': ['text/plain'],
}, {
'name': 'Recipe Input 2',
'type': 'property'
}],
'jobs': [{
'name': 'Job 1',
'job_type': {
'name': self.job_a.job_type.name,
'version': self.job_a.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 1',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job 2',
'job_type': {
'name': self.job_b.job_type.name,
'version': self.job_b.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job 4',
'job_type': {
'name': self.job_d.job_type.name,
'version': self.job_d.job_type.version,
},
'dependencies': [{
'name': 'Job 2',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}]
}, {
'name': 'Job 5',
'job_type': {
'name': self.job_a.job_type.name,
'version': self.job_a.job_type.version,
},
'dependencies': [{
'name': 'Job 4',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}]
}]
}
graph_b = RecipeDefinition(definition_b).get_graph()
# Initial delta
delta = RecipeGraphDelta(graph_a, graph_b)
expected_identical = {'Job 1': 'Job A', 'Job 2': 'Job B', 'Job 4': 'Job D'}
expected_deleted = {'Job C'}
expected_new = {'Job 5'}
self.assertTrue(delta.can_be_reprocessed)
self.assertDictEqual(delta.get_changed_nodes(), {})
self.assertSetEqual(delta.get_deleted_nodes(), expected_deleted)
self.assertDictEqual(delta.get_identical_nodes(), expected_identical)
self.assertSetEqual(delta.get_new_nodes(), expected_new)
# Mark Job 2 (and its child Job 4) as changed so it will be reprocessed
delta.reprocess_identical_node('Job 2')
expected_changed = {'Job 2': 'Job B', 'Job 4': 'Job D'}
expected_identical = {'Job 1': 'Job A'}
expected_deleted = {'Job C'}
expected_new = {'Job 5'}
self.assertTrue(delta.can_be_reprocessed)
self.assertDictEqual(delta.get_changed_nodes(), expected_changed)
self.assertSetEqual(delta.get_deleted_nodes(), expected_deleted)
self.assertDictEqual(delta.get_identical_nodes(), expected_identical)
self.assertSetEqual(delta.get_new_nodes(), expected_new)
def test_init_changed(self):
"""Tests creating a RecipeGraphDelta between two graphs where some nodes were changed (and 1 deleted)"""
definition_a = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input 1',
'type': 'file',
'media_types': ['text/plain'],
}, {
'name': 'Recipe Input 2',
'type': 'property'
}],
'jobs': [{
'name': 'Job A',
'job_type': {
'name': self.job_a.job_type.name,
'version': self.job_a.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 1',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job B',
'job_type': {
'name': self.job_b.job_type.name,
'version': self.job_b.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job C',
'job_type': {
'name': self.job_c.job_type.name,
'version': self.job_c.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job D',
'job_type': {
'name': self.job_d.job_type.name,
'version': self.job_d.job_type.version,
},
'dependencies': [{
'name': 'Job A',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}, {
'name': 'Job B',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 2',
}],
}]
}, {
'name': 'Job E',
'job_type': {
'name': self.job_d.job_type.name,
'version': self.job_d.job_type.version,
},
'dependencies': [{
'name': 'Job D',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}]
}]
}
graph_a = RecipeDefinition(definition_a).get_graph()
definition_b = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input 1',
'type': 'file',
'media_types': ['text/plain'],
}, {
'name': 'Recipe Input 2',
'type': 'property'
}],
'jobs': [{
'name': 'Job A',
'job_type': {
'name': self.job_a.job_type.name,
'version': self.job_a.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 1',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job B',
'job_type': {
'name': self.job_b.job_type.name,
'version': self.job_b.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job D',
'job_type': {
'name': self.job_d.job_type.name,
'version': 'new_version',
},
'dependencies': [{
'name': 'Job A',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}, {
'name': 'Job B',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 2',
}],
}]
}, {
'name': 'Job E',
'job_type': {
'name': self.job_d.job_type.name,
'version': self.job_d.job_type.version,
},
'dependencies': [{
'name': 'Job D',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}]
}]
}
graph_b = RecipeDefinition(definition_b).get_graph()
delta = RecipeGraphDelta(graph_a, graph_b)
expected_identical = {'Job A': 'Job A', 'Job B': 'Job B'}
expected_changed = {'Job D': 'Job D', 'Job E': 'Job E'}
expected_deleted = {'Job C'}
expected_new = set()
self.assertTrue(delta.can_be_reprocessed)
self.assertDictEqual(delta.get_changed_nodes(), expected_changed)
self.assertSetEqual(delta.get_deleted_nodes(), expected_deleted)
self.assertDictEqual(delta.get_identical_nodes(), expected_identical)
self.assertSetEqual(delta.get_new_nodes(), expected_new)
def test_init_new_required_input(self):
"""Tests creating a RecipeGraphDelta between two graphs where the new graph has a new required input that blocks
reprocessing
"""
definition_a = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input 1',
'type': 'file',
'media_types': ['text/plain'],
}, {
'name': 'Recipe Input 2',
'type': 'property'
}],
'jobs': [{
'name': 'Job A',
'job_type': {
'name': self.job_a.job_type.name,
'version': self.job_a.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 1',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job B',
'job_type': {
'name': self.job_b.job_type.name,
'version': self.job_b.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job C',
'job_type': {
'name': self.job_c.job_type.name,
'version': self.job_c.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}]
}
graph_a = RecipeDefinition(definition_a).get_graph()
definition_b = {
'version': '1.0',
'input_data': [{
'name': 'New Recipe Input 1',
'type': 'file',
'media_types': ['text/plain'],
}, {
'name': 'Recipe Input 2',
'type': 'property'
}],
'jobs': [{
'name': 'Job A',
'job_type': {
'name': self.job_a.job_type.name,
'version': self.job_a.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'New Recipe Input 1',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job 2',
'job_type': {
'name': self.job_b.job_type.name,
'version': self.job_b.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job 3',
'job_type': {
'name': self.job_c.job_type.name,
'version': self.job_c.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}]
}
graph_b = RecipeDefinition(definition_b).get_graph()
delta = RecipeGraphDelta(graph_a, graph_b)
self.assertFalse(delta.can_be_reprocessed)
self.assertDictEqual(delta.get_changed_nodes(), {'Job A': 'Job A'})
self.assertSetEqual(delta.get_deleted_nodes(), set())
self.assertDictEqual(delta.get_identical_nodes(), {'Job 2': 'Job B', 'Job 3': 'Job C'})
self.assertSetEqual(delta.get_new_nodes(), set())
def test_init_identical(self):
"""Tests creating a RecipeGraphDelta between two identical graphs"""
definition_a = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input 1',
'type': 'file',
'media_types': ['text/plain'],
}, {
'name': 'Recipe Input 2',
'type': 'property'
}],
'jobs': [{
'name': 'Job A',
'job_type': {
'name': self.job_a.job_type.name,
'version': self.job_a.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 1',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job B',
'job_type': {
'name': self.job_b.job_type.name,
'version': self.job_b.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job C',
'job_type': {
'name': self.job_c.job_type.name,
'version': self.job_c.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job D',
'job_type': {
'name': self.job_d.job_type.name,
'version': self.job_d.job_type.version,
},
'dependencies': [{
'name': 'Job A',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}, {
'name': 'Job B',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 2',
}],
}]
}, {
'name': 'Job E',
'job_type': {
'name': self.job_e.job_type.name,
'version': self.job_e.job_type.version,
},
'dependencies': [{
'name': 'Job B',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}]
}, {
'name': 'Job F',
'job_type': {
'name': self.job_f.job_type.name,
'version': self.job_f.job_type.version,
},
'dependencies': [{
'name': 'Job D',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}]
}, {
'name': 'Job G',
'job_type': {
'name': self.job_g.job_type.name,
'version': self.job_g.job_type.version,
},
'dependencies': [{
'name': 'Job D',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}, {
'name': 'Job E',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 2',
}],
}]
}, {
'name': 'Job H',
'job_type': {
'name': self.job_h.job_type.name,
'version': self.job_h.job_type.version,
},
'dependencies': [{
'name': 'Job C',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}, {
'name': 'Job D',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 2',
}],
}]
}]
}
graph_a = RecipeDefinition(definition_a).get_graph()
definition_b = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input 1',
'type': 'file',
'media_types': ['text/plain'],
}, {
'name': 'Recipe Input 2',
'type': 'property'
}],
'jobs': [{
'name': 'Job 1',
'job_type': {
'name': self.job_a.job_type.name,
'version': self.job_a.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 1',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job 2',
'job_type': {
'name': self.job_b.job_type.name,
'version': self.job_b.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job 3',
'job_type': {
'name': self.job_c.job_type.name,
'version': self.job_c.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job 4',
'job_type': {
'name': self.job_d.job_type.name,
'version': self.job_d.job_type.version,
},
'dependencies': [{
'name': 'Job 1',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}, {
'name': 'Job 2',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 2',
}],
}]
}, {
'name': 'Job 5',
'job_type': {
'name': self.job_e.job_type.name,
'version': self.job_e.job_type.version,
},
'dependencies': [{
'name': 'Job 2',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}]
}, {
'name': 'Job 6',
'job_type': {
'name': self.job_f.job_type.name,
'version': self.job_f.job_type.version,
},
'dependencies': [{
'name': 'Job 4',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}]
}, {
'name': 'Job 7',
'job_type': {
'name': self.job_g.job_type.name,
'version': self.job_g.job_type.version,
},
'dependencies': [{
'name': 'Job 4',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}, {
'name': 'Job 5',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 2',
}],
}]
}, {
'name': 'Job 8',
'job_type': {
'name': self.job_h.job_type.name,
'version': self.job_h.job_type.version,
},
'dependencies': [{
'name': 'Job 3',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}, {
'name': 'Job 4',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 2',
}],
}]
}]
}
graph_b = RecipeDefinition(definition_b).get_graph()
delta = RecipeGraphDelta(graph_a, graph_b)
expected_results = {'Job 1': 'Job A', 'Job 2': 'Job B', 'Job 3': 'Job C', 'Job 4': 'Job D', 'Job 5': 'Job E',
'Job 6': 'Job F', 'Job 7': 'Job G', 'Job 8': 'Job H'}
self.assertTrue(delta.can_be_reprocessed)
self.assertDictEqual(delta.get_changed_nodes(), {})
self.assertSetEqual(delta.get_deleted_nodes(), set())
self.assertDictEqual(delta.get_identical_nodes(), expected_results)
self.assertSetEqual(delta.get_new_nodes(), set())
def setUp(self):
django.setup()
self.workspace = storage_test_utils.create_workspace()
interface_1 = {
'version':
'1.0',
'command':
'my_command',
'command_arguments':
'args',
'input_data': [{
'name': 'Test Input 1',
'type': 'file',
'media_types': ['text/plain'],
}],
'output_data': [{
'name': 'Test Output 1',
'type': 'files',
'media_type': 'image/png',
}]
}
self.job_type_1 = job_test_utils.create_job_type(interface=interface_1)
interface_2 = {
'version':
'1.0',
'command':
'my_command',
'command_arguments':
'args',
'input_data': [{
'name': 'Test Input 2',
'type': 'files',
'media_types': ['image/png', 'image/tiff'],
}],
'output_data': [{
'name': 'Test Output 2',
'type': 'file',
}]
}
self.job_type_2 = job_test_utils.create_job_type(interface=interface_2)
self.definition = {
'version':
'1.0',
'input_data': [{
'name': 'Recipe Input',
'type': 'file',
'media_types': ['text/plain'],
}],
'jobs': [{
'name':
'Job 1',
'job_type': {
'name': self.job_type_1.name,
'version': self.job_type_1.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input',
'job_input': 'Test Input 1',
}]
}, {
'name':
'Job 2',
'job_type': {
'name': self.job_type_2.name,
'version': self.job_type_2.version,
},
'dependencies': [{
'name':
'Job 1',
'connections': [{
'output': 'Test Output 1',
'input': 'Test Input 2',
}]
}]
}]
}
self.recipe_def = RecipeDefinition(self.definition)
self.recipe_def.validate_job_interfaces()
self.new_definition = {
'version':
'1.0',
'input_data': [{
'name': 'Recipe Input',
'type': 'file',
'media_types': ['text/plain'],
}],
'jobs': [{
'name':
'Job 1',
'job_type': {
'name': self.job_type_1.name,
'version': self.job_type_1.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input',
'job_input': 'Test Input 1',
}]
}]
}
self.new_recipe_def = RecipeDefinition(self.new_definition)
self.new_recipe_def.validate_job_interfaces()
self.configuration = {
'version': '1.0',
'condition': {
'media_type': 'text/plain'
},
'data': {
'input_data_name': 'Recipe Input',
'workspace_name': self.workspace.name
}
}
self.trigger_config = recipe_test_utils.MockTriggerRuleConfiguration(
recipe_test_utils.MOCK_TYPE, self.configuration)
self.new_configuration = {
'version': '1.0',
'condition': {
'media_type': 'application/json'
},
'data': {
'input_data_name': 'Recipe Input',
'workspace_name': self.workspace.name
}
}
self.new_trigger_config = recipe_test_utils.MockTriggerRuleConfiguration(
recipe_test_utils.MOCK_TYPE, self.new_configuration)
def test_init_bare_min(self):
"""Tests calling RecipeDefinition constructor with bare minimum JSON."""
# No exception is success
RecipeDefinition({'jobs': []})
def setUp(self):
django.setup()
self.workspace = storage_test_utils.create_workspace()
self.file = storage_test_utils.create_file()
configuration = {
'version': '1.0',
'condition': {
'media_type': 'text/plain',
},
'data': {
'input_data_name': 'Recipe Input',
'workspace_name': self.workspace.name,
},
}
self.rule = trigger_test_utils.create_trigger_rule(
configuration=configuration)
self.event = trigger_test_utils.create_trigger_event(rule=self.rule)
interface_1 = {
'version':
'1.0',
'command':
'my_command',
'command_arguments':
'args',
'input_data': [{
'name': 'Test Input 1',
'type': 'file',
'media_types': ['text/plain'],
}],
'output_data': [{
'name': 'Test Output 1',
'type': 'files',
'media_type': 'image/png',
}],
}
self.job_type_1 = job_test_utils.create_job_type(interface=interface_1)
self.definition_1 = {
'version':
'1.0',
'input_data': [{
'name': 'Recipe Input',
'type': 'file',
'media_types': ['text/plain'],
}],
'jobs': [{
'name':
'Job 1',
'job_type': {
'name': self.job_type_1.name,
'version': self.job_type_1.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input',
'job_input': 'Test Input 1',
}],
}],
}
RecipeDefinition(self.definition_1).validate_job_interfaces()
self.recipe_type = recipe_test_utils.create_recipe_type(
definition=self.definition_1, trigger_rule=self.rule)
self.interface_2 = {
'version':
'1.0',
'command':
'my_command',
'command_arguments':
'args',
'input_data': [{
'name': 'Test Input 2',
'type': 'files',
'media_types': ['image/tiff'],
}],
'output_data': [{
'name': 'Test Output 2',
'type': 'file',
}],
}
self.job_type_2 = job_test_utils.create_job_type(
interface=self.interface_2)
self.definition_2 = {
'version':
'1.0',
'input_data': [{
'name': 'Recipe Input',
'type': 'file',
'media_types': ['text/plain'],
}],
'jobs': [{
'name':
'Job 1',
'job_type': {
'name': self.job_type_1.name,
'version': self.job_type_1.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input',
'job_input': 'Test Input 1',
}]
}, {
'name':
'Job 2',
'job_type': {
'name': self.job_type_2.name,
'version': self.job_type_2.version,
},
'dependencies': [{
'name':
'Job 1',
'connections': [{
'output': 'Test Output 1',
'input': 'Test Input 2',
}],
}],
}],
}
self.data = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input',
'file_id': self.file.id,
}],
'workspace_id': self.workspace.id,
}
def setUp(self):
django.setup()
self.job_a = job_test_utils.create_job()
self.job_b = job_test_utils.create_job()
self.job_c = job_test_utils.create_job()
self.job_d = job_test_utils.create_job()
self.job_e = job_test_utils.create_job()
self.job_f = job_test_utils.create_job()
self.job_g = job_test_utils.create_job()
self.job_h = job_test_utils.create_job()
definition = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input 1',
'type': 'file',
'media_types': ['text/plain'],
}, {
'name': 'Recipe Input 2',
'type': 'property'
}],
'jobs': [{
'name': 'Job A',
'job_type': {
'name': self.job_a.job_type.name,
'version': self.job_a.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 1',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job B',
'job_type': {
'name': self.job_b.job_type.name,
'version': self.job_b.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job C',
'job_type': {
'name': self.job_c.job_type.name,
'version': self.job_c.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input 2',
'job_input': 'Job Input 1',
}]
}, {
'name': 'Job D',
'job_type': {
'name': self.job_d.job_type.name,
'version': self.job_d.job_type.version,
},
'dependencies': [{
'name': 'Job A',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}, {
'name': 'Job B',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 2',
}],
}]
}, {
'name': 'Job E',
'job_type': {
'name': self.job_e.job_type.name,
'version': self.job_e.job_type.version,
},
'dependencies': [{
'name': 'Job B',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}]
}, {
'name': 'Job F',
'job_type': {
'name': self.job_f.job_type.name,
'version': self.job_f.job_type.version,
},
'dependencies': [{
'name': 'Job D',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}]
}, {
'name': 'Job G',
'job_type': {
'name': self.job_g.job_type.name,
'version': self.job_g.job_type.version,
},
'dependencies': [{
'name': 'Job D',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}, {
'name': 'Job E',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 2',
}],
}]
}, {
'name': 'Job H',
'job_type': {
'name': self.job_h.job_type.name,
'version': self.job_h.job_type.version,
},
'dependencies': [{
'name': 'Job C',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 1',
}],
}, {
'name': 'Job D',
'connections': [{
'output': 'Job Output 1',
'input': 'Job Input 2',
}],
}]
}]
}
recipe_definition = RecipeDefinition(definition)
self.graph = recipe_definition.get_graph()
def setUp(self):
django.setup()
workspace = storage_test_utils.create_workspace()
source_file = source_test_utils.create_source(workspace=workspace)
self.event = trigger_test_utils.create_trigger_event()
interface_1 = {
'version':
'1.0',
'command':
'test_command',
'command_arguments':
'test_arg',
'input_data': [{
'name': 'Test Input 1',
'type': 'file',
'media_types': ['text/plain'],
}],
'output_data': [{
'name': 'Test Output 1',
'type': 'files',
'media_type': 'image/png',
}]
}
self.job_type_1 = job_test_utils.create_job_type(interface=interface_1)
interface_2 = {
'version':
'1.0',
'command':
'test_command',
'command_arguments':
'test_arg',
'input_data': [{
'name': 'Test Input 2',
'type': 'files',
'media_types': ['image/png', 'image/tiff'],
}],
'output_data': [{
'name': 'Test Output 2',
'type': 'file',
}]
}
self.job_type_2 = job_test_utils.create_job_type(interface=interface_2)
definition = {
'version':
'1.0',
'input_data': [{
'name': 'Recipe Input',
'type': 'file',
'media_types': ['text/plain'],
}],
'jobs': [{
'name':
'Job 1',
'job_type': {
'name': self.job_type_1.name,
'version': self.job_type_1.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input',
'job_input': 'Test Input 1',
}]
}, {
'name':
'Job 2',
'job_type': {
'name': self.job_type_2.name,
'version': self.job_type_2.version,
},
'dependencies': [{
'name':
'Job 1',
'connections': [{
'output': 'Test Output 1',
'input': 'Test Input 2',
}]
}]
}]
}
recipe_definition = RecipeDefinition(definition)
recipe_definition.validate_job_interfaces()
self.recipe_type = recipe_test_utils.create_recipe_type(
definition=definition)
self.data = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input',
'file_id': source_file.id,
}],
'workspace_id': workspace.id,
}
# Register a fake processor
self.mock_processor = MagicMock(QueueEventProcessor)
Queue.objects.register_processor(lambda: self.mock_processor)
