def setUp(self):
django.setup()
self.job_type1 = job_test_utils.create_seed_job_type(manifest=job_test_utils.MINIMUM_MANIFEST)
self.job_type2 = job_test_utils.create_seed_job_type()
self.sub_definition = copy.deepcopy(recipe_test_utils.SUB_RECIPE_DEFINITION)
self.sub_definition['nodes']['node_a']['node_type']['job_type_name'] = self.job_type1.name
self.sub_definition['nodes']['node_a']['node_type']['job_type_version'] = self.job_type1.version
self.sub_definition['nodes']['node_a']['node_type']['job_type_revision'] = self.job_type1.revision_num
self.sub_def = RecipeDefinitionV6(self.sub_definition).get_definition()
self.recipe_type1 = recipe_test_utils.create_recipe_type_v6(definition=self.sub_definition,
description="A sub recipe",
is_active=False,
is_system=False)
self.main_definition = copy.deepcopy(recipe_test_utils.RECIPE_DEFINITION)
self.main_definition['nodes']['node_a']['node_type']['job_type_name'] = self.job_type2.name
self.main_definition['nodes']['node_a']['node_type']['job_type_version'] = self.job_type2.version
self.main_definition['nodes']['node_a']['node_type']['job_type_revision'] = self.job_type2.revision_num
self.main_definition['nodes']['node_b']['node_type']['job_type_name'] = self.job_type2.name
self.main_definition['nodes']['node_b']['node_type']['job_type_version'] = self.job_type2.version
self.main_definition['nodes']['node_b']['node_type']['job_type_revision'] = self.job_type2.revision_num
self.main_definition['nodes']['node_d']['node_type']['recipe_type_name'] = self.recipe_type1.name
self.main_definition['nodes']['node_d']['node_type']['recipe_type_revision'] = self.recipe_type1.revision_num
self.v6_recipe_def = RecipeDefinitionV6(self.main_definition).get_definition()
def test_init_validation(self):
"""Tests the validation done in __init__"""
# Try minimal acceptable configuration
RecipeDefinitionV6(do_validate=True)
# Invalid version
definition = {'version': 'BAD'}
self.assertRaises(InvalidDefinition, RecipeDefinitionV6, definition, True)
# Valid v6 definition
def_v6_dict = {'version': '6',
'input': {'files': [{'name': 'foo', 'media_types': ['image/tiff'], 'required': True,
'multiple': True}],
'json': [{'name': 'bar', 'type': 'string', 'required': False}]},
'nodes': {'node_a': {'dependencies': [],
'input': {'input_a': {'type': 'recipe', 'input': 'foo'}},
'node_type': {'node_type': 'job', 'job_type_name': 'job-type-1',
'job_type_version': '1.0', 'job_type_revision': 1}},
'node_b': {'dependencies': [{'name': 'node_a'}],
'input': {'input_a': {'type': 'recipe', 'input': 'foo'},
'input_b': {'type': 'dependency', 'node': 'node_a',
'output': 'output_a'}},
'node_type': {'node_type': 'job', 'job_type_name': 'job-type-2',
'job_type_version': '2.0', 'job_type_revision': 1}},
'node_c': {'dependencies': [{'name': 'node_b'}],
'input': {'input_a': {'type': 'recipe', 'input': 'bar'},
'input_b': {'type': 'dependency', 'node': 'node_b',
'output': 'output_a'}},
'node_type': { 'node_type': 'condition',
'interface': {'files': [{'name': 'input_b',
'media_types': ['image/tiff'],
'required': True,
'multiple': True}],
'json': []},
'data_filter': {'filters': [{'name': 'output_a',
'type': 'media-type',
'condition': '==',
'values': ['image/tiff']}]}}},
'node_d': {'dependencies': [{'name': 'node_c'}],
'input': {'input_a': {'type': 'recipe', 'input': 'bar'},
'input_b': {'type': 'dependency', 'node': 'node_c',
'output': 'output_a'}},
'node_type': {'node_type': 'recipe', 'recipe_type_name': 'recipe-type-1',
'recipe_type_revision': 5}}}}
try:
RecipeDefinitionV6(definition=def_v6_dict, do_validate=True)
except InvalidDefinition:
self.fail('Recipe definition failed validation unexpectedly')
def edit_recipe_type_v6(recipe_type, title=None, description=None, definition=None, auto_update=True, is_active=True):
"""Updates the definition of a recipe type, including creating a new revision for unit testing
"""
with transaction.atomic():
RecipeType.objects.edit_recipe_type_v6(recipe_type.id, title=title, description=description,
definition=RecipeDefinitionV6(definition).get_definition(),
auto_update=auto_update, is_active=is_active)
def test_get_definition_empty(self):
"""Tests calling get_definition() from an empty JSON"""
json = RecipeDefinitionV6(do_validate=True)
definition = json.get_definition()
self.assertDictEqual(definition.input_interface.parameters, {})
self.assertDictEqual(definition.graph, {})
def test_convert_recipe_definition_to_v6_json_full(self):
"""Tests calling convert_recipe_definition_to_v6_json() with a full definition"""
interface = Interface()
interface.add_parameter(FileParameter('file_param_a', ['image/gif']))
interface.add_parameter(JsonParameter('json_param_a', 'object'))
interface.add_parameter(
JsonParameter('json_param_b', 'object', required=False))
definition = RecipeDefinition(interface)
definition.add_job_node('A', 'job_type_1', '1.0', 1)
definition.add_job_node('B', 'job_type_2', '2.0', 1)
definition.add_job_node('C', 'job_type_3', '1.0', 2)
definition.add_recipe_node('D', 'recipe_type_1', 1)
definition.add_job_node('E', 'job_type_4', '1.0', 1)
definition.add_dependency('A', 'B')
definition.add_dependency('A', 'C')
definition.add_dependency('B', 'E')
definition.add_dependency('C', 'D')
definition.add_recipe_input_connection('A', 'input_1', 'file_param_a')
definition.add_dependency_input_connection('B', 'b_input_1', 'A',
'a_output_1')
definition.add_dependency_input_connection('C', 'c_input_1', 'A',
'a_output_2')
definition.add_dependency_input_connection('D', 'd_input_1', 'C',
'c_output_1')
definition.add_recipe_input_connection('D', 'd_input_2',
'json_param_a')
json = convert_recipe_definition_to_v6_json(definition)
RecipeDefinitionV6(definition=json.get_dict(),
do_validate=True) # Revalidate
self.assertSetEqual(set(json.get_dict()['nodes'].keys()),
{'A', 'B', 'C', 'D', 'E'})
def test_convert_recipe_definition_to_v6_json_empty(self):
"""Tests calling convert_recipe_definition_to_v6_json() with an empty definition"""
interface = Interface()
definition = RecipeDefinition(interface)
json = convert_recipe_definition_to_v6_json(definition)
RecipeDefinitionV6(definition=json.get_dict(), do_validate=True) # Revalidate
self.assertDictEqual(json.get_dict()['input'], {'files': [], 'json': []})
def patch_v6(self, request, name):
"""Edits an existing recipe type and returns the updated details
:param request: the HTTP GET request
:type request: :class:`rest_framework.request.Request`
:param name: The name of the recipe type
:type name: string
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
"""
title = rest_util.parse_string(request, 'title', required=False)
description = rest_util.parse_string(request,
'description',
required=False)
definition_dict = rest_util.parse_dict(request,
'definition',
required=False)
auto_update = rest_util.parse_bool(request,
'auto_update',
required=False,
default_value=True)
is_active = rest_util.parse_bool(request, 'is_active', required=False)
# Fetch the current recipe type model
try:
recipe_type = RecipeType.objects.filter(name=name).first()
except RecipeType.DoesNotExist:
raise Http404
try:
with transaction.atomic():
# Validate the recipe definition
recipe_def = None
if definition_dict:
recipe_def = RecipeDefinitionV6(
definition=definition_dict,
do_validate=True).get_definition()
# Edit the recipe type
validation = RecipeType.objects.edit_recipe_type_v6(
recipe_type_id=recipe_type.id,
title=title,
description=description,
definition=recipe_def,
auto_update=auto_update,
is_active=is_active)
except InvalidDefinition as ex:
logger.exception('Unable to update recipe type: %s', name)
raise BadParameter(unicode(ex))
resp_dict = {
'is_valid': validation.is_valid,
'errors': [e.to_dict() for e in validation.errors],
'warnings': [w.to_dict() for w in validation.warnings],
'diff': validation.diff
}
return Response(resp_dict)
def _create_v6(self, request):
"""Creates a new recipe type and returns a link to the detail URL
:param request: the HTTP POST request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
"""
title = rest_util.parse_string(request, 'title', required=True)
description = rest_util.parse_string(request,
'description',
required=False)
definition_dict = rest_util.parse_dict(request,
'definition',
required=True)
basename = title_to_basename(title)
existing_recipes = RecipeType.objects.filter(name=basename)
if existing_recipes.count() > 0:
logger.exception(
'Existing recipe types found for %s - will not re-create.',
basename)
raise BadParameter(
unicode(
'Existing recipe types found for %s - will not re-create. Please change the title or patch the existing recipe type.'
% basename))
name = title_to_name(self.queryset, title)
try:
with transaction.atomic():
# Validate the recipe definition
recipe_def = RecipeDefinitionV6(
definition=definition_dict,
do_validate=True).get_definition()
# Create the recipe type
recipe_type = RecipeType.objects.create_recipe_type_v6(
name, title, description, recipe_def)
except InvalidDefinition as ex:
logger.exception('Unable to create new recipe type: %s', name)
raise BadParameter(unicode(ex))
# Fetch the full recipe type with details
try:
recipe_type = RecipeType.objects.get_details_v6(recipe_type.name)
except RecipeType.DoesNotExist:
raise Http404
url = reverse('recipe_type_details_view',
args=[recipe_type.name],
request=request)
serializer = RecipeTypeDetailsSerializerV6(recipe_type)
return Response(serializer.data,
status=status.HTTP_201_CREATED,
headers=dict(location=url))
def test_invalid_definition(self):
"""Tests calling RecipeTypeManager.create_recipe_type_v6() with an invalid definition"""
# Create recipe_type
name = 'test-recipe'
title = 'Test Recipe'
desc = 'Test description'
invalid = copy.deepcopy(recipe_test_utils.RECIPE_DEFINITION)
invalid_def = RecipeDefinitionV6(definition=invalid, do_validate=False).get_definition()
invalid_def.add_dependency('node_b', 'node_a')
self.assertRaises(InvalidDefinition, RecipeType.objects.create_recipe_type_v6, name, title, desc, invalid_def)
def test_change_to_invalid_definition(self, mock_msg_mgr) :
"""Tests calling RecipeTypeManager.edit_recipe_type() with an invalid change to the definition"""
# Create recipe_type
name = 'test-recipe'
title = 'Test Recipe'
desc = 'Test description'
recipe_type = RecipeType.objects.create_recipe_type_v6(name, title, desc, self.v6_recipe_def)
with transaction.atomic():
recipe_type = RecipeType.objects.select_for_update().get(pk=recipe_type.id)
# Edit the recipe
invalid = copy.deepcopy(recipe_test_utils.RECIPE_DEFINITION)
invalid_def = RecipeDefinitionV6(definition=invalid, do_validate=False).get_definition()
invalid_def.add_dependency('node_b', 'node_a')
self.assertRaises(InvalidDefinition, RecipeType.objects.edit_recipe_type_v6, recipe_type.id,
title=None, description=None, definition=invalid_def, auto_update=True, is_active=True)
def create_recipe_type_job_links_from_definition(apps, recipe_type):
"""Goes through a recipe type definition and gets all the job types it contains and creates the appropriate links
:param recipe_type: New/updated recipe type
:type recipe_type: :class:`recipe.models.RecipeType`
:raises :class:`recipe.models.JobType.DoesNotExist`: If it contains a job type that does not exist
"""
definition = RecipeDefinitionV6(definition=recipe_type.definition,
do_validate=False).get_definition()
job_type_ids = get_recipe_job_type_ids(apps, definition)
if len(job_type_ids) > 0:
recipe_type_ids = [recipe_type.id] * len(job_type_ids)
create_recipe_type_job_links(apps, recipe_type_ids, job_type_ids)
def create_recipe_type_sub_links_from_definition(apps, recipe_type):
"""Goes through a recipe type definition, gets all the recipe types it contains and creates the appropriate links
:param recipe_type: New/updated recipe type
:type recipe_type: :class:`recipe.models.RecipeType`
:raises :class:`recipe.models.RecipeType.DoesNotExist`: If it contains a sub recipe type that does not exist
"""
RecipeType = apps.get_model('recipe', 'RecipeType')
definition = RecipeDefinitionV6(definition=recipe_type.definition,
do_validate=False).get_definition()
sub_type_names = definition.get_recipe_type_names()
sub_type_ids = RecipeType.objects.all().filter(
name__in=sub_type_names).values_list('pk', flat=True)
if len(sub_type_ids) > 0:
recipe_type_ids = [recipe_type.id] * len(sub_type_ids)
create_recipe_type_sub_links(recipe_type_ids, sub_type_ids)
def test_get_definition_full(self):
"""Tests calling get_definition() from a full JSON"""
json_dict = {
'input': {
'files': [{
'name': 'foo',
'media_types': ['image/tiff'],
'required': True,
'multiple': True
}],
'json': [{
'name': 'bar',
'type': 'integer',
'required': False
}]
},
'nodes': {
'node_a': {
'dependencies': [],
'input': {
'input_1': {
'type': 'recipe',
'input': 'foo'
}
},
'node_type': {
'node_type': 'job',
'job_type_name': 'job-type-1',
'job_type_version': '1.0',
'job_type_revision': 1
}
},
'node_b': {
'dependencies': [{
'name': 'node_a'
}],
'input': {
'input_1': {
'type': 'recipe',
'input': 'foo'
},
'input_2': {
'type': 'dependency',
'node': 'node_a',
'output': 'output_1'
}
},
'node_type': {
'node_type': 'job',
'job_type_name': 'job-type-2',
'job_type_version': '2.0',
'job_type_revision': 1
}
},
'node_c': {
'dependencies': [{
'name': 'node_b'
}],
'input': {
'input_1': {
'type': 'recipe',
'input': 'bar'
},
'input_2': {
'type': 'dependency',
'node': 'node_b',
'output': 'output_1'
}
},
'node_type': {
'node_type': 'recipe',
'recipe_type_name': 'recipe-type-1',
'recipe_type_revision': 5
}
}
}
}
json = RecipeDefinitionV6(definition=json_dict, do_validate=True)
definition = json.get_definition()
self.assertSetEqual(set(definition.input_interface.parameters.keys()),
{'foo', 'bar'})
self.assertSetEqual(set(definition.graph.keys()),
{'node_a', 'node_b', 'node_c'})
def test_init_validation(self):
"""Tests the validation done in __init__"""
# Try minimal acceptable configuration
RecipeDefinitionV6(do_validate=True)
# Invalid version
definition = {'version': 'BAD'}
self.assertRaises(InvalidDefinition, RecipeDefinitionV6, definition,
True)
# Valid v6 definition
def_v6_dict = {
'version': '6',
'input': {
'files': [{
'name': 'foo',
'media_types': ['image/tiff'],
'required': True,
'multiple': True
}],
'json': [{
'name': 'bar',
'type': 'string',
'required': False
}]
},
'nodes': {
'node_a': {
'dependencies': [],
'input': {
'input_a': {
'type': 'recipe',
'input': 'foo'
}
},
'node_type': {
'node_type': 'job',
'job_type_name': 'job-type-1',
'job_type_version': '1.0',
'job_type_revision': 1
}
},
'node_b': {
'dependencies': [{
'name': 'node_a'
}],
'input': {
'input_a': {
'type': 'recipe',
'input': 'foo'
},
'input_b': {
'type': 'dependency',
'node': 'node_a',
'output': 'output_a'
}
},
'node_type': {
'node_type': 'job',
'job_type_name': 'job-type-2',
'job_type_version': '2.0',
'job_type_revision': 1
}
},
'node_c': {
'dependencies': [{
'name': 'node_b'
}],
'input': {
'input_a': {
'type': 'recipe',
'input': 'bar'
},
'input_b': {
'type': 'dependency',
'node': 'node_b',
'output': 'output_a'
}
},
'node_type': {
'node_type': 'recipe',
'recipe_type_name': 'recipe-type-1',
'recipe_type_revision': 5
}
}
}
}
RecipeDefinitionV6(definition=def_v6_dict, do_validate=True)
# Conversion from v1 definition
job_test_utils.create_job_type(name='job-type-1', version='1.0')
job_test_utils.create_job_type(name='job-type-2', version='2.0')
def_v6_dict = {
'version': '6',
'input': {
'files': [{
'name': 'foo',
'media_types': ['image/tiff'],
'required': True,
'multiple': True
}],
'json': [{
'name': 'bar',
'type': 'string',
'required': False
}]
},
'nodes': {
'node_a': {
'dependencies': [],
'input': {
'input_a': {
'type': 'recipe',
'input': 'foo'
}
},
'node_type': {
'node_type': 'job',
'job_type_name': 'job-type-1',
'job_type_version': '1.0',
'job_type_revision': 1
}
},
'node_b': {
'dependencies': [{
'name': 'node_a'
}],
'input': {
'input_a': {
'type': 'recipe',
'input': 'foo'
},
'input_b': {
'type': 'dependency',
'node': 'node_a',
'output': 'output_a'
}
},
'node_type': {
'node_type': 'job',
'job_type_name': 'job-type-2',
'job_type_version': '2.0',
'job_type_revision': 1
}
}
}
}
def_v1_dict = {
'version':
'1.0',
'input_data': [{
'name': 'foo',
'media_types': ['image/tiff'],
'type': 'files'
}, {
'name': 'bar',
'type': 'property',
'required': False
}],
'jobs': [{
'name':
'node_a',
'job_type': {
'name': 'job-type-1',
'version': '1.0'
},
'recipe_inputs': [{
'recipe_input': 'foo',
'job_input': 'input_a'
}]
}, {
'name':
'node_b',
'job_type': {
'name': 'job-type-2',
'version': '2.0'
},
'recipe_inputs': [{
'recipe_input': 'foo',
'job_input': 'input_a'
}],
'dependencies': [{
'name':
'node_a',
'connections': [{
'output': 'output_a',
'input': 'input_b'
}]
}]
}]
}
def_v6_json = RecipeDefinitionV6(definition=def_v1_dict,
do_validate=True)
self.assertDictEqual(def_v6_json.get_dict(), def_v6_dict)
def edit_recipe_type_v6(recipe_type, definition):
"""Updates the definition of a recipe type, including creating a new revision for unit testing
"""
RecipeType.objects.edit_recipe_type(
recipe_type.id, None, None,
RecipeDefinitionV6(definition).get_definition(), None, False)