def test_successful_file_list(self):
'''Tests calling RecipeData.add_input_to_data() successfully with a file list parameter'''
recipe_input_name = u'foo'
file_ids = [1, 2, 3, 4]
job_input_name = u'bar'
recipe_data = RecipeData({u'input_data': [{u'name': recipe_input_name, u'file_ids': file_ids}]})
job_data = MagicMock()
recipe_data.add_input_to_data(recipe_input_name, job_data, job_input_name)
job_data.add_file_list_input.assert_called_with(job_input_name, file_ids)
def test_successful_file_list(self):
"""Tests calling RecipeData.add_input_to_data() successfully with a file list parameter"""
recipe_input_name = "foo"
file_ids = [1, 2, 3, 4]
job_input_name = "bar"
recipe_data = RecipeData({"input_data": [{"name": recipe_input_name, "file_ids": file_ids}]})
job_data = MagicMock()
recipe_data.add_input_to_data(recipe_input_name, job_data, job_input_name)
job_data.add_file_list_input.assert_called_with(job_input_name, file_ids)
def test_successful_property(self):
"""Tests calling RecipeData.add_input_to_data() successfully with a property parameter"""
recipe_input_name = "foo"
value = "Doctor Who?"
job_input_name = "bar"
recipe_data = RecipeData({"input_data": [{"name": recipe_input_name, "value": value}]})
job_data = MagicMock()
recipe_data.add_input_to_data(recipe_input_name, job_data, job_input_name)
job_data.add_property_input.assert_called_with(job_input_name, value)
def test_successful_property(self):
'''Tests calling RecipeData.add_input_to_data() successfully with a property parameter'''
recipe_input_name = u'foo'
value = u'Doctor Who?'
job_input_name = u'bar'
recipe_data = RecipeData({u'input_data': [{u'name': recipe_input_name, u'value': value}]})
job_data = MagicMock()
recipe_data.add_input_to_data(recipe_input_name, job_data, job_input_name)
job_data.add_property_input.assert_called_with(job_input_name, value)
def process_ingested_source_file(self, source_file, when):
"""Processes the given ingested source file by checking it against all ingest trigger rules and creating the
corresponding jobs and recipes for any triggered rules. All database changes are made in an atomic transaction.
:param source_file: The source file that was ingested
:type source_file: :class:`source.models.SourceFile`
:param when: When the source file was ingested
:type when: :class:`datetime.datetime`
"""
msg = 'Processing trigger rules for ingested source file with media type %s and data types %s'
logger.info(msg, source_file.media_type, str(list(source_file.get_data_type_tags())))
any_rules = False
for entry in RecipeType.objects.get_active_trigger_rules(INGEST_TYPE):
rule = entry[0]
thing_to_create = entry[1]
rule_config = rule.get_configuration()
condition = rule_config.get_condition()
if condition.is_condition_met(source_file):
logger.info(condition.get_triggered_message())
any_rules = True
event = self._create_ingest_trigger_event(source_file, rule, when)
workspace = Workspace.objects.get(name=rule_config.get_workspace_name())
if isinstance(thing_to_create, JobType):
job_type = thing_to_create
job_data = JobData({})
job_data.add_file_input(rule_config.get_input_data_name(), source_file.id)
job_type.get_job_interface().add_workspace_to_data(job_data, workspace.id)
logger.info('Queuing new job of type %s %s', job_type.name, job_type.version)
Queue.objects.queue_new_job(job_type, job_data, event)
elif isinstance(thing_to_create, RecipeType):
recipe_type = thing_to_create
recipe_data = RecipeData({})
recipe_data.add_file_input(rule_config.get_input_data_name(), source_file.id)
recipe_data.set_workspace_id(workspace.id)
logger.info('Queuing new recipe of type %s %s', recipe_type.name, recipe_type.version)
Queue.objects.queue_new_recipe(recipe_type, recipe_data, event)
if not any_rules:
logger.info('No rules triggered')
def process_parse(self, source_file):
'''Processes the given source file parse by creating the appropriate jobs if the rule is triggered. All
database changes are made in an atomic transaction.
:param source_file_id: The source file that was parsed
:type source_file_id: :class:`source.models.SourceFile`
'''
# If this parse file has the correct media type or the correct data types, the rule is triggered
media_type_match = not self._media_type or self._media_type == source_file.media_type
data_types_match = not self._data_types or self._data_types <= source_file.get_data_type_tags()
if not media_type_match or not data_types_match:
return
msg = 'Parse rule for '
if not self._media_type:
msg += 'all media types '
else:
msg += 'media type %s ' % self._media_type
if self._data_types:
msg += 'and data types %s ' % ','.join(self._data_types)
msg += 'was triggered'
logger.info(msg)
event = ParseTriggerEvent(self._model, source_file).save_to_db()
# Create triggered jobs
for job in self._jobs_to_create:
job_type = self._job_type_map[(job['job_type']['name'], job['job_type']['version'])]
file_input_name = job['file_input_name']
job_data = JobData({})
job_data.add_file_input(file_input_name, source_file.id)
# If workspace name has been provided, add that to the job data for each output file
if 'workspace_name' in job:
workspace = self._workspace_map[job['workspace_name']]
job_type.get_job_interface().add_workspace_to_data(job_data, workspace.id)
logger.info('Queuing new job of type %s %s', job_type.name, job_type.version)
Queue.objects.queue_new_job(job_type, job_data.get_dict(), event)
# Create triggered recipes
for recipe in self._recipes_to_create:
recipe_type = self._recipe_type_map[(recipe['recipe_type']['name'], recipe['recipe_type']['version'])]
file_input_name = recipe['file_input_name']
recipe_data = RecipeData({})
recipe_data.add_file_input(file_input_name, source_file.id)
# If workspace name has been provided, add that to the recipe data for each output file
if 'workspace_name' in recipe:
workspace = self._workspace_map[recipe['workspace_name']]
recipe_data.set_workspace_id(workspace.id)
logger.info('Queuing new recipe of type %s %s', recipe_type.name, recipe_type.version)
Queue.objects.queue_new_recipe(recipe_type, recipe_data.get_dict(), event)
def process_ingest(self, ingest, source_file_id):
"""Processes the given source file ingest by creating the appropriate jobs if the rule is triggered. All
database changes are made in an atomic transaction.
:param ingest: The ingest to process
:type ingest: :class:`ingest.models.Ingest`
:param source_file_id: The ID of the source file that was ingested
:type source_file_id: long
"""
# Only trigger when this ingest file has the correct media type and ingest types
if self._media_type and self._media_type != ingest.media_type:
return
if not self._data_types.issubset(ingest.get_data_type_tags()):
return
if not self._media_type:
logger.info("Ingest rule for all media types was triggered")
else:
logger.info("Ingest rule for media type %s was triggered", self._media_type)
event = IngestTriggerEvent(self._model, ingest).save_to_db()
# Create triggered jobs
for job in self._jobs_to_create:
job_type = self._job_type_map[(job["job_type"]["name"], job["job_type"]["version"])]
file_input_name = job["file_input_name"]
job_data = JobData({})
job_data.add_file_input(file_input_name, source_file_id)
# If workspace name has been provided, add that to the job data for each output file
if "workspace_name" in job:
workspace = self._workspace_map[job["workspace_name"]]
job_type.get_job_interface().add_workspace_to_data(job_data, workspace.id)
logger.info("Queuing new job of type %s %s", job_type.name, job_type.version)
Queue.objects.queue_new_job(job_type, job_data.get_dict(), event)
# Create triggered recipes
for recipe in self._recipes_to_create:
recipe_type = self._recipe_type_map[(recipe["recipe_type"]["name"], recipe["recipe_type"]["version"])]
file_input_name = recipe["file_input_name"]
recipe_data = RecipeData({})
recipe_data.add_file_input(file_input_name, source_file_id)
# If workspace name has been provided, add that to the recipe data for each output file
if "workspace_name" in recipe:
workspace = self._workspace_map[recipe["workspace_name"]]
recipe_data.set_workspace_id(workspace.id)
logger.info("Queuing new recipe of type %s %s", recipe_type.name, recipe_type.version)
Queue.objects.queue_new_recipe(recipe_type, recipe_data.get_dict(), event)
def test_missing_value(self):
"""Tests calling RecipeData.validate_properties() when a property is missing a value"""
data = {'input_data': [{'name': 'Param1'}]}
properties = {'Param1': False}
self.assertRaises(InvalidRecipeData, RecipeData(data).validate_properties, properties)
def test_schedule_date_range_data_full(self):
"""Tests calling BatchManager.schedule_recipes() for a batch with a data date range restriction"""
file1 = storage_test_utils.create_file()
file1.data_started = datetime.datetime(2016, 1, 1, tzinfo=utc)
file1.save()
data1 = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input',
'file_id': file1.id,
}],
'workspace_id': self.workspace.id,
}
Recipe.objects.create_recipe(recipe_type=self.recipe_type,
data=RecipeData(data1),
event=self.event)
file2 = storage_test_utils.create_file()
file2.data_started = datetime.datetime(2016, 2, 1, tzinfo=utc)
file2.data_ended = datetime.datetime(2016, 2, 10, tzinfo=utc)
file2.save()
data2 = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input',
'file_id': file2.id,
}],
'workspace_id': self.workspace.id,
}
recipe2 = Recipe.objects.create_recipe(recipe_type=self.recipe_type,
data=RecipeData(data2),
event=self.event).recipe
file3 = storage_test_utils.create_file()
file3.data_ended = datetime.datetime(2016, 3, 1, tzinfo=utc)
file3.save()
data3 = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input',
'file_id': file3.id,
}],
'workspace_id': self.workspace.id,
}
Recipe.objects.create_recipe(recipe_type=self.recipe_type,
data=RecipeData(data3),
event=self.event)
recipe_test_utils.edit_recipe_type(self.recipe_type, self.definition_2)
definition = {
'date_range': {
'type': 'data',
'started': '2016-02-01T00:00:00.000Z',
'ended': '2016-02-10T00:00:00.000Z',
},
}
batch = batch_test_utils.create_batch(recipe_type=self.recipe_type,
definition=definition)
Batch.objects.schedule_recipes(batch.id)
batch = Batch.objects.get(pk=batch.id)
self.assertEqual(batch.status, 'CREATED')
self.assertEqual(batch.created_count, 1)
self.assertEqual(batch.total_count, 1)
batch_recipes = BatchRecipe.objects.all()
self.assertEqual(len(batch_recipes), 1)
self.assertEqual(batch_recipes[0].superseded_recipe, recipe2)
def test_successful_job_1_completed(self, mock_store):
'''Tests calling RecipeDefinition.get_next_jobs_to_queue() successfully when job 1 has been completed.'''
definition = {
'version':
'1.0',
'input_data': [{
'name': 'Recipe Input',
'type': 'file',
'media_types': ['text/plain'],
}],
'jobs': [{
'name':
'Job 1',
'job_type': {
'name': self.job_type_1.name,
'version': self.job_type_1.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input',
'job_input': self.input_name_1,
}]
}, {
'name':
'Job 2',
'job_type': {
'name': self.job_type_2.name,
'version': self.job_type_2.version,
},
'dependencies': [{
'name':
'Job 1',
'connections': [{
'output': self.output_name_1,
'input': self.input_name_2,
}],
}],
}],
}
recipe_definition = RecipeDefinition(definition)
recipe_definition.validate_job_interfaces()
data = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input',
'file_id': self.file_1.id,
}],
'workspace_id': 1,
}
recipe_data = RecipeData(data)
recipe_definition.validate_data(recipe_data)
png_file_ids = [98, 99, 100]
job_results = JobResults()
job_results.add_file_list_parameter(self.output_name_1, png_file_ids)
job_1 = Job.objects.select_related('job_type').get(pk=self.job_1.id)
job_1.results = job_results.get_dict()
job_1.save()
job_2 = Job.objects.select_related('job_type').get(pk=self.job_2.id)
results = recipe_definition.get_next_jobs_to_queue(
recipe_data, {'Job 2': job_2}, {'Job 1': job_1})
# Make sure only Job 2 is returned and that its job data is correct
self.assertListEqual([self.job_2.id], results.keys())
self.assertDictEqual(
results[self.job_2.id].get_dict(), {
'version':
'1.0',
'input_data': [{
'name': self.input_name_2,
'file_ids': png_file_ids,
}],
'output_data': [{
'name': self.output_name_2,
'workspace_id': 1,
}],
})
def test_workspace_not_active(self, mock_store):
"""Tests calling RecipeData.validate_workspace() with a workspace that is not active"""
data = {'workspace_id': 3}
self.assertRaises(InvalidRecipeData, RecipeData(data).validate_workspace)
def test_init_blank(self):
"""Tests calling RecipeData constructor with blank JSON."""
# No exception is success
RecipeData({})
def test_missing_workspace_id(self, mock_store):
"""Tests calling RecipeData.validate_workspace() when missing the workspace_id field"""
data = {}
self.assertRaises(InvalidRecipeData, RecipeData(data).validate_workspace)
def test_workspace_not_exist(self, mock_store):
"""Tests calling RecipeData.validate_workspace() with a workspace that does not exist"""
data = {'workspace_id': 2}
self.assertRaises(InvalidRecipeData, RecipeData(data).validate_workspace)
def test_missing_required(self):
"""Tests calling RecipeData.validate_input_files() when a file is required, but missing"""
data = {'input_data': []}
files = {'File1': (True, True, ScaleFileDescription())}
self.assertRaises(InvalidRecipeData, RecipeData(data).validate_input_files, files)
def test_missing_required(self):
"""Tests calling RecipeData.validate_properties() when a property is required, but missing"""
data = {'input_data': []}
properties = {'Param1': True}
self.assertRaises(InvalidRecipeData, RecipeData(data).validate_properties, properties)
def setUp(self):
django.setup()
workspace = storage_test_utils.create_workspace()
source_file = source_test_utils.create_source(workspace=workspace)
self.event = trigger_test_utils.create_trigger_event()
interface_1 = {
'version': '1.0',
'command': 'test_command',
'command_arguments': 'test_arg',
'input_data': [{
'name': 'Test Input 1',
'type': 'file',
'media_types': ['text/plain'],
}],
'output_data': [{
'name': 'Test Output 1',
'type': 'files',
'media_type': 'image/png',
}]
}
self.job_type_1 = job_test_utils.create_job_type(interface=interface_1)
interface_2 = {
'version': '1.0',
'command': 'test_command',
'command_arguments': 'test_arg',
'input_data': [{
'name': 'Test Input 2',
'type': 'files',
'media_types': ['image/png', 'image/tiff'],
}],
'output_data': [{
'name': 'Test Output 2',
'type': 'file',
}]
}
self.job_type_2 = job_test_utils.create_job_type(interface=interface_2)
definition = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input',
'type': 'file',
'media_types': ['text/plain'],
}],
'jobs': [{
'name': 'Job 1',
'job_type': {
'name': self.job_type_1.name,
'version': self.job_type_1.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input',
'job_input': 'Test Input 1',
}]
}, {
'name': 'Job 2',
'job_type': {
'name': self.job_type_2.name,
'version': self.job_type_2.version,
},
'dependencies': [{
'name': 'Job 1',
'connections': [{
'output': 'Test Output 1',
'input': 'Test Input 2',
}]
}]
}]
}
recipe_definition = RecipeDefinition(definition)
recipe_definition.validate_job_interfaces()
self.recipe_type = recipe_test_utils.create_recipe_type(definition=definition)
data = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input',
'file_id': source_file.id,
}],
'workspace_id': workspace.id,
}
self.data = RecipeData(data)
# Register a fake processor
self.mock_processor = MagicMock(QueueEventProcessor)
Queue.objects.register_processor(lambda: self.mock_processor)
def test_bad_file_id(self):
"""Tests calling RecipeData.validate_input_files() with a file that has an invalid ID"""
data = {'input_data': [{'name': 'File1', 'file_id': 999999}]}
files = {'File1': (True, False, ScaleFileDescription())}
self.assertRaises(InvalidRecipeData, RecipeData(data).validate_input_files, files)
def test_single_non_integral(self):
"""Tests calling RecipeData.validate_input_files() with a single file param with a non-integral for file_id field"""
data = {'input_data': [{'name': 'File1', 'file_id': 'STRING'}]}
files = {'File1': (True, False, ScaleFileDescription())}
self.assertRaises(InvalidRecipeData, RecipeData(data).validate_input_files, files)
def test_single_missing_file_id(self):
"""Tests calling RecipeData.validate_input_files() with a single file param missing the file_id field"""
data = {'input_data': [{'name': 'File1'}]}
files = {'File1': (True, False, ScaleFileDescription())}
self.assertRaises(InvalidRecipeData, RecipeData(data).validate_input_files, files)
def test_multiple_non_integrals(self):
"""Tests calling RecipeData.validate_input_files() with a multiple file param with a list of non-integrals for file_ids field"""
data = {'input_data': [{'name': 'File1', 'file_ids': [123, 'STRING']}]}
files = {'File1': (True, True, ScaleFileDescription())}
self.assertRaises(InvalidRecipeData, RecipeData(data).validate_input_files, files)
def test_value_not_string(self):
"""Tests calling RecipeData.validate_properties() when a property has a non-string value"""
data = {'input_data': [{'name': 'Param1', 'value': 123}]}
properties = {'Param1': False}
self.assertRaises(InvalidRecipeData, RecipeData(data).validate_properties, properties)
def test_successful_supersede_different_recipe_type(self):
"""Tests calling RecipeManager.create_recipe() to supersede a recipe with a different recipe type version that
has one identical node, and deletes another node to replace it with a new one.
"""
interface_3 = {
'version':
'1.0',
'command':
'my_command',
'command_arguments':
'args',
'input_data': [{
'name': 'Test Input 3',
'type': 'files',
'media_types': ['image/tiff'],
}],
'output_data': [{
'name': 'Test Output 3',
'type': 'file',
}]
}
job_type_3 = job_test_utils.create_job_type(interface=interface_3)
new_definition = {
'version':
'1.0',
'input_data': [{
'name': 'Recipe Input',
'type': 'file',
'media_types': ['text/plain'],
}],
'jobs': [{
'name':
'Job 1',
'job_type': {
'name': self.job_type_1.name,
'version': self.job_type_1.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input',
'job_input': 'Test Input 1',
}]
}, {
'name':
'Job 3',
'job_type': {
'name': job_type_3.name,
'version': job_type_3.version,
},
'dependencies': [{
'name':
'Job 1',
'connections': [{
'output': 'Test Output 1',
'input': 'Test Input 3',
}]
}]
}]
}
new_recipe_type = recipe_test_utils.create_recipe_type(
name=self.recipe_type.name, definition=new_definition)
event = trigger_test_utils.create_trigger_event()
handler = Recipe.objects.create_recipe(recipe_type=self.recipe_type,
event=event,
data=RecipeData(self.data))
recipe = Recipe.objects.get(id=handler.recipe.id)
recipe_job_1 = RecipeJob.objects.select_related('job').get(
recipe_id=handler.recipe.id, job_name='Job 1')
recipe_job_2 = RecipeJob.objects.select_related('job').get(
recipe_id=handler.recipe.id, job_name='Job 2')
job_exe_2 = job_test_utils.create_job_exe(job=recipe_job_2.job)
try:
from product.models import ProductFile
from product.test import utils as product_test_utils
product = product_test_utils.create_product(
job_exe=job_exe_2, has_been_published=True, is_published=True)
except ImportError:
product = None
superseded_jobs = {
'Job 1': recipe_job_1.job,
'Job 2': recipe_job_2.job
}
# Create a new recipe with a different version
graph_a = self.recipe_type.get_recipe_definition().get_graph()
graph_b = new_recipe_type.get_recipe_definition().get_graph()
delta = RecipeGraphDelta(graph_a, graph_b)
new_handler = Recipe.objects.create_recipe(
recipe_type=new_recipe_type,
event=event,
data=None,
superseded_recipe=recipe,
delta=delta,
superseded_jobs=superseded_jobs)
# Check that old recipe and job 2 are superseded, job 1 should be copied (not superseded)
recipe = Recipe.objects.get(id=recipe.id)
job_1 = Job.objects.get(id=recipe_job_1.job_id)
job_2 = Job.objects.get(id=recipe_job_2.job_id)
self.assertTrue(recipe.is_superseded)
self.assertFalse(job_1.is_superseded)
self.assertTrue(job_2.is_superseded)
# Check that product of job 2 (which was superseded with no new job) was unpublished
if product:
product = ProductFile.objects.get(id=product.id)
self.assertFalse(product.is_published)
self.assertIsNotNone(product.unpublished)
# Check that new recipe supersedes the old one, job 1 is copied from old recipe, and job 2 is new and does not
# supersede anything
new_recipe = Recipe.objects.get(id=new_handler.recipe.id)
new_recipe_job_1 = RecipeJob.objects.select_related('job').get(
recipe_id=new_handler.recipe.id, job_name='Job 1')
new_recipe_job_2 = RecipeJob.objects.select_related('job').get(
recipe_id=new_handler.recipe.id, job_name='Job 3')
self.assertEqual(new_recipe.superseded_recipe_id, recipe.id)
self.assertEqual(new_recipe.root_superseded_recipe_id, recipe.id)
self.assertDictEqual(new_recipe.data, recipe.data)
self.assertEqual(new_recipe_job_1.job.id, job_1.id)
self.assertFalse(new_recipe_job_1.is_original)
self.assertIsNone(new_recipe_job_1.job.superseded_job)
self.assertIsNone(new_recipe_job_1.job.root_superseded_job)
self.assertNotEqual(new_recipe_job_2.job.id, job_2.id)
self.assertTrue(new_recipe_job_2.is_original)
self.assertIsNone(new_recipe_job_2.job.superseded_job_id)
self.assertIsNone(new_recipe_job_2.job.root_superseded_job_id)
