def test_schedule_date_range_data_ended(self):
"""Tests calling BatchManager.schedule_recipes() for a batch with a data ended date range restriction"""
file1 = storage_test_utils.create_file()
file1.data_started = datetime.datetime(2016, 1, 1, tzinfo=utc)
file1.data_ended = datetime.datetime(2016, 1, 10, tzinfo=utc)
file1.save()
data1 = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input',
'file_id': file1.id,
}],
'workspace_id': self.workspace.id,
}
recipe1 = Recipe.objects.create_recipe(recipe_type=self.recipe_type,
data=RecipeData(data1),
event=self.event).recipe
file2 = storage_test_utils.create_file()
file2.data_started = datetime.datetime(2016, 2, 1, tzinfo=utc)
file2.data_ended = datetime.datetime(2016, 2, 10, tzinfo=utc)
file2.save()
data2 = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input',
'file_id': file2.id,
}],
'workspace_id': self.workspace.id,
}
Recipe.objects.create_recipe(recipe_type=self.recipe_type,
data=RecipeData(data2),
event=self.event)
recipe_test_utils.edit_recipe_type(self.recipe_type, self.definition_2)
definition = {
'date_range': {
'type': 'data',
'ended': '2016-01-15T00:00:00.000Z',
},
}
batch = batch_test_utils.create_batch(recipe_type=self.recipe_type,
definition=definition)
Batch.objects.schedule_recipes(batch.id)
batch = Batch.objects.get(pk=batch.id)
self.assertEqual(batch.status, 'CREATED')
self.assertEqual(batch.created_count, 1)
self.assertEqual(batch.total_count, 1)
batch_recipes = BatchRecipe.objects.all()
self.assertEqual(len(batch_recipes), 1)
self.assertEqual(batch_recipes[0].superseded_recipe, recipe1)
def create_recipe_handler(recipe_type=None,
data=None,
event=None,
superseded_recipe=None,
delta=None,
superseded_jobs=None):
"""Creates a recipe along with its declared jobs for unit testing
:returns: The recipe handler with created recipe and jobs
:rtype: :class:`recipe.handlers.handler.RecipeHandler`
"""
if not recipe_type:
recipe_type = create_recipe_type()
if not data:
data = {}
if not isinstance(data, RecipeData):
data = RecipeData(data)
if not event:
event = trigger_test_utils.create_trigger_event()
if superseded_recipe and not delta:
delta = RecipeGraphDelta(RecipeGraph(), RecipeGraph())
return Recipe.objects.create_recipe(recipe_type,
data,
event,
superseded_recipe=superseded_recipe,
delta=delta,
superseded_jobs=superseded_jobs)
def post(self, request):
"""Queue a recipe and returns the new job information in JSON form
:param request: the HTTP POST request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
"""
recipe_type_id = rest_util.parse_int(request, 'recipe_type_id')
recipe_data = rest_util.parse_dict(request, 'recipe_data', {})
try:
recipe_type = RecipeType.objects.get(pk=recipe_type_id)
except RecipeType.DoesNotExist:
raise Http404
try:
recipe_id = Queue.objects.queue_new_recipe_for_user(
recipe_type, RecipeData(recipe_data))
except InvalidRecipeData:
return Response('Invalid recipe information.',
status=status.HTTP_400_BAD_REQUEST)
recipe_details = Recipe.objects.get_details(recipe_id)
serializer = self.get_serializer(recipe_details)
recipe_url = urlresolvers.reverse('recipe_details_view',
args=[recipe_id])
return Response(serializer.data,
status=status.HTTP_201_CREATED,
headers=dict(location=recipe_url))
def test_missing_required(self):
'''Tests calling RecipeData.validate_properties() when a property is required, but missing'''
data = {u'input_data': []}
properties = {u'Param1': True}
self.assertRaises(InvalidRecipeData,
RecipeData(data).validate_properties, properties)
def test_missing_value(self):
'''Tests calling RecipeData.validate_properties() when a property is missing a value'''
data = {u'input_data': [{u'name': u'Param1'}]}
properties = {u'Param1': False}
self.assertRaises(InvalidRecipeData,
RecipeData(data).validate_properties, properties)
def test_single_non_integral(self):
'''Tests calling RecipeData.validate_input_files() with a single file param with a non-integral for file_id field'''
data = {u'input_data': [{u'name': u'File1', u'file_id': 'STRING'}]}
files = {u'File1': (True, False, ScaleFileDescription())}
self.assertRaises(InvalidRecipeData,
RecipeData(data).validate_input_files, files)
def test_bad_file_id(self):
'''Tests calling RecipeData.validate_input_files() with a file that has an invalid ID'''
data = {u'input_data': [{u'name': u'File1', u'file_id': 999999}]}
files = {u'File1': (True, False, ScaleFileDescription())}
self.assertRaises(InvalidRecipeData,
RecipeData(data).validate_input_files, files)
def test_single_missing_file_id(self):
'''Tests calling RecipeData.validate_input_files() with a single file param missing the file_id field'''
data = {u'input_data': [{u'name': u'File1'}]}
files = {u'File1': (True, False, ScaleFileDescription())}
self.assertRaises(InvalidRecipeData,
RecipeData(data).validate_input_files, files)
def test_multiple_non_list(self):
'''Tests calling RecipeData.validate_input_files() with a multiple file param with a non-list for file_ids field'''
data = {u'input_data': [{u'name': u'File1', u'file_ids': u'STRING'}]}
files = {u'File1': (True, True, ScaleFileDescription())}
self.assertRaises(InvalidRecipeData,
RecipeData(data).validate_input_files, files)
def test_missing_required(self):
'''Tests calling RecipeData.validate_input_files() when a file is required, but missing'''
data = {u'input_data': []}
files = {u'File1': (True, True, ScaleFileDescription())}
self.assertRaises(InvalidRecipeData,
RecipeData(data).validate_input_files, files)
def test_schedule_job_names(self):
"""Tests calling BatchManager.schedule_recipes() for a batch that forces all jobs to be re-processed"""
handler = Recipe.objects.create_recipe(recipe_type=self.recipe_type,
data=RecipeData(self.data),
event=self.event)
recipe_test_utils.edit_recipe_type(self.recipe_type, self.definition_2)
definition = {
'job_names': ['Job 1'],
}
batch = batch_test_utils.create_batch(recipe_type=self.recipe_type,
definition=definition)
Batch.objects.schedule_recipes(batch.id)
batch = Batch.objects.get(pk=batch.id)
self.assertEqual(batch.status, 'CREATED')
self.assertEqual(batch.created_count, 1)
self.assertEqual(batch.total_count, 1)
batch_recipes = BatchRecipe.objects.all()
self.assertEqual(len(batch_recipes), 1)
self.assertEqual(batch_recipes[0].batch, batch)
self.assertEqual(batch_recipes[0].recipe.recipe_type, self.recipe_type)
self.assertEqual(batch_recipes[0].superseded_recipe, handler.recipe)
batch_jobs = BatchJob.objects.all()
self.assertEqual(len(batch_jobs), 2)
for batch_job in batch_jobs:
self.assertIn(batch_job.job.job_type,
[self.job_type_1, self.job_type_2])
def test_value_not_string(self):
'''Tests calling RecipeData.validate_properties() when a property has a non-string value'''
data = {u'input_data': [{u'name': u'Param1', u'value': 123}]}
properties = {u'Param1': False}
self.assertRaises(InvalidRecipeData,
RecipeData(data).validate_properties, properties)
def test_init_successful_one_property(self):
"""Tests calling RecipeData constructor successfully with a single property input."""
data = {'input_data': [{'name': 'My Name', 'value': '1'}]}
# No exception is success
RecipeData(data)
def test_successful(self, mock_store):
"""Tests calling RecipeData.validate_workspace() with successful data"""
data = {'workspace_id': 1}
# No exception is success
warnings = RecipeData(data).validate_workspace()
self.assertFalse(warnings)
def test_init_successful_one_property(self):
'''Tests calling RecipeData constructor successfully with a single property input.'''
data = {u'input_data': [{u'name': u'My Name', u'value': u'1'}]}
# No exception is success
RecipeData(data)
def test_not_required(self):
"""Tests calling RecipeData.validate_input_files() when a file is missing, but required"""
data = {'input_data': []}
files = {'File1': (False, True, ScaleFileDescription())}
# No exception is success
warnings = RecipeData(data).validate_input_files(files)
self.assertFalse(warnings)
def test_multiple_given_single(self):
"""Tests calling RecipeData.validate_input_files() with a multiple file param that is provided with a single file ID"""
data = {'input_data': [{'name': 'File1', 'file_id': self.file_1.id}]}
files = {'File1': (True, True, ScaleFileDescription())}
# No exception is success
warnings = RecipeData(data).validate_input_files(files)
self.assertFalse(warnings)
def get_recipe_data(self):
"""Returns the data for this recipe
:returns: The data for this recipe
:rtype: :class:`recipe.configuration.data.recipe_data.RecipeData`
"""
return RecipeData(self.data)
def test_required_successful(self):
"""Tests calling RecipeData.validate_properties() successfully with a required property"""
data = {'input_data': [{'name': 'Param1', 'value': 'Value1'}]}
properties = {'Param1': True}
# No exception is success
warnings = RecipeData(data).validate_properties(properties)
self.assertFalse(warnings)
def test_not_required(self):
"""Tests calling RecipeData.validate_properties() when a property is missing, but is not required"""
data = {'input_data': []}
properties = {'Param1': False}
# No exception is success
warnings = RecipeData(data).validate_properties(properties)
self.assertFalse(warnings)
def populate_recipe_file(apps, schema_editor):
# Go through all of the recipe models and create a recipe file model for each of the recipe input files
Recipe = apps.get_model('recipe', 'Recipe')
RecipeFile = apps.get_model('recipe', 'RecipeFile')
ScaleFile = apps.get_model('storage', 'ScaleFile')
total_count = Recipe.objects.all().count()
if not total_count:
return
print('\nCreating new recipe_file table rows: %i' % total_count)
done_count = 0
fail_count = 0
batch_size = 500
while done_count < total_count:
batch_end = done_count + batch_size
# Build a unique list of all valid input file ids
batch_file_ids = set()
recipes = Recipe.objects.all().order_by('id')[done_count:batch_end]
for recipe in recipes:
batch_file_ids.update(RecipeData(recipe.data).get_input_file_ids())
valid_file_ids = {scale_file.id for scale_file in ScaleFile.objects.filter(pk__in=batch_file_ids)}
# Create a model for each recipe input file
recipe_files = []
for recipe in recipes:
input_file_ids = RecipeData(recipe.data).get_input_file_ids()
for input_file_id in input_file_ids:
if input_file_id in valid_file_ids:
recipe_file = RecipeFile()
recipe_file.recipe_id = recipe.id
recipe_file.scale_file_id = input_file_id
recipe_file.created = recipe.created
recipe_files.append(recipe_file)
else:
fail_count += 1
print('Failed recipe: %i -> file: %i' % (recipe.id, input_file_id))
RecipeFile.objects.bulk_create(recipe_files)
done_count += len(recipes)
percent = (float(done_count) / float(total_count)) * 100.00
print('Progress: %i/%i (%.2f%%)' % (done_count, total_count, percent))
print ('Migration finished. Failed: %i' % fail_count)
def test_bad_media_type(self):
"""Tests calling RecipeData.validate_input_files() with a file that has an invalid media type"""
data = {'input_data': [{'name': 'File1', 'file_id': self.file_1.id}]}
file_desc_1 = ScaleFileDescription()
file_desc_1.add_allowed_media_type('text/plain')
files = {'File1': (True, False, file_desc_1)}
warnings = RecipeData(data).validate_input_files(files)
self.assertTrue(warnings)
def test_successful_supersede_same_recipe_type(self):
"""Tests calling RecipeManager.create_recipe() to supersede a recipe with the same recipe type."""
event = trigger_test_utils.create_trigger_event()
handler = Recipe.objects.create_recipe(recipe_type=self.recipe_type,
event=event,
data=RecipeData(self.data))
recipe = Recipe.objects.get(id=handler.recipe.id)
recipe_job_1 = RecipeJob.objects.select_related('job').get(
recipe_id=handler.recipe.id, job_name='Job 1')
recipe_job_2 = RecipeJob.objects.select_related('job').get(
recipe_id=handler.recipe.id, job_name='Job 2')
superseded_jobs = {
'Job 1': recipe_job_1.job,
'Job 2': recipe_job_2.job
}
# Create a new recipe of the same type where we want to reprocess Job 2
graph = self.recipe_type.get_recipe_definition().get_graph()
delta = RecipeGraphDelta(graph, graph)
delta.reprocess_identical_node('Job 2') # We want to reprocess Job 2
new_handler = Recipe.objects.create_recipe(
recipe_type=self.recipe_type,
event=event,
data=None,
superseded_recipe=recipe,
delta=delta,
superseded_jobs=superseded_jobs)
# Check that old recipe and job 2 are superseded, job 1 should be copied (not superseded)
recipe = Recipe.objects.get(id=recipe.id)
job_1 = Job.objects.get(id=recipe_job_1.job_id)
job_2 = Job.objects.get(id=recipe_job_2.job_id)
self.assertTrue(recipe.is_superseded)
self.assertFalse(job_1.is_superseded)
self.assertTrue(job_2.is_superseded)
# Check that new recipe supersedes the old one, job 1 is copied from old recipe, and job 2 supersedes old job 2
new_recipe = Recipe.objects.get(id=new_handler.recipe.id)
new_recipe_job_1 = RecipeJob.objects.select_related('job').get(
recipe_id=new_handler.recipe.id, job_name='Job 1')
new_recipe_job_2 = RecipeJob.objects.select_related('job').get(
recipe_id=new_handler.recipe.id, job_name='Job 2')
self.assertEqual(new_recipe.superseded_recipe_id, recipe.id)
self.assertEqual(new_recipe.root_superseded_recipe_id, recipe.id)
self.assertDictEqual(new_recipe.data, recipe.data)
self.assertEqual(new_recipe_job_1.job.id, job_1.id)
self.assertFalse(new_recipe_job_1.is_original)
self.assertIsNone(new_recipe_job_1.job.superseded_job)
self.assertIsNone(new_recipe_job_1.job.root_superseded_job)
self.assertNotEqual(new_recipe_job_2.job.id, job_2.id)
self.assertTrue(new_recipe_job_2.is_original)
self.assertEqual(new_recipe_job_2.job.superseded_job_id, job_2.id)
self.assertEqual(new_recipe_job_2.job.root_superseded_job_id, job_2.id)
def test_successful(self, mock_store):
'''Tests calling RecipeDefinition.validate_data() successfully.'''
definition = {
'version':
'1.0',
'input_data': [{
'name': 'Recipe Input',
'type': 'file',
'media_types': ['text/plain'],
}],
'jobs': [{
'name':
'Job 1',
'job_type': {
'name': self.job_type_1.name,
'version': self.job_type_1.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input',
'job_input': self.input_name_1,
}],
}, {
'name':
'Job 2',
'job_type': {
'name': self.job_type_2.name,
'version': self.job_type_2.version,
},
'dependencies': [{
'name':
'Job 1',
'connections': [{
'output': self.output_name_1,
'input': self.input_name_2,
}],
}],
}],
}
recipe = RecipeDefinition(definition)
recipe.validate_job_interfaces()
data = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input',
'file_id': self.file_1.id,
}],
'workspace_id': 1,
}
recipe_data = RecipeData(data)
# No exception is success
recipe.validate_data(recipe_data)
def test_schedule_invalid_status(self):
"""Tests calling BatchManager.schedule_recipes() for a batch that was already created"""
Recipe.objects.create_recipe(recipe_type=self.recipe_type,
data=RecipeData(self.data),
event=self.event)
batch = batch_test_utils.create_batch(recipe_type=self.recipe_type)
Batch.objects.schedule_recipes(batch.id)
self.assertRaises(BatchError, Batch.objects.schedule_recipes, batch.id)
def process_ingested_source_file(self, source_file, when):
"""Processes the given ingested source file by checking it against all ingest trigger rules and creating the
corresponding jobs and recipes for any triggered rules. All database changes are made in an atomic transaction.
:param source_file: The source file that was ingested
:type source_file: :class:`source.models.SourceFile`
:param when: When the source file was ingested
:type when: :class:`datetime.datetime`
"""
msg = 'Processing trigger rules for ingested source file with media type %s and data types %s'
logger.info(msg, source_file.media_type,
str(list(source_file.get_data_type_tags())))
any_rules = False
for entry in RecipeType.objects.get_active_trigger_rules(INGEST_TYPE):
rule = entry[0]
thing_to_create = entry[1]
rule_config = rule.get_configuration()
condition = rule_config.get_condition()
if condition.is_condition_met(source_file):
logger.info(condition.get_triggered_message())
any_rules = True
event = self._create_ingest_trigger_event(
source_file, rule, when)
workspace = Workspace.objects.get(
name=rule_config.get_workspace_name())
if isinstance(thing_to_create, JobType):
job_type = thing_to_create
job_data = JobData({})
job_data.add_file_input(rule_config.get_input_data_name(),
source_file.id)
job_type.get_job_interface().add_workspace_to_data(
job_data, workspace.id)
logger.info('Queuing new job of type %s %s', job_type.name,
job_type.version)
Queue.objects.queue_new_job(job_type, job_data, event)
elif isinstance(thing_to_create, RecipeType):
recipe_type = thing_to_create
recipe_data = RecipeData({})
recipe_data.add_file_input(
rule_config.get_input_data_name(), source_file.id)
recipe_data.set_workspace_id(workspace.id)
logger.info('Queuing new recipe of type %s %s',
recipe_type.name, recipe_type.version)
Queue.objects.queue_new_recipe(recipe_type, recipe_data,
event)
if not any_rules:
logger.info('No rules triggered')
def test_successful_property(self):
"""Tests calling RecipeData.add_input_to_data() successfully with a property parameter"""
recipe_input_name = 'foo'
value = 'Doctor Who?'
job_input_name = 'bar'
recipe_data = RecipeData({'input_data': [{'name': recipe_input_name, 'value': value}]})
job_data = MagicMock()
recipe_data.add_input_to_data(recipe_input_name, job_data, job_input_name)
job_data.add_property_input.assert_called_with(job_input_name, value)
def test_successful_file_list(self):
"""Tests calling RecipeData.add_input_to_data() successfully with a file list parameter"""
recipe_input_name = 'foo'
file_ids = [1, 2, 3, 4]
job_input_name = 'bar'
recipe_data = RecipeData({'input_data': [{'name': recipe_input_name, 'file_ids': file_ids}]})
job_data = MagicMock()
recipe_data.add_input_to_data(recipe_input_name, job_data, job_input_name)
job_data.add_file_list_input.assert_called_with(job_input_name, file_ids)
def test_missing_workspace(self, mock_store):
'''Tests calling RecipeDefinition.validate_data() with a missing required workspace.'''
definition = {
'version':
'1.0',
'input_data': [{
'name': 'Recipe Input',
'type': 'file',
'media_types': ['text/plain'],
}],
'jobs': [{
'name':
'Job 1',
'job_type': {
'name': self.job_type_1.name,
'version': self.job_type_1.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input',
'job_input': self.input_name_1,
}],
}, {
'name':
'Job 2',
'job_type': {
'name': self.job_type_2.name,
'version': self.job_type_2.version,
},
'dependencies': [{
'name':
'Job 1',
'connections': [{
'output': self.output_name_1,
'input': self.input_name_2,
}],
}],
}],
}
recipe = RecipeDefinition(definition)
recipe.validate_job_interfaces()
data = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input',
'file_id': self.file_1.id,
}],
}
recipe_data = RecipeData(data)
self.assertRaises(InvalidRecipeData, recipe.validate_data, recipe_data)
def test_schedule_date_range_created(self):
"""Tests calling BatchManager.schedule_recipes() for a batch with a created date range restriction"""
recipe1 = Recipe.objects.create_recipe(recipe_type=self.recipe_type,
data=RecipeData(self.data),
event=self.event).recipe
Recipe.objects.filter(pk=recipe1.id).update(
created=datetime.datetime(2016, 1, 1, tzinfo=utc))
recipe2 = Recipe.objects.create_recipe(recipe_type=self.recipe_type,
data=RecipeData(self.data),
event=self.event).recipe
Recipe.objects.filter(pk=recipe2.id).update(
created=datetime.datetime(2016, 2, 1, tzinfo=utc))
recipe3 = Recipe.objects.create_recipe(recipe_type=self.recipe_type,
data=RecipeData(self.data),
event=self.event).recipe
Recipe.objects.filter(pk=recipe3.id).update(
created=datetime.datetime(2016, 3, 1, tzinfo=utc))
recipe_test_utils.edit_recipe_type(self.recipe_type, self.definition_2)
definition = {
'date_range': {
'started': '2016-01-10T00:00:00.000Z',
'ended': '2016-02-10T00:00:00.000Z',
},
}
batch = batch_test_utils.create_batch(recipe_type=self.recipe_type,
definition=definition)
Batch.objects.schedule_recipes(batch.id)
batch = Batch.objects.get(pk=batch.id)
self.assertEqual(batch.status, 'CREATED')
self.assertEqual(batch.created_count, 1)
self.assertEqual(batch.total_count, 1)
batch_recipes = BatchRecipe.objects.all()
self.assertEqual(len(batch_recipes), 1)
self.assertEqual(batch_recipes[0].superseded_recipe, recipe2)