def test_schedule_partial_batch(self, mock_msg_mgr):
"""Tests calling BatchManager.schedule_recipes() for a batch that is incomplete"""
for i in range(5):
Recipe.objects.create_recipe_old(recipe_type=self.recipe_type,
input=LegacyRecipeData(self.data),
event=self.event)
partials = []
for i in range(5):
handler = Recipe.objects.create_recipe_old(
recipe_type=self.recipe_type,
input=LegacyRecipeData(self.data),
event=self.event)
handler.recipe.is_superseded = True
handler.recipe.save()
partials.append(handler.recipe)
recipe_test_utils.edit_recipe_type(self.recipe_type, self.definition_2)
batch = batch_test_utils.create_batch_old(recipe_type=self.recipe_type)
Batch.objects.schedule_recipes(batch.id)
batch = Batch.objects.get(pk=batch.id)
self.assertEqual(batch.created_count, 5)
self.assertEqual(batch.total_count, 5)
for recipe in partials:
recipe.is_superseded = False
recipe.save()
batch.status = 'SUBMITTED'
batch.save()
Batch.objects.schedule_recipes(batch.id)
batch = Batch.objects.get(pk=batch.id)
self.assertEqual(batch.status, 'CREATED')
def populate_recipe_file(apps, schema_editor):
# Go through all of the recipe models and create a recipe file model for each of the recipe input files
Recipe = apps.get_model('recipe', 'Recipe')
RecipeFile = apps.get_model('recipe', 'RecipeFile')
ScaleFile = apps.get_model('storage', 'ScaleFile')
total_count = Recipe.objects.all().count()
if not total_count:
return
print('\nCreating new recipe_file table rows: %i' % total_count)
done_count = 0
fail_count = 0
batch_size = 500
while done_count < total_count:
batch_end = done_count + batch_size
# Build a unique list of all valid input file ids
batch_file_ids = set()
recipes = Recipe.objects.all().order_by('id')[done_count:batch_end]
for recipe in recipes:
batch_file_ids.update(
LegacyRecipeData(recipe.data).get_input_file_ids())
valid_file_ids = {
scale_file.id
for scale_file in ScaleFile.objects.filter(
pk__in=batch_file_ids)
}
# Create a model for each recipe input file
recipe_files = []
for recipe in recipes:
input_file_ids = LegacyRecipeData(
recipe.data).get_input_file_ids()
for input_file_id in input_file_ids:
if input_file_id in valid_file_ids:
recipe_file = RecipeFile()
recipe_file.recipe_id = recipe.id
recipe_file.scale_file_id = input_file_id
recipe_file.created = recipe.created
recipe_files.append(recipe_file)
else:
fail_count += 1
print('Failed recipe: %i -> file: %i' %
(recipe.id, input_file_id))
RecipeFile.objects.bulk_create(recipe_files)
done_count += len(recipes)
percent = (float(done_count) / float(total_count)) * 100.00
print('Progress: %i/%i (%.2f%%)' %
(done_count, total_count, percent))
print('Migration finished. Failed: %i' % fail_count)
def test_schedule_date_range_data_ended(self, mock_msg_mgr):
"""Tests calling BatchManager.schedule_recipes() for a batch with a data ended date range restriction"""
file1 = storage_test_utils.create_file()
file1.data_started = datetime.datetime(2016, 1, 1, tzinfo=utc)
file1.data_ended = datetime.datetime(2016, 1, 10, tzinfo=utc)
file1.save()
data1 = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input',
'file_id': file1.id,
}],
'workspace_id': self.workspace.id,
}
recipe1 = Recipe.objects.create_recipe_old(
recipe_type=self.recipe_type,
input=LegacyRecipeData(data1),
event=self.event).recipe
file2 = storage_test_utils.create_file()
file2.data_started = datetime.datetime(2016, 2, 1, tzinfo=utc)
file2.data_ended = datetime.datetime(2016, 2, 10, tzinfo=utc)
file2.save()
data2 = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input',
'file_id': file2.id,
}],
'workspace_id': self.workspace.id,
}
Recipe.objects.create_recipe_old(recipe_type=self.recipe_type,
input=LegacyRecipeData(data2),
event=self.event)
recipe_test_utils.edit_recipe_type(self.recipe_type, self.definition_2)
definition = {
'date_range': {
'type': 'data',
'ended': '2016-01-15T00:00:00.000Z',
},
}
batch = batch_test_utils.create_batch_old(recipe_type=self.recipe_type,
definition=definition)
Batch.objects.schedule_recipes(batch.id)
batch = Batch.objects.get(pk=batch.id)
self.assertEqual(batch.status, 'CREATED')
self.assertEqual(batch.created_count, 1)
self.assertEqual(batch.total_count, 1)
def test_bad_file_id(self):
"""Tests calling RecipeData.validate_input_files() with a file that has an invalid ID"""
data = {'input_data': [{'name': 'File1', 'file_id': 999999}]}
files = {'File1': (True, False, ScaleFileDescription())}
self.assertRaises(InvalidRecipeData,
LegacyRecipeData(data).validate_input_files, files)
def post(self, request):
"""Queue a recipe and returns the new job information in JSON form
:param request: the HTTP POST request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
"""
recipe_type_id = rest_util.parse_int(request, 'recipe_type_id')
recipe_data = rest_util.parse_dict(request, 'recipe_data', {})
try:
recipe_type = RecipeType.objects.get(pk=recipe_type_id)
except RecipeType.DoesNotExist:
raise Http404
try:
handler = Queue.objects.queue_new_recipe_for_user(recipe_type, LegacyRecipeData(recipe_data))
except InvalidRecipeData as err:
return Response('Invalid recipe data: ' + unicode(err), status=status.HTTP_400_BAD_REQUEST)
try:
# TODO: remove this check when REST API v5 is removed
if request.version == 'v6':
recipe = Recipe.objects.get_details(handler.recipe.id)
else:
recipe = Recipe.objects.get_details_v5(handler.recipe.id)
except Recipe.DoesNotExist:
raise Http404
serializer = self.get_serializer(recipe)
recipe_url = reverse('recipe_details_view', args=[recipe.id], request=request)
return Response(serializer.data, status=status.HTTP_201_CREATED, headers=dict(location=recipe_url))
def create_recipe_handler(recipe_type=None,
data=None,
event=None,
superseded_recipe=None,
delta=None,
superseded_jobs=None):
"""Creates a recipe along with its declared jobs for unit testing
:returns: The recipe handler with created recipe and jobs
:rtype: :class:`recipe.handlers.handler.RecipeHandler`
"""
if not recipe_type:
recipe_type = create_recipe_type()
if not data:
data = {}
if not isinstance(data, LegacyRecipeData):
data = LegacyRecipeData(data)
if not event:
event = trigger_test_utils.create_trigger_event()
if superseded_recipe and not delta:
delta = RecipeGraphDelta(RecipeGraph(), RecipeGraph())
return Recipe.objects.create_recipe_old(
recipe_type,
data,
event,
superseded_recipe=superseded_recipe,
delta=delta,
superseded_jobs=superseded_jobs)
def test_successful(self, mock_store):
"""Tests calling RecipeData.validate_workspace() with successful data"""
data = {'workspace_id': 1}
# No exception is success
warnings = LegacyRecipeData(data).validate_workspace()
self.assertFalse(warnings)
def create(definition, data=None):
"""Instantiate an appropriately typed RecipeData based on definition"""
if RecipeDefinitionSunset.is_seed(definition):
return RecipeData(data)
else:
return LegacyRecipeData(data)
def test_single_missing_file_id(self):
"""Tests calling RecipeData.validate_input_files() with a single file param missing the file_id field"""
data = {'input_data': [{'name': 'File1'}]}
files = {'File1': (True, False, ScaleFileDescription())}
self.assertRaises(InvalidRecipeData,
LegacyRecipeData(data).validate_input_files, files)
def test_single_non_integral(self):
"""Tests calling RecipeData.validate_input_files() with a single file param with a non-integral for file_id field"""
data = {'input_data': [{'name': 'File1', 'file_id': 'STRING'}]}
files = {'File1': (True, False, ScaleFileDescription())}
self.assertRaises(InvalidRecipeData,
LegacyRecipeData(data).validate_input_files, files)
def test_multiple_non_integrals(self):
"""Tests calling RecipeData.validate_input_files() with a multiple file param with a list of non-integrals for file_ids field"""
data = {'input_data': [{'name': 'File1', 'file_ids': [123, 'STRING']}]}
files = {'File1': (True, True, ScaleFileDescription())}
self.assertRaises(InvalidRecipeData,
LegacyRecipeData(data).validate_input_files, files)
def test_missing_required(self):
"""Tests calling RecipeData.validate_input_files() when a file is required, but missing"""
data = {'input_data': []}
files = {'File1': (True, True, ScaleFileDescription())}
self.assertRaises(InvalidRecipeData,
LegacyRecipeData(data).validate_input_files, files)
def test_init_successful_one_property(self):
"""Tests calling RecipeData constructor successfully with a single property input."""
data = {'input_data': [{'name': 'My Name', 'value': '1'}]}
# No exception is success
LegacyRecipeData(data)
def test_required_successful(self):
"""Tests calling RecipeData.validate_properties() successfully with a required property"""
data = {'input_data': [{'name': 'Param1', 'value': 'Value1'}]}
properties = {'Param1': True}
# No exception is success
warnings = LegacyRecipeData(data).validate_properties(properties)
self.assertFalse(warnings)
def test_not_required(self):
"""Tests calling RecipeData.validate_properties() when a property is missing, but is not required"""
data = {'input_data': []}
properties = {'Param1': False}
# No exception is success
warnings = LegacyRecipeData(data).validate_properties(properties)
self.assertFalse(warnings)
def test_missing_required(self):
"""Tests calling RecipeData.validate_properties() when a property is required, but missing"""
data = {'input_data': []}
properties = {'Param1': True}
self.assertRaises(InvalidRecipeData,
LegacyRecipeData(data).validate_properties,
properties)
def test_missing_value(self):
"""Tests calling RecipeData.validate_properties() when a property is missing a value"""
data = {'input_data': [{'name': 'Param1'}]}
properties = {'Param1': False}
self.assertRaises(InvalidRecipeData,
LegacyRecipeData(data).validate_properties,
properties)
def test_multiple_given_single(self):
"""Tests calling RecipeData.validate_input_files() with a multiple file param that is provided with a single file ID"""
data = {'input_data': [{'name': 'File1', 'file_id': self.file_1.id}]}
files = {'File1': (True, True, ScaleFileDescription())}
# No exception is success
warnings = LegacyRecipeData(data).validate_input_files(files)
self.assertFalse(warnings)
def test_not_required(self):
"""Tests calling RecipeData.validate_input_files() when a file is missing, but required"""
data = {'input_data': []}
files = {'File1': (False, True, ScaleFileDescription())}
# No exception is success
warnings = LegacyRecipeData(data).validate_input_files(files)
self.assertFalse(warnings)
def test_successful_file(self):
"""Tests calling RecipeData.add_input_to_data() successfully with a file parameter"""
recipe_input_name = 'foo'
file_id = 1337
job_input_name = 'bar'
recipe_data = LegacyRecipeData(
{'input_data': [{
'name': recipe_input_name,
'file_id': file_id
}]})
job_data = MagicMock()
recipe_data.add_input_to_data(recipe_input_name, job_data,
job_input_name)
job_data.add_file_input.assert_called_with(job_input_name, file_id)
def test_successful_property(self):
"""Tests calling RecipeData.add_input_to_data() successfully with a property parameter"""
recipe_input_name = 'foo'
value = 'Doctor Who?'
job_input_name = 'bar'
recipe_data = LegacyRecipeData(
{'input_data': [{
'name': recipe_input_name,
'value': value
}]})
job_data = MagicMock()
recipe_data.add_input_to_data(recipe_input_name, job_data,
job_input_name)
job_data.add_property_input.assert_called_with(job_input_name, value)
def test_value_not_string(self):
"""Tests calling RecipeData.validate_properties() when a property has a non-string value"""
data = {'input_data': [{'name': 'Param1', 'value': 123}]}
properties = {'Param1': False}
self.assertRaises(InvalidRecipeData,
LegacyRecipeData(data).validate_properties,
properties)
def test_bad_media_type(self):
"""Tests calling RecipeData.validate_input_files() with a file that has an invalid media type"""
data = {'input_data': [{'name': 'File1', 'file_id': self.file_1.id}]}
file_desc_1 = ScaleFileDescription()
file_desc_1.add_allowed_media_type('text/plain')
files = {'File1': (True, False, file_desc_1)}
warnings = LegacyRecipeData(data).validate_input_files(files)
self.assertTrue(warnings)
def test_schedule_invalid_status(self):
"""Tests calling BatchManager.schedule_recipes() for a batch that was already created"""
Recipe.objects.create_recipe_old(recipe_type=self.recipe_type,
input=LegacyRecipeData(self.data),
event=self.event)
batch = batch_test_utils.create_batch_old(recipe_type=self.recipe_type)
Batch.objects.schedule_recipes(batch.id)
self.assertRaises(BatchError, Batch.objects.schedule_recipes, batch.id)
def test_successful(self, mock_store):
"""Tests calling RecipeDefinition.validate_data() successfully."""
definition = {
'version':
'1.0',
'input_data': [{
'name': 'Recipe Input',
'type': 'file',
'media_types': ['text/plain'],
}],
'jobs': [{
'name':
'Job 1',
'job_type': {
'name': self.job_type_1.name,
'version': self.job_type_1.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input',
'job_input': self.input_name_1,
}],
}, {
'name':
'Job 2',
'job_type': {
'name': self.job_type_2.name,
'version': self.job_type_2.version,
},
'dependencies': [{
'name':
'Job 1',
'connections': [{
'output': self.output_name_1,
'input': self.input_name_2,
}],
}],
}],
}
recipe = RecipeDefinition(definition)
recipe.validate_job_interfaces()
data = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input',
'file_id': self.file_1.id,
}],
'workspace_id': 1,
}
recipe_data = LegacyRecipeData(data)
# No exception is success
recipe.validate_data(recipe_data)
def test_missing_workspace(self, mock_store):
"""Tests calling RecipeDefinition.validate_data() with a missing required workspace."""
definition = {
'version':
'1.0',
'input_data': [{
'name': 'Recipe Input',
'type': 'file',
'media_types': ['text/plain'],
}],
'jobs': [{
'name':
'Job 1',
'job_type': {
'name': self.job_type_1.name,
'version': self.job_type_1.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input',
'job_input': self.input_name_1,
}],
}, {
'name':
'Job 2',
'job_type': {
'name': self.job_type_2.name,
'version': self.job_type_2.version,
},
'dependencies': [{
'name':
'Job 1',
'connections': [{
'output': self.output_name_1,
'input': self.input_name_2,
}],
}],
}],
}
recipe = RecipeDefinition(definition)
recipe.validate_job_interfaces()
data = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input',
'file_id': self.file_1.id,
}],
}
recipe_data = LegacyRecipeData(data)
self.assertRaises(InvalidRecipeData, recipe.validate_data, recipe_data)
def test_schedule_date_range_created(self, mock_msg_mgr):
"""Tests calling BatchManager.schedule_recipes() for a batch with a created date range restriction"""
recipe1 = Recipe.objects.create_recipe_old(
recipe_type=self.recipe_type,
input=LegacyRecipeData(self.data),
event=self.event).recipe
Recipe.objects.filter(pk=recipe1.id).update(
created=datetime.datetime(2016, 1, 1, tzinfo=utc))
recipe2 = Recipe.objects.create_recipe_old(
recipe_type=self.recipe_type,
input=LegacyRecipeData(self.data),
event=self.event).recipe
Recipe.objects.filter(pk=recipe2.id).update(
created=datetime.datetime(2016, 2, 1, tzinfo=utc))
recipe3 = Recipe.objects.create_recipe_old(
recipe_type=self.recipe_type,
input=LegacyRecipeData(self.data),
event=self.event).recipe
Recipe.objects.filter(pk=recipe3.id).update(
created=datetime.datetime(2016, 3, 1, tzinfo=utc))
recipe_test_utils.edit_recipe_type(self.recipe_type, self.definition_2)
definition = {
'date_range': {
'started': '2016-01-10T00:00:00.000Z',
'ended': '2016-02-10T00:00:00.000Z',
},
}
batch = batch_test_utils.create_batch_old(recipe_type=self.recipe_type,
definition=definition)
Batch.objects.schedule_recipes(batch.id)
batch = Batch.objects.get(pk=batch.id)
self.assertEqual(batch.status, 'CREATED')
self.assertEqual(batch.created_count, 1)
self.assertEqual(batch.total_count, 1)
def test_schedule_new_batch(self, mock_msg_mgr):
"""Tests calling BatchManager.schedule_recipes() for a batch that has never been started"""
handler = Recipe.objects.create_recipe_old(
recipe_type=self.recipe_type,
input=LegacyRecipeData(self.data),
event=self.event)
recipe_test_utils.edit_recipe_type(self.recipe_type, self.definition_2)
batch = batch_test_utils.create_batch_old(recipe_type=self.recipe_type)
Batch.objects.schedule_recipes(batch.id)
batch = Batch.objects.get(pk=batch.id)
self.assertEqual(batch.status, 'CREATED')
self.assertEqual(batch.created_count, 1)
self.assertEqual(batch.total_count, 1)
def test_schedule_no_changes(self):
"""Tests calling BatchManager.schedule_recipes() for a recipe type that has nothing to reprocess"""
Recipe.objects.create_recipe_old(recipe_type=self.recipe_type,
input=LegacyRecipeData(self.data),
event=self.event)
batch = batch_test_utils.create_batch_old(recipe_type=self.recipe_type)
Batch.objects.schedule_recipes(batch.id)
batch = Batch.objects.get(pk=batch.id)
self.assertEqual(batch.status, 'CREATED')
self.assertEqual(batch.total_count, 0)
batch_recipes = BatchRecipe.objects.all()
self.assertEqual(len(batch_recipes), 0)
def process_ingested_source_file(self, source_file, when):
"""Processes the given ingested source file by checking it against all ingest trigger rules and creating the
corresponding jobs and recipes for any triggered rules. All database changes are made in an atomic transaction.
:param source_file: The source file that was ingested
:type source_file: :class:`source.models.SourceFile`
:param when: When the source file was ingested
:type when: :class:`datetime.datetime`
"""
msg = 'Processing trigger rules for ingested source file with media type %s and data types %s'
logger.info(msg, source_file.media_type,
str(list(source_file.get_data_type_tags())))
any_rules = False
for entry in RecipeType.objects.get_active_trigger_rules(INGEST_TYPE):
rule = entry[0]
thing_to_create = entry[1]
rule_config = rule.get_configuration()
condition = rule_config.get_condition()
if condition.is_condition_met(source_file):
logger.info(condition.get_triggered_message())
any_rules = True
event = self._create_ingest_trigger_event(
source_file, rule, when)
workspace = Workspace.objects.get(
name=rule_config.get_workspace_name())
if isinstance(thing_to_create, JobType):
job_type = thing_to_create
job_data = JobData({})
job_data.add_file_input(rule_config.get_input_data_name(),
source_file.id)
job_type.get_job_interface().add_workspace_to_data(
job_data, workspace.id)
logger.info('Queuing new job of type %s %s', job_type.name,
job_type.version)
Queue.objects.queue_new_job(job_type, job_data, event)
elif isinstance(thing_to_create, RecipeType):
recipe_type = thing_to_create
recipe_data = LegacyRecipeData({})
recipe_data.add_file_input(
rule_config.get_input_data_name(), source_file.id)
recipe_data.set_workspace_id(workspace.id)
logger.info('Queuing new recipe of type %s %s',
recipe_type.name, recipe_type.version)
Queue.objects.queue_new_recipe(recipe_type, recipe_data,
event)
if not any_rules:
logger.info('No rules triggered')