def test_uuid_use_properties(self):
"""Tests setting UUIDs on products with different property values."""
inputs_json=[
{'name': 'property1', 'type': 'string'},
{'name': 'property2', 'type': 'string'}
]
manifest = job_test_utils.create_seed_manifest(name='test-job', inputs_json=inputs_json, command='my_command')
manifest['job']['interface']['inputs']['files'] = []
job_type = job_test_utils.create_seed_job_type(manifest=manifest)
job1 = job_test_utils.create_job(job_type=job_type)
job_exe1 = job_test_utils.create_job_exe(job=job1)
data1 = job_exe1.job.get_input_data()
data1.add_value(JsonValue('property1', 'value1'))
data1.add_value(JsonValue('property2', 'value2'))
job_exe1.job.input = convert_data_to_v6_json(data1).get_dict()
job2 = job_test_utils.create_job(job_type=job_type)
job_exe2 = job_test_utils.create_job_exe(job=job2)
data2 = job_exe2.job.get_input_data()
data2.add_value(JsonValue('property1', 'diffvalue1'))
data2.add_value(JsonValue('property2', 'value2'))
job_exe2.job.input = convert_data_to_v6_json(data2).get_dict()
products1 = ProductFile.objects.upload_files(self.files, [self.source_file.id], job_exe1, self.workspace)
products2 = ProductFile.objects.upload_files(self.files, [self.source_file.id], job_exe2, self.workspace)
# Make sure the product files have different UUIDs
self.assertIsNotNone(products1[0].uuid)
self.assertIsNotNone(products1[1].uuid)
self.assertNotEqual(products1[0].uuid, products2[0].uuid)
self.assertNotEqual(products1[1].uuid, products2[1].uuid)
def process_manual_ingested_source_file(self, ingest_id, source_file, when,
recipe_type_id):
"""Processes a manual ingest where a strike or scan is not involved. All database
changes are made in an atomic transaction
:param ingest_id:
:type ingest_id: int
:param source_file: The source file that was ingested
:type source_file: :class:`source.models.SourceFile`
:param when: When the source file was ingested
:type when: :class:`datetime.datetime`
:param recipe_type_id: id of the Recipe type to kick off
:type recipe_type_id: int
"""
recipe_type = RecipeType.objects.get(id=recipe_type_id)
if recipe_type and recipe_type.is_active:
recipe_data = Data()
input_name = recipe_type.get_definition().get_input_keys()[0]
recipe_data.add_value(FileValue(input_name, [source_file.id]))
event = self._create_trigger_event(None, source_file, when)
ingest_event = self._create_ingest_event(ingest_id, None,
source_file, when)
messages = create_recipes_messages(
recipe_type.name, recipe_type.revision_num,
convert_data_to_v6_json(recipe_data).get_dict(), event.id,
ingest_event.id)
CommandMessageManager().send_messages(messages)
else:
logger.info(
'No recipe type found for id %s or recipe type is inactive' %
recipe_type_id)
def test_json_forced_nodes(self):
"""Tests coverting a ProcessRecipeInput message to and from JSON with forced nodes provided"""
data_dict = convert_data_to_v6_json(Data()).get_dict()
recipe = recipe_test_utils.create_recipe(input=data_dict)
forced_nodes = ForcedNodes()
forced_nodes.set_all_nodes()
forced_nodes_dict = convert_forced_nodes_to_v6(forced_nodes).get_dict()
# Create message
message = create_process_recipe_input_messages(
[recipe.id], forced_nodes=forced_nodes)[0]
# Convert message to JSON and back, and then execute
message_json_dict = message.to_json()
new_message = ProcessRecipeInput.from_json(message_json_dict)
result = new_message.execute()
self.assertTrue(result)
recipe = Recipe.objects.get(id=recipe.id)
self.assertEqual(len(new_message.new_messages), 1)
msg = new_message.new_messages[0]
self.assertEqual(msg.type, 'update_recipe')
self.assertEqual(msg.root_recipe_id, recipe.id)
self.assertDictEqual(
convert_forced_nodes_to_v6(msg.forced_nodes).get_dict(),
forced_nodes_dict)
# Recipe should have input_file_size set to 0 (no input files)
self.assertEqual(recipe.input_file_size, 0.0)
def test_json(self):
"""Tests converting an UpdateRecipe message to and from JSON"""
data_dict = convert_data_to_v6_json(Data()).get_dict()
job_failed = job_test_utils.create_job(status='FAILED', input=data_dict)
job_pending = job_test_utils.create_job(status='PENDING')
definition = RecipeDefinition(Interface())
definition.add_job_node('job_failed', job_failed.job_type.name, job_failed.job_type.version,
job_failed.job_type_rev.revision_num)
definition.add_job_node('job_pending', job_pending.job_type.name, job_pending.job_type.version,
job_pending.job_type_rev.revision_num)
definition.add_dependency('job_failed', 'job_pending')
definition_dict = convert_recipe_definition_to_v6_json(definition).get_dict()
recipe_type = recipe_test_utils.create_recipe_type_v6(definition=definition_dict)
recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type)
recipe_test_utils.create_recipe_job(recipe=recipe, job_name='job_failed', job=job_failed)
recipe_test_utils.create_recipe_job(recipe=recipe, job_name='job_pending', job=job_pending)
# Create message
message = create_update_recipe_message(recipe.id)
# Convert message to JSON and back, and then execute
message_json_dict = message.to_json()
new_message = UpdateRecipe.from_json(message_json_dict)
result = new_message.execute()
self.assertTrue(result)
# Check for message to set job_pending to BLOCKED
self.assertEqual(len(new_message.new_messages), 1)
msg = new_message.new_messages[0]
self.assertEqual(msg.type, 'blocked_jobs')
self.assertListEqual(msg._blocked_job_ids, [job_pending.id])
def _find_existing_jobs(self):
"""Searches to determine if this message already ran and the jobs already exist
:returns: The list of job models found
:rtype: :func:`list`
"""
if self.create_jobs_type == INPUT_DATA_TYPE:
jobs = Job.objects.filter(job_type__name=self.job_type_name, job_type__version=self.job_type_version,
job_type_rev__revision_num=self.job_type_rev_num, event_id=self.event_id,
input=convert_data_to_v6_json(self.input_data).get_dict())
elif self.create_jobs_type == RECIPE_TYPE:
from recipe.models import RecipeNode
node_names = [recipe_job.node_name for recipe_job in self.recipe_jobs]
qry = RecipeNode.objects.select_related('job')
qry = qry.filter(recipe_id=self.recipe_id, node_name__in=node_names, job__event_id=self.event_id)
jobs_by_node = {recipe_node.node_name: recipe_node.job for recipe_node in qry}
jobs = jobs_by_node.values()
if jobs_by_node:
# Set up process input dict
for recipe_job in self.recipe_jobs:
job = jobs_by_node[recipe_job.node_name]
self._process_input[job.id] = recipe_job.process_input
return jobs
def create_dataset_members(self, dataset, data_list):
"""Creates a dataset member
:param dataset: The dataset the member is a part of
:type dataset: :class:`data.models.DataSet`
:param data_list: Data definitions of the dataset members
:type data_list: [:class:`data.data.data.Data`]
"""
with transaction.atomic():
dataset_members = []
datasetfiles = []
existing_scale_ids = DataSetFile.objects.get_file_ids(
dataset_ids=[dataset.id])
for d in data_list:
dataset_member = DataSetMember()
dataset_member.dataset = dataset
dataset_member.data = convert_data_to_v6_json(d).get_dict()
dataset_member.file_ids = list(data_util.get_file_ids(d))
dataset_members.append(dataset_member)
datasetfiles.extend(
DataSetFile.objects.create_dataset_files(
dataset, d, existing_scale_ids))
existing_scale_ids.append(dataset_member.file_ids)
DataSetFile.objects.bulk_create(datasetfiles)
return DataSetMember.objects.bulk_create(dataset_members)
def convert_definition_to_v6_json(definition):
"""Returns the v6 dataset definition JSON for the given definition
:param definition: The dataset definition
:type definition: :class:`data.dataset.dataset.DataSetDefinition`
:returns: The v6 dataset definition JSON
:rtype: :class:`data.dataset.json.dataset_v6.DataSetDefinitionV6`
"""
def_dict = {
'version': SCHEMA_VERSION
}
if definition.parameters:
interface_dict = convert_interface_to_v6_json(definition.parameters).get_dict()
def_dict['parameters'] = rest_utils.strip_schema_version(interface_dict)
if definition.global_parameters:
interface_dict = convert_interface_to_v6_json(definition.global_parameters).get_dict()
def_dict['global_parameters'] = rest_utils.strip_schema_version(interface_dict)
if definition.global_data:
data_dict = convert_data_to_v6_json(definition.global_data).get_dict()
def_dict['global_data'] = rest_utils.strip_schema_version(data_dict)
return DataSetDefinitionV6(definition=def_dict, do_validate=False)
def to_json(self):
"""See :meth:`messaging.messages.message.CommandMessage.to_json`
"""
json_dict = {'event_id': self.event_id, 'create_jobs_type': self.create_jobs_type}
if self.create_jobs_type == INPUT_DATA_TYPE:
json_dict['job_type_name'] = self.job_type_name
json_dict['job_type_version'] = self.job_type_version
json_dict['job_type_rev_num'] = self.job_type_rev_num
json_dict['input_data'] = convert_data_to_v6_json(self.input_data).get_dict()
elif self.create_jobs_type == RECIPE_TYPE:
json_dict['recipe_id'] = self.recipe_id
if self.root_recipe_id:
json_dict['root_recipe_id'] = self.root_recipe_id
if self.superseded_recipe_id:
json_dict['superseded_recipe_id'] = self.superseded_recipe_id
if self.batch_id:
json_dict['batch_id'] = self.batch_id
recipe_jobs = []
for recipe_job in self.recipe_jobs:
recipe_jobs.append({'job_type_name': recipe_job.job_type_name,
'job_type_version': recipe_job.job_type_version,
'job_type_rev_num': recipe_job.job_type_rev_num, 'node_name': recipe_job.node_name,
'process_input': recipe_job.process_input})
json_dict['recipe_jobs'] = recipe_jobs
json_dict['recipe_config'] = self.recipe_config
if self.ingest_event_id:
json_dict['ingest_event_id'] = self.ingest_event_id
return json_dict
def execute(self):
"""See :meth:`messaging.messages.message.CommandMessage.execute`
"""
condition = RecipeCondition.objects.get_condition_with_interfaces(
self.condition_id)
if not condition.is_processed:
definition = condition.recipe.recipe_type_rev.get_definition()
# Get condition data from dependencies in the recipe
recipe_input_data = condition.recipe.get_input_data()
node_outputs = RecipeNode.objects.get_recipe_node_outputs(
condition.recipe_id)
for node_output in node_outputs.values():
if node_output.node_type == 'condition' and node_output.id == condition.id:
node_name = node_output.node_name
break
# Set data on the condition model
try:
data = definition.generate_node_input_data(
node_name, recipe_input_data, node_outputs)
RecipeCondition.objects.set_condition_data_v6(
condition, data, node_name)
except InvalidData:
logger.exception(
'Recipe %d created invalid input data for condition %d. Message will not re-run.',
condition.recipe_id, self.condition_id)
return True
# Process filter and set whether condition was accepted
data_filter = definition.graph[node_name].data_filter
is_accepted = data_filter.is_data_accepted(data)
RecipeCondition.objects.set_processed(condition.id, is_accepted)
# Log results
filter_str = json.dumps(data_filter.filter_list,
sort_keys=True,
indent=4,
separators=(',', ': '))
data_str = json.dumps(convert_data_to_v6_json(data).get_dict(),
sort_keys=True,
indent=4,
separators=(',', ': '))
logger.info(
'Condition %d (recipe %d at %s) evaluated to %s:\nCondition: %s\nInput Data: %s',
condition.id, condition.recipe_id, node_name, is_accepted,
filter_str, data_str)
# Create message to update the condition's recipe
from recipe.messages.update_recipe import create_update_recipe_message
root_recipe_id = condition.recipe.root_recipe_id if condition.recipe.root_recipe_id else condition.recipe_id
logger.info(
'Processed data for condition %d, sending message to update recipe %d',
self.condition_id, root_recipe_id)
self.new_messages.append(create_update_recipe_message(root_recipe_id))
return True
def get_dict(self):
"""Returns the internal dictionary that represents these job results
:returns: The dictionary representing the results
:rtype: dict
"""
return convert_data_to_v6_json(self._results_data).get_dict()
def test_convert_data_to_v6_json(self):
"""Tests calling convert_data_to_v6_json()"""
# Try interface with nothing set
data = Data()
json = convert_data_to_v6_json(data)
DataV6(data=json.get_dict(), do_validate=True) # Revalidate
# Try data with a variety of values
data = Data()
data.add_value(FileValue('input_a', [1234]))
data.add_value(FileValue('input_b', [1235, 1236]))
data.add_value(JsonValue('input_c', 'hello'))
data.add_value(JsonValue('input_d', 11.9))
json = convert_data_to_v6_json(data)
DataV6(data=json.get_dict(), do_validate=True) # Revalidate
self.assertSetEqual(set(json.get_data().values.keys()), {'input_a', 'input_b', 'input_c', 'input_d'})
def get_dict(self):
"""Returns the internal dictionary that represents this job data
:returns: The internal dictionary
:rtype: dict
"""
return convert_data_to_v6_json(self._new_data).get_dict()
def get_v6_data_json(self):
"""Returns the data for this datasetmember as v6 json with the version stripped
:returns: The v6 JSON output data dict for this datasetmember
:rtype: dict
"""
return rest_utils.strip_schema_version(
convert_data_to_v6_json(self.get_data()).get_dict())
def test_json(self):
"""Tests converting a ProcessCondition message to and from JSON"""
definition = RecipeDefinition(Interface())
cond_interface_1 = Interface()
cond_interface_1.add_parameter(JsonParameter('cond_int', 'integer'))
df1 = DataFilter(filter_list=[{
'name': 'cond_int',
'type': 'integer',
'condition': '==',
'values': [0]
}])
definition = RecipeDefinition(cond_interface_1)
definition.add_condition_node('node_a', cond_interface_1, df1)
definition.add_recipe_input_connection('node_a', 'cond_int',
'cond_int')
definition_dict = convert_recipe_definition_to_v6_json(
definition).get_dict()
recipe_type = recipe_test_utils.create_recipe_type_v6(
definition=definition_dict)
data_1 = Data()
data_1.add_value(JsonValue('cond_int', 0))
data_1_dict = convert_data_to_v6_json(data_1).get_dict()
recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type,
input=data_1_dict)
condition = recipe_test_utils.create_recipe_condition(recipe=recipe,
save=True)
recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='node_a',
condition=condition,
save=True)
# Create message
message = create_process_condition_messages([condition.id])[0]
# Convert message to JSON and back, and then execute
message_json_dict = message.to_json()
new_message = ProcessCondition.from_json(message_json_dict)
result = new_message.execute()
self.assertTrue(result)
condition = RecipeCondition.objects.get(id=condition.id)
self.assertEqual(len(new_message.new_messages), 1)
self.assertEqual(new_message.new_messages[0].type, 'update_recipe')
self.assertEqual(new_message.new_messages[0].root_recipe_id, recipe.id)
self.assertTrue(condition.is_processed)
self.assertIsNotNone(condition.processed)
self.assertTrue(condition.is_accepted)
def setUp(self):
django.setup()
def upload_files(file_uploads):
for file_upload in file_uploads:
file_upload.file.save()
def delete_files(files):
for scale_file in files:
scale_file.save()
self.workspace = storage_test_utils.create_workspace()
self.workspace.upload_files = MagicMock(side_effect=upload_files)
self.workspace.delete_files = MagicMock(side_effect=delete_files)
self.source_file = source_test_utils.create_source(file_name='input1.txt', workspace=self.workspace)
inputs_json=[
{'name': 'property1', 'type': 'string'},
{'name': 'property2', 'type': 'string'}
]
manifest = job_test_utils.create_seed_manifest(inputs_json=inputs_json, command='my_command')
manifest['job']['interface']['inputs']['files'] = []
job_type = job_test_utils.create_seed_job_type(manifest=manifest)
self.job_exe = job_test_utils.create_job_exe(job_type=job_type)
data = self.job_exe.job.get_input_data()
data.add_value(JsonValue('property1', 'value1'))
data.add_value(JsonValue('property2', 'value2'))
self.job_exe.job.input = convert_data_to_v6_json(data).get_dict()
self.job_exe.job.source_sensor_class = 'classA'
self.job_exe.job.source_sensor = '1'
self.job_exe.job.source_collection = '12345'
self.job_exe.job.source_task = 'my-task'
self.job_exe.job.save()
self.job_exe_no = job_test_utils.create_job_exe()
self.local_path_1 = os.path.join(SCALE_JOB_EXE_OUTPUT_PATH, 'local/1/file.txt')
self.local_path_2 = os.path.join(SCALE_JOB_EXE_OUTPUT_PATH, 'local/2/file.json')
self.local_path_3 = os.path.join(SCALE_JOB_EXE_OUTPUT_PATH, 'local/3/file.h5')
self.files = [
ProductFileMetadata(output_name='output_name_1', local_path=self.local_path_1,
remote_path='remote/1/file.txt'),
ProductFileMetadata(output_name='output_name_2', local_path=self.local_path_2,
media_type='application/x-custom-json', remote_path='remote/2/file.json',
source_sensor_class='classB', source_sensor='2', source_collection='12346',
source_task='my-task-2'),
]
self.files_no = [
ProductFileMetadata(output_name='output_name_3', local_path=self.local_path_3, media_type='image/x-hdf5-image', remote_path='remote/3/file.h5')
]
def process_ingested_source_file_cm(self, ingest_id, source, source_file,
when):
"""Processes the given ingested source file by kicking off its recipe.
All database changes are made in an atomic transaction.
:param source: The strike that triggered the ingest
:type scan: `object`
:param source_file: The source file that was ingested
:type source_file: :class:`source.models.SourceFile`
:param when: When the source file was ingested
:type when: :class:`datetime.datetime`
"""
# Create the recipe handler associated with the ingest strike/scan
source_recipe_config = source.configuration['recipe']
recipe_name = source_recipe_config['name']
recipe_revision = source_recipe_config[
'revision_num'] if 'revision_num' in source_recipe_config else None
recipe_type = RecipeType.objects.get(name=recipe_name)
if recipe_revision:
recipe_type = RecipeTypeRevision.objects.get_revision(
recipe_name, recipe_revision).recipe_type
if len(recipe_type.get_definition().get_input_keys()) == 0:
logger.info(
'No inputs defined for recipe %s. Recipe will not be run.' %
recipe_name)
return
if recipe_type and recipe_type.is_active:
# Assuming one input per recipe, so pull the first defined input you find
recipe_data = Data()
input_name = recipe_type.get_definition().get_input_keys()[0]
recipe_data.add_value(FileValue(input_name, [source_file.id]))
event = self._create_trigger_event(source, source_file, when)
ingest_event = self._create_ingest_event(ingest_id, source,
source_file, when)
# This can cause a race condition with a slow DB.
messages = create_recipes_messages(
recipe_type.name, recipe_type.revision_num,
convert_data_to_v6_json(recipe_data).get_dict(), event.id,
ingest_event.id)
CommandMessageManager().send_messages(messages)
else:
logger.info(
'No recipe type found for %s %s or recipe type is inactive' %
(recipe_name, recipe_revision))
def test_execute_invalid_data(self):
"""Tests calling CreateJobs.execute() when the input data is invalid"""
manifest = {
'seedVersion': '1.0.0',
'job': {
'name': 'name',
'jobVersion': '1.0.0',
'packageVersion': '1.0.0',
'title': 'Title',
'description': 'This is a description',
'maintainer': {
'name': 'John Doe',
'email': '*****@*****.**'
},
'timeout': 10,
'interface': {
'command': 'the command',
'inputs': {
'files': [{
'name': 'input_a'
}]
}
}
}
}
job_type = job_test_utils.create_seed_job_type(manifest=manifest)
event = trigger_test_utils.create_trigger_event()
# Data does not provide required input_a so it is invalid
data_dict = convert_data_to_v6_json(Data()).get_dict()
# Create and execute message
message = create_jobs_message(job_type.name,
job_type.version,
job_type.revision_num,
event.id,
count=10,
input_data_dict=data_dict)
result = message.execute()
self.assertTrue(result)
self.assertEqual(
Job.objects.filter(job_type_id=job_type.id,
event_id=event.id).count(), 0)
# Should be no new messages
self.assertEqual(len(message.new_messages), 0)
def test_queue_job_timestamps(self):
"""Tests that job attributes are updated when a job is queued."""
data_dict = convert_data_to_v6_json(Data()).get_dict()
job = job_test_utils.create_job(num_exes=1,
status='CANCELED',
input=data_dict,
started=timezone.now(),
ended=timezone.now())
Job.objects.update_jobs_to_queued([job], timezone.now(), requeue=True)
job = Job.objects.get(pk=job.id)
self.assertEqual(job.status, 'QUEUED')
self.assertIsNotNone(job.queued)
self.assertIsNone(job.started)
self.assertIsNone(job.ended)
def test_process_event_range(self, mock_Queue, mock_msg_mgr):
"""Tests processing an event that requires catching up for a range of previous days."""
event = job_test_utils.create_clock_event(
occurred=datetime.datetime(2015, 1, 10, 10, tzinfo=utc))
last = job_test_utils.create_clock_event(
occurred=datetime.datetime(2015, 1, 7, 12, tzinfo=utc))
self.processor.process_event(event, last)
i = 1
for call_args in mock_Queue.objects.queue_new_job_v6.call_args_list:
args = call_args[0]
self.assertEqual(self.job_type, args[0])
self.assertEqual(event, args[2])
inputs = convert_data_to_v6_json(args[1])
if i == 1:
self.assertDictEqual(
{
u'files': {},
u'json': {
u'DAY': '2015-01-07'
},
u'version': u'7'
}, inputs.get_dict())
if i == 2:
self.assertDictEqual(
{
u'files': {},
u'json': {
u'DAY': '2015-01-08'
},
u'version': u'7'
}, inputs.get_dict())
if i == 3:
self.assertDictEqual(
{
u'files': {},
u'json': {
u'DAY': '2015-01-09'
},
u'version': u'7'
}, inputs.get_dict())
i += 1
self.assertEqual(mock_Queue.objects.queue_new_job_v6.call_count, 3)
def test_json_no_recipe(self):
"""Tests converting a CreateJobs message (without a recipe) to and from JSON"""
manifest = {
'seedVersion': '1.0.0',
'job': {
'name': 'name',
'jobVersion': '1.0.0',
'packageVersion': '1.0.0',
'title': 'Title',
'description': 'This is a description',
'maintainer': {
'name': 'John Doe',
'email': '*****@*****.**'
},
'timeout': 10
}
}
job_type = job_test_utils.create_seed_job_type(manifest=manifest)
event = trigger_test_utils.create_trigger_event()
data_dict = convert_data_to_v6_json(Data()).get_dict()
count = 10
# Create message
message = create_jobs_message(job_type.name,
job_type.version,
job_type.revision_num,
event.id,
count=count,
input_data_dict=data_dict)
# Convert message to JSON and back, and then execute
message_json_dict = message.to_json()
new_message = CreateJobs.from_json(message_json_dict)
result = new_message.execute()
self.assertTrue(result)
self.assertEqual(
Job.objects.filter(job_type_id=job_type.id,
event_id=event.id).count(), count)
# Check for process_job_input messages
self.assertEqual(len(new_message.new_messages), count)
for msg in new_message.new_messages:
self.assertEqual(msg.type, 'process_job_input')
def test_json_forced_nodes(self):
"""Tests converting an UpdateRecipe message to and from JSON when forced nodes are provided"""
data_dict = convert_data_to_v6_json(Data()).get_dict()
job_completed = job_test_utils.create_job(status='COMPLETED', input=data_dict, output=data_dict)
sub_recipe_type = recipe_test_utils.create_recipe_type_v6()
definition = RecipeDefinition(Interface())
definition.add_job_node('job_completed', job_completed.job_type.name, job_completed.job_type.version,
job_completed.job_type_rev.revision_num)
definition.add_recipe_node('the_sub_recipe', sub_recipe_type.name, sub_recipe_type.revision_num)
definition.add_dependency('job_completed', 'the_sub_recipe')
definition_dict = convert_recipe_definition_to_v6_json(definition).get_dict()
recipe_type = recipe_test_utils.create_recipe_type_v6(definition=definition_dict)
recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type, input=data_dict)
recipe_test_utils.create_recipe_job(recipe=recipe, job_name='job_completed', job=job_completed)
forced_nodes = ForcedNodes()
sub_forced_nodes = ForcedNodes()
sub_forced_nodes.set_all_nodes()
forced_nodes.add_subrecipe('the_sub_recipe', sub_forced_nodes)
# Create message
message = create_update_recipe_message(recipe.id, forced_nodes=forced_nodes)
# Convert message to JSON and back, and then execute
message_json_dict = message.to_json()
new_message = UpdateRecipe.from_json(message_json_dict)
result = new_message.execute()
self.assertTrue(result)
# Check for message to create sub-recipe
self.assertEqual(len(new_message.new_messages), 1)
msg = new_message.new_messages[0]
self.assertEqual(msg.type, 'create_recipes')
self.assertEqual(msg.event_id, recipe.event_id)
msg_forced_nodes_dict = convert_forced_nodes_to_v6(msg.forced_nodes).get_dict()
expected_forced_nodes_dict = convert_forced_nodes_to_v6(forced_nodes).get_dict()
self.assertDictEqual(msg_forced_nodes_dict, expected_forced_nodes_dict)
self.assertEqual(msg.create_recipes_type, SUB_RECIPE_TYPE)
self.assertEqual(msg.recipe_id, recipe.id)
self.assertEqual(msg.root_recipe_id, recipe.root_superseded_recipe_id)
self.assertIsNone(msg.superseded_recipe_id)
sub = SubRecipe(sub_recipe_type.name, sub_recipe_type.revision_num, 'the_sub_recipe', True)
self.assertListEqual(msg.sub_recipes, [sub])
def test_process_event_first(self, mock_Queue, mock_msg_mgr):
"""Tests processing an event that was never triggered before."""
event = job_test_utils.create_clock_event(
occurred=datetime.datetime(2015, 1, 10, 12, tzinfo=utc))
self.processor.process_event(event, None)
call_args = mock_Queue.objects.queue_new_job_v6.call_args[0]
inputs = convert_data_to_v6_json(call_args[1])
self.assertEqual(self.job_type, call_args[0])
self.assertDictEqual(
{
u'files': {},
u'json': {
u'DAY': '2015-01-09'
},
u'version': u'7'
}, inputs.get_dict())
self.assertEqual(event, call_args[2])
def execute(self):
"""See :meth:`messaging.messages.message.CommandMessage.execute`
"""
files_to_delete = ScaleFile.objects.filter_files(job_ids=[self.job_id])
if files_to_delete:
# Construct input data
files = []
workspaces = []
for f in files_to_delete:
files.append({
'id': f.id,
'file_path': f.file_path,
'workspace': f.workspace.name
})
if f.workspace.name not in [
k for wrkspc in workspaces for k in wrkspc.keys()
]:
workspaces.append(
{f.workspace.name: f.workspace.json_config})
inputs = Data()
inputs.add_value(JsonValue('job_id', str(self.job_id)))
inputs.add_value(JsonValue('trigger_id', str(self.trigger_id)))
inputs.add_value(
JsonValue('source_file_id', str(self.source_file_id)))
inputs.add_value(JsonValue('purge', str(self.purge)))
inputs.add_value(JsonValue('files', json.dumps(files)))
inputs.add_value(JsonValue('workspaces', json.dumps(workspaces)))
inputs_json = convert_data_to_v6_json(inputs)
# Send message to create system job to delete files
msg = create_jobs_message(job_type_name="scale-delete-files",
job_type_version="1.0.0",
event_id=self.trigger_id,
job_type_rev_num=1,
input_data_dict=inputs_json.get_dict())
self.new_messages.append(msg)
return True
def test_json(self):
"""Tests coverting a ProcessRecipeInput message to and from JSON"""
data_dict = convert_data_to_v6_json(Data()).get_dict()
recipe = recipe_test_utils.create_recipe(input=data_dict)
# Create message
message = ProcessRecipeInput()
message.recipe_id = recipe.id
# Convert message to JSON and back, and then execute
message_json_dict = message.to_json()
new_message = ProcessRecipeInput.from_json(message_json_dict)
result = new_message.execute()
self.assertTrue(result)
recipe = Recipe.objects.get(id=recipe.id)
self.assertEqual(len(new_message.new_messages), 1)
self.assertEqual(new_message.new_messages[0].type, 'update_recipes')
# Recipe should have input_file_size set to 0 (no input files)
self.assertEqual(recipe.input_file_size, 0.0)
def test_execute_with_data(self):
"""Tests calling ProcessRecipeInput.execute() successfully when the recipe already has data populated"""
workspace = storage_test_utils.create_workspace()
file_1 = storage_test_utils.create_file(workspace=workspace,
file_size=10485760.0)
file_2 = storage_test_utils.create_file(workspace=workspace,
file_size=104857600.0)
file_3 = storage_test_utils.create_file(workspace=workspace,
file_size=987654321.0)
recipe_interface = Interface()
recipe_interface.add_parameter(FileParameter('input_a',
['text/plain']))
recipe_interface.add_parameter(
FileParameter('input_b', ['text/plain'], multiple=True))
definition = RecipeDefinition(recipe_interface)
definition_dict = convert_recipe_definition_to_v6_json(
definition).get_dict()
recipe_type = recipe_test_utils.create_recipe_type(
definition=definition_dict)
data = Data()
data.add_value(FileValue('input_a', [file_1.id]))
data.add_value(FileValue('input_b', [file_2.id, file_3.id]))
data_dict = convert_data_to_v6_json(data).get_dict()
recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type,
input=data_dict)
# Create message
message = ProcessRecipeInput()
message.recipe_id = recipe.id
# Execute message
result = message.execute()
self.assertTrue(result)
recipe = Recipe.objects.get(id=recipe.id)
# Check for update_recipes message
self.assertEqual(len(message.new_messages), 1)
self.assertEqual(message.new_messages[0].type, 'update_recipes')
# Check recipe for expected input_file_size
self.assertEqual(recipe.input_file_size, 1052.0)
# Make sure recipe input file models are created
recipe_input_files = RecipeInputFile.objects.filter(
recipe_id=recipe.id)
self.assertEqual(len(recipe_input_files), 3)
for recipe_input_file in recipe_input_files:
if recipe_input_file.input_file_id == file_1.id:
self.assertEqual(recipe_input_file.recipe_input, 'input_a')
elif recipe_input_file.input_file_id == file_2.id:
self.assertEqual(recipe_input_file.recipe_input, 'input_b')
elif recipe_input_file.input_file_id == file_3.id:
self.assertEqual(recipe_input_file.recipe_input, 'input_b')
else:
self.fail('Invalid input file ID: %s' %
recipe_input_file.input_file_id)
# Test executing message again
message_json_dict = message.to_json()
message = ProcessRecipeInput.from_json(message_json_dict)
result = message.execute()
self.assertTrue(result)
# Still should have update_recipes message
self.assertEqual(len(message.new_messages), 1)
self.assertEqual(message.new_messages[0].type, 'update_recipes')
# Make sure recipe input file models are unchanged
recipe_input_files = RecipeInputFile.objects.filter(
recipe_id=recipe.id)
self.assertEqual(len(recipe_input_files), 3)
def test_has_completed_true(self):
"""Tests calling Recipe.has_completed() when an entire recipe has completed"""
data_dict = convert_data_to_v6_json(Data()).get_dict()
job_type = job_test_utils.create_seed_job_type()
sub_recipe_type = recipe_test_utils.create_recipe_type_v6()
cond_interface_1 = Interface()
cond_interface_1.add_parameter(JsonParameter('cond_int', 'integer'))
df2 = DataFilter(filter_list=[{
'name': 'cond_int',
'type': 'integer',
'condition': '==',
'values': [0]
}, {
'name': 'cond_int',
'type': 'integer',
'condition': '!=',
'values': [0]
}],
all=True) #always False
definition = RecipeDefinition(Interface())
definition.add_job_node('A', job_type.name, job_type.version,
job_type.revision_num)
definition.add_recipe_node('B', sub_recipe_type.name,
sub_recipe_type.revision_num)
definition.add_job_node('C', job_type.name, job_type.version,
job_type.revision_num)
definition.add_job_node('D', job_type.name, job_type.version,
job_type.revision_num)
definition.add_job_node('E', job_type.name, job_type.version,
job_type.revision_num)
definition.add_job_node('F', job_type.name, job_type.version,
job_type.revision_num)
definition.add_recipe_node('G', sub_recipe_type.name,
sub_recipe_type.revision_num)
definition.add_job_node('H', job_type.name, job_type.version,
job_type.revision_num)
definition.add_condition_node('I', cond_interface_1, df2) #False
definition.add_job_node('J', job_type.name, job_type.version,
job_type.revision_num)
definition.add_dependency('A', 'C')
definition.add_dependency('A', 'E')
definition.add_dependency('A', 'H')
definition.add_dependency('C', 'D')
definition.add_dependency('G', 'H')
definition.add_dependency('A', 'I')
definition.add_dependency('I', 'J')
job_a = job_test_utils.create_job(job_type=job_type,
status='COMPLETED',
output=data_dict,
save=False)
job_c = job_test_utils.create_job(job_type=job_type,
status='COMPLETED',
output=data_dict,
save=False)
job_d = job_test_utils.create_job(job_type=job_type,
status='COMPLETED',
output=data_dict,
save=False)
job_e = job_test_utils.create_job(job_type=job_type,
status='COMPLETED',
output=data_dict,
save=False)
job_f = job_test_utils.create_job(job_type=job_type,
status='COMPLETED',
output=data_dict,
save=False)
job_h = job_test_utils.create_job(job_type=job_type,
status='COMPLETED',
output=data_dict,
save=False)
Job.objects.bulk_create([job_a, job_c, job_d, job_e, job_f, job_h])
condition_i = recipe_test_utils.create_recipe_condition(
is_processed=True, is_accepted=False, save=True)
recipe_b = recipe_test_utils.create_recipe(recipe_type=sub_recipe_type,
save=False)
recipe_b.is_completed = True
recipe_g = recipe_test_utils.create_recipe(recipe_type=sub_recipe_type,
save=False)
recipe_g.is_completed = True
Recipe.objects.bulk_create([recipe_b, recipe_g])
definition_json_dict = convert_recipe_definition_to_v6_json(
definition).get_dict()
recipe_type = recipe_test_utils.create_recipe_type_v6(
definition=definition_json_dict)
recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type)
recipe_node_a = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='A',
job=job_a,
save=False,
is_original=False)
recipe_node_c = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='C',
job=job_c,
save=False,
is_original=False)
recipe_node_d = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='D',
job=job_d,
save=False,
is_original=False)
recipe_node_e = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='E',
job=job_e,
save=False)
recipe_node_f = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='F',
job=job_f,
save=False)
recipe_node_h = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='H',
job=job_h,
save=False)
recipe_node_i = recipe_test_utils.create_recipe_node(
recipe=recipe, node_name='I', condition=condition_i, save=False)
recipe_node_g = recipe_test_utils.create_recipe_node(
recipe=recipe,
node_name='G',
sub_recipe=recipe_g,
save=False,
is_original=False)
recipe_node_b = recipe_test_utils.create_recipe_node(
recipe=recipe, node_name='B', sub_recipe=recipe_b, save=False)
RecipeNode.objects.bulk_create([
recipe_node_a, recipe_node_b, recipe_node_c, recipe_node_d,
recipe_node_e, recipe_node_f, recipe_node_g, recipe_node_h,
recipe_node_i
])
recipe_instance = Recipe.objects.get_recipe_instance(recipe.id)
self.assertTrue(recipe_instance.has_completed())
def test_has_completed_false(self):
"""Tests calling Recipe.has_completed() when an entire recipe has not completed"""
data_dict = convert_data_to_v6_json(Data()).get_dict()
job_type = job_test_utils.create_seed_job_type()
sub_recipe_type = recipe_test_utils.create_recipe_type_v6()
definition = RecipeDefinition(Interface())
definition.add_job_node('A', job_type.name, job_type.version,
job_type.revision_num)
definition.add_recipe_node('B', sub_recipe_type.name,
sub_recipe_type.revision_num)
definition.add_job_node('C', job_type.name, job_type.version,
job_type.revision_num)
definition.add_job_node('D', job_type.name, job_type.version,
job_type.revision_num)
definition.add_job_node('E', job_type.name, job_type.version,
job_type.revision_num)
definition.add_job_node('F', job_type.name, job_type.version,
job_type.revision_num)
definition.add_recipe_node('G', sub_recipe_type.name,
sub_recipe_type.revision_num)
definition.add_job_node('H', job_type.name, job_type.version,
job_type.revision_num)
definition.add_dependency('A', 'C')
definition.add_dependency('A', 'E')
definition.add_dependency('A', 'H')
definition.add_dependency('C', 'D')
definition.add_dependency('G', 'H')
job_a = job_test_utils.create_job(job_type=job_type,
status='COMPLETED',
output=data_dict,
save=False)
job_c = job_test_utils.create_job(job_type=job_type,
status='COMPLETED',
output=data_dict,
save=False)
job_d = job_test_utils.create_job(job_type=job_type,
status='COMPLETED',
output=data_dict,
save=False)
job_e = job_test_utils.create_job(job_type=job_type,
status='COMPLETED',
output=data_dict,
save=False)
job_f = job_test_utils.create_job(job_type=job_type,
status='COMPLETED',
output=data_dict,
save=False)
job_h = job_test_utils.create_job(job_type=job_type,
status='COMPLETED',
output=data_dict,
save=False)
Job.objects.bulk_create([job_a, job_c, job_d, job_e, job_f, job_h])
recipe_b = recipe_test_utils.create_recipe(recipe_type=sub_recipe_type,
save=False)
recipe_b.is_completed = True
recipe_g = recipe_test_utils.create_recipe(recipe_type=sub_recipe_type,
save=False)
recipe_g.is_completed = False
Recipe.objects.bulk_create([recipe_b, recipe_g])
definition_json_dict = convert_recipe_definition_to_v6_json(
definition).get_dict()
recipe_type = recipe_test_utils.create_recipe_type_v6(
definition=definition_json_dict)
recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type)
recipe_node_a = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='A',
job=job_a,
save=False,
is_original=False)
recipe_node_c = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='C',
job=job_c,
save=False,
is_original=False)
recipe_node_d = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='D',
job=job_d,
save=False,
is_original=False)
recipe_node_e = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='E',
job=job_e,
save=False)
recipe_node_f = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='F',
job=job_f,
save=False)
recipe_node_h = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='H',
job=job_h,
save=False)
recipe_node_g = recipe_test_utils.create_recipe_node(
recipe=recipe,
node_name='G',
sub_recipe=recipe_g,
save=False,
is_original=False)
recipe_node_b = recipe_test_utils.create_recipe_node(
recipe=recipe, node_name='B', sub_recipe=recipe_b, save=False)
RecipeNode.objects.bulk_create([
recipe_node_a, recipe_node_b, recipe_node_c, recipe_node_d,
recipe_node_e, recipe_node_f, recipe_node_g, recipe_node_h
])
recipe_instance = Recipe.objects.get_recipe_instance(recipe.id)
self.assertFalse(recipe_instance.has_completed())
def test_get_nodes_to_process_input(self):
"""Tests calling Recipe.get_nodes_to_process_input()"""
data_dict = convert_data_to_v6_json(Data()).get_dict()
job_type = job_test_utils.create_seed_job_type()
sub_recipe_type = recipe_test_utils.create_recipe_type_v6()
# Create recipe
definition = RecipeDefinition(Interface())
cond_interface_1 = Interface()
cond_interface_1.add_parameter(JsonParameter('cond_int', 'integer'))
definition.add_job_node('A', job_type.name, job_type.version,
job_type.revision_num)
df1 = DataFilter(filter_list=[{
'name': 'cond_int',
'type': 'integer',
'condition': '==',
'values': [0]
}, {
'name': 'cond_int',
'type': 'integer',
'condition': '!=',
'values': [0]
}],
all=False) #always True
df2 = DataFilter(filter_list=[{
'name': 'cond_int',
'type': 'integer',
'condition': '==',
'values': [0]
}, {
'name': 'cond_int',
'type': 'integer',
'condition': '!=',
'values': [0]
}],
all=True) #always False
definition.add_condition_node('B', cond_interface_1, df1) #True
definition.add_condition_node('C', cond_interface_1, df1) #True
definition.add_condition_node('D', cond_interface_1, df2) #False
definition.add_job_node('E', job_type.name, job_type.version,
job_type.revision_num)
definition.add_job_node('F', job_type.name, job_type.version,
job_type.revision_num)
definition.add_recipe_node('G', sub_recipe_type.name,
sub_recipe_type.revision_num)
definition.add_recipe_node('H', sub_recipe_type.name,
sub_recipe_type.revision_num)
definition.add_dependency('A', 'D')
definition.add_dependency('A', 'E')
definition.add_dependency('B', 'E')
definition.add_dependency('B', 'F')
definition.add_dependency('C', 'F')
definition.add_dependency('D', 'G')
definition.add_dependency('E', 'G')
definition.add_dependency('E', 'H')
definition_json_dict = convert_recipe_definition_to_v6_json(
definition).get_dict()
recipe_type = recipe_test_utils.create_recipe_type_v6(
definition=definition_json_dict)
recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type,
input=data_dict)
# Nodes A, B, and D already exist
job_a = job_test_utils.create_job(job_type=job_type,
status='COMPLETED',
input=data_dict,
output=data_dict,
save=True)
condition_b = recipe_test_utils.create_recipe_condition(
is_processed=True, is_accepted=True, save=False)
condition_d = recipe_test_utils.create_recipe_condition(
is_processed=True, is_accepted=False, save=False)
RecipeCondition.objects.bulk_create([condition_b, condition_d])
recipe_node_a = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='A',
job=job_a,
save=False)
recipe_node_b = recipe_test_utils.create_recipe_node(
recipe=recipe, node_name='B', condition=condition_b, save=False)
recipe_node_d = recipe_test_utils.create_recipe_node(
recipe=recipe, node_name='D', condition=condition_d, save=False)
RecipeNode.objects.bulk_create(
[recipe_node_a, recipe_node_b, recipe_node_d])
recipe_instance = Recipe.objects.get_recipe_instance(recipe.id)
nodes_to_process = recipe_instance.get_nodes_to_process_input()
self.assertSetEqual(set(nodes_to_process.keys()), {'C', 'E'})
def json(self):
"""Accessor for json in results"""
return convert_data_to_v6_json(self._results_data).get_dict()['json']
def files(self):
"""Accessor for files in results"""
return convert_data_to_v6_json(self._results_data).get_dict()['files']
