def process_event(self, event, last_event=None):
"""See :meth:`job.clock.ClockEventProcessor.process_event`.
Compares the new event with the last event any missing metrics jobs.
"""
# Attempt to get the daily metrics job type
try:
job_type = JobType.objects.filter(
name='scale-daily-metrics').last()
except JobType.DoesNotExist:
raise ClockEventError(
'Missing required job type: scale-daily-metrics')
if last_event:
# Build a list of days that require metrics
day_count = xrange(
(event.occurred.date() - last_event.occurred.date()).days)
days = [
last_event.occurred.date() + datetime.timedelta(days=d)
for d in day_count
]
else:
# Use the previous day when first triggered
days = [timezone.now().date() - datetime.timedelta(days=1)]
# Schedule one job for each required day
for day in days:
job_data = Data()
job_data.add_value(JsonValue('DAY', day.strftime('%Y-%m-%d')))
job = Queue.objects.queue_new_job_v6(job_type, job_data, event)
CommandMessageManager().send_messages(
create_process_job_input_messages([job.id]))
def process_manual_ingested_source_file(self, ingest_id, source_file, when,
recipe_type_id):
"""Processes a manual ingest where a strike or scan is not involved. All database
changes are made in an atomic transaction
:param ingest_id:
:type ingest_id: int
:param source_file: The source file that was ingested
:type source_file: :class:`source.models.SourceFile`
:param when: When the source file was ingested
:type when: :class:`datetime.datetime`
:param recipe_type_id: id of the Recipe type to kick off
:type recipe_type_id: int
"""
recipe_type = RecipeType.objects.get(id=recipe_type_id)
if recipe_type and recipe_type.is_active:
recipe_data = Data()
input_name = recipe_type.get_definition().get_input_keys()[0]
recipe_data.add_value(FileValue(input_name, [source_file.id]))
event = self._create_trigger_event(None, source_file, when)
ingest_event = self._create_ingest_event(ingest_id, None,
source_file, when)
logger.info('Queueing new recipe of type %s %s', recipe_type.name,
recipe_type.revision_num)
Queue.objects.queue_new_recipe_v6(recipe_type, recipe_data, event,
ingest_event)
else:
logger.info(
'No recipe type found for id %s or recipe type is inactive' %
recipe_type_id)
def process_ingested_source_file_cm(self, ingest_id, source, source_file,
when):
"""Processes the given ingested source file by kicking off its recipe.
All database changes are made in an atomic transaction.
:param source: The strike that triggered the ingest
:type scan: `object`
:param source_file: The source file that was ingested
:type source_file: :class:`source.models.SourceFile`
:param when: When the source file was ingested
:type when: :class:`datetime.datetime`
"""
# Create the recipe handler associated with the ingest strike/scan
source_recipe_config = source.configuration['recipe']
recipe_name = source_recipe_config['name']
recipe_revision = source_recipe_config[
'revision_num'] if 'revision_num' in source_recipe_config else None
recipe_type = RecipeType.objects.get(name=recipe_name)
if recipe_revision:
recipe_type = RecipeTypeRevision.objects.get_revision(
recipe_name, recipe_revision).recipe_type
if len(recipe_type.get_definition().get_input_keys()) == 0:
logger.info(
'No inputs defined for recipe %s. Recipe will not be run.' %
recipe_name)
return
if recipe_type and recipe_type.is_active:
# Assuming one input per recipe, so pull the first defined input you find
recipe_data = Data()
input_name = recipe_type.get_definition().get_input_keys()[0]
recipe_data.add_value(FileValue(input_name, [source_file.id]))
event = self._create_trigger_event(source, source_file, when)
ingest_event = self._create_ingest_event(ingest_id, source,
source_file, when)
# This can cause a race condition with a slow DB.
messages = create_recipes_messages(
recipe_type.name, recipe_type.revision_num,
convert_data_to_v6_json(recipe_data).get_dict(), event.id,
ingest_event.id)
CommandMessageManager().send_messages(messages)
else:
logger.info(
'No recipe type found for %s %s or recipe type is inactive' %
(recipe_name, recipe_revision))
def test_json(self):
"""Tests converting a ProcessCondition message to and from JSON"""
definition = RecipeDefinition(Interface())
cond_interface_1 = Interface()
cond_interface_1.add_parameter(JsonParameter('cond_int', 'integer'))
df1 = DataFilter(filter_list=[{
'name': 'cond_int',
'type': 'integer',
'condition': '==',
'values': [0]
}])
definition = RecipeDefinition(cond_interface_1)
definition.add_condition_node('node_a', cond_interface_1, df1)
definition.add_recipe_input_connection('node_a', 'cond_int',
'cond_int')
definition_dict = convert_recipe_definition_to_v6_json(
definition).get_dict()
recipe_type = recipe_test_utils.create_recipe_type_v6(
definition=definition_dict)
data_1 = Data()
data_1.add_value(JsonValue('cond_int', 0))
data_1_dict = convert_data_to_v6_json(data_1).get_dict()
recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type,
input=data_1_dict)
condition = recipe_test_utils.create_recipe_condition(recipe=recipe,
save=True)
recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='node_a',
condition=condition,
save=True)
# Create message
message = create_process_condition_messages([condition.id])[0]
# Convert message to JSON and back, and then execute
message_json_dict = message.to_json()
new_message = ProcessCondition.from_json(message_json_dict)
result = new_message.execute()
self.assertTrue(result)
condition = RecipeCondition.objects.get(id=condition.id)
self.assertEqual(len(new_message.new_messages), 1)
self.assertEqual(new_message.new_messages[0].type, 'update_recipe')
self.assertEqual(new_message.new_messages[0].root_recipe_id, recipe.id)
self.assertTrue(condition.is_processed)
self.assertIsNotNone(condition.processed)
self.assertTrue(condition.is_accepted)
def test_json(self):
"""Tests converting an UpdateRecipe message to and from JSON"""
data_dict = convert_data_to_v6_json(Data()).get_dict()
job_failed = job_test_utils.create_job(status='FAILED', input=data_dict)
job_pending = job_test_utils.create_job(status='PENDING')
definition = RecipeDefinition(Interface())
definition.add_job_node('job_failed', job_failed.job_type.name, job_failed.job_type.version,
job_failed.job_type_rev.revision_num)
definition.add_job_node('job_pending', job_pending.job_type.name, job_pending.job_type.version,
job_pending.job_type_rev.revision_num)
definition.add_dependency('job_failed', 'job_pending')
definition_dict = convert_recipe_definition_to_v6_json(definition).get_dict()
recipe_type = recipe_test_utils.create_recipe_type_v6(definition=definition_dict)
recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type)
recipe_test_utils.create_recipe_job(recipe=recipe, job_name='job_failed', job=job_failed)
recipe_test_utils.create_recipe_job(recipe=recipe, job_name='job_pending', job=job_pending)
# Create message
message = create_update_recipe_message(recipe.id)
# Convert message to JSON and back, and then execute
message_json_dict = message.to_json()
new_message = UpdateRecipe.from_json(message_json_dict)
result = new_message.execute()
self.assertTrue(result)
# Check for message to set job_pending to BLOCKED
self.assertEqual(len(new_message.new_messages), 1)
msg = new_message.new_messages[0]
self.assertEqual(msg.type, 'blocked_jobs')
self.assertListEqual(msg._blocked_job_ids, [job_pending.id])
def queue_bake_jobs(self, request):
"""Creates and queues the specified number of Scale Bake jobs
:param request: the HTTP POST request
:type request: :class:`rest_framework.request.Request`
:rtype: :class:`rest_framework.response.Response`
:returns: the HTTP response to send back to the user
"""
num = rest_util.parse_int(request, 'num')
if num < 1:
raise BadParameter('num must be at least 1')
# TODO: in the future, send command message to do this asynchronously
try:
recipe_type = RecipeType.objects.get(name='scale-bake',
revision_num='1')
for _ in xrange(num):
Queue.objects.queue_new_recipe_for_user_v6(recipe_type, Data())
except (InvalidData, InvalidRecipeData, InactiveRecipeType) as ex:
message = 'Unable to create new recipe'
logger.exception(message)
raise BadParameter('%s: %s' % (message, unicode(ex)))
return Response(status=status.HTTP_202_ACCEPTED)
def test_json_forced_nodes(self):
"""Tests coverting a ProcessRecipeInput message to and from JSON with forced nodes provided"""
data_dict = convert_data_to_v6_json(Data()).get_dict()
recipe = recipe_test_utils.create_recipe(input=data_dict)
forced_nodes = ForcedNodes()
forced_nodes.set_all_nodes()
forced_nodes_dict = convert_forced_nodes_to_v6(forced_nodes).get_dict()
# Create message
message = create_process_recipe_input_messages(
[recipe.id], forced_nodes=forced_nodes)[0]
# Convert message to JSON and back, and then execute
message_json_dict = message.to_json()
new_message = ProcessRecipeInput.from_json(message_json_dict)
result = new_message.execute()
self.assertTrue(result)
recipe = Recipe.objects.get(id=recipe.id)
self.assertEqual(len(new_message.new_messages), 1)
msg = new_message.new_messages[0]
self.assertEqual(msg.type, 'update_recipe')
self.assertEqual(msg.root_recipe_id, recipe.id)
self.assertDictEqual(
convert_forced_nodes_to_v6(msg.forced_nodes).get_dict(),
forced_nodes_dict)
# Recipe should have input_file_size set to 0 (no input files)
self.assertEqual(recipe.input_file_size, 0.0)
def generate_input_data(self, recipe_input_data, node_outputs,
optional_outputs):
"""Generates the input data for this node
:param recipe_input_data: The input data for the recipe
:type recipe_input_data: :class:`data.data.data.Data`
:param node_outputs: The RecipeNodeOutput tuples stored in a dict by node name
:type node_outputs: dict
:returns: The input data for this node
:rtype: :class:`data.data.data.Data`
:raises :class:`data.data.exceptions.InvalidData`: If there is a duplicate data value
"""
input_data = Data()
for connection in self.connections.values():
try:
connection.add_value_to_data(input_data, recipe_input_data,
node_outputs)
except InvalidData as ex:
if optional_outputs:
if not connection.output_name in optional_outputs:
logger.warning(
"Output name %s not found in optional outputs",
connection.output_name)
raise
else:
logger.info(
"InvalidData exception occured due to optional output not present. Proceeding with job execution."
)
else:
logger.exception(
"InvalidData exception occurred generating input data")
return input_data
def test_execute_input_data(self):
"""Tests calling CreateJobs.execute() with input data"""
manifest = {
'seedVersion': '1.0.0',
'job': {
'name': 'name',
'jobVersion': '1.0.0',
'packageVersion': '1.0.0',
'title': 'Title',
'description': 'This is a description',
'maintainer': {
'name': 'John Doe',
'email': '*****@*****.**'
},
'timeout': 10,
'interface': {
'command': 'the command'
}
}
}
job_type = job_test_utils.create_seed_job_type(manifest=manifest)
event = trigger_test_utils.create_trigger_event()
data = Data()
# Create and execute message
message = create_jobs_message(job_type.name,
job_type.version,
job_type.revision_num,
event.id,
input_data=data)
result = message.execute()
self.assertTrue(result)
# Check for job creation
self.assertEqual(
Job.objects.filter(job_type_id=job_type.id,
event_id=event.id).count(), 1)
# Check for process_job_input message
self.assertEqual(len(message.new_messages), 1)
msg = message.new_messages[0]
self.assertEqual(msg.type, 'process_job_input')
# Test executing message again
message_json_dict = message.to_json()
message = CreateJobs.from_json(message_json_dict)
result = message.execute()
self.assertTrue(result)
# Check that a second job is not created
self.assertEqual(
Job.objects.filter(job_type_id=job_type.id,
event_id=event.id).count(), 1)
# Check for process_job_input message
self.assertEqual(len(message.new_messages), 1)
msg = message.new_messages[0]
self.assertEqual(msg.type, 'process_job_input')
def execute(self):
"""See :meth:`messaging.messages.message.CommandMessage.execute`
"""
from ingest.models import Ingest
ingest_job_type = Ingest.objects.get_ingest_job_type()
# Grab the ingest object
ingest = Ingest.objects.get(pk=self.ingest_id)
when = ingest.transfer_ended if ingest.transfer_ended else now()
desc = {'file_name': ingest.file_name}
event = None
ingest_id = ingest.id
with transaction.atomic():
# Create the appropriate triggerevent
if self.create_ingest_type == STRIKE_JOB_TYPE:
desc['strike_id'] = self.strike_id
event = TriggerEvent.objects.create_trigger_event('STRIKE_TRANSFER', None, desc, when)
elif self.create_ingest_type == SCAN_JOB_TYPE:
ingest_id = Ingest.objects.get(scan_id=self.scan_id, file_name=ingest.file_name).id
desc['scan_id'] = self.scan_id
event = TriggerEvent.objects.create_trigger_event('SCAN_TRANSFER', None, desc, when)
data = Data()
data.add_value(JsonValue('ingest_id', ingest_id))
data.add_value(JsonValue('workspace', ingest.workspace.name))
if ingest.new_workspace:
data.add_value(JsonValue('new_workspace', ingest.new_workspace.name))
ingest_job = None
with transaction.atomic():
ingest_job = Queue.objects.queue_new_job_v6(ingest_job_type, data, event)
ingest.job = ingest_job
ingest.status = 'QUEUED'
ingest.save()
# Send message to start processing job input (done outside the transaction to hope the job exists)
# This can cause a race condition with a slow DB.
job = Job.objects.get_details(ingest_job.id)
self.new_messages.extend(create_process_job_input_messages([job.id]))
return True
def test_add_value(self):
"""Tests calling Data.add_value()"""
data = Data()
file_value = FileValue('input_1', [123])
data.add_value(file_value)
json_value = JsonValue('input_2', {'foo': 'bar'})
data.add_value(json_value)
self.assertSetEqual(set(data.values.keys()), {'input_1', 'input_2'})
# Duplicate value
dup_value = FileValue('input_1', [123])
with self.assertRaises(InvalidData) as context:
data.add_value(dup_value)
self.assertEqual(context.exception.error.name, 'DUPLICATE_VALUE')
def test_execute_invalid_data(self):
"""Tests calling CreateJobs.execute() when the input data is invalid"""
manifest = {
'seedVersion': '1.0.0',
'job': {
'name': 'name',
'jobVersion': '1.0.0',
'packageVersion': '1.0.0',
'title': 'Title',
'description': 'This is a description',
'maintainer': {
'name': 'John Doe',
'email': '*****@*****.**'
},
'timeout': 10,
'interface': {
'command': 'the command',
'inputs': {
'files': [{
'name': 'input_a'
}]
}
}
}
}
job_type = job_test_utils.create_seed_job_type(manifest=manifest)
event = trigger_test_utils.create_trigger_event()
# Data does not provide required input_a so it is invalid
data_dict = convert_data_to_v6_json(Data()).get_dict()
# Create and execute message
message = create_jobs_message(job_type.name,
job_type.version,
job_type.revision_num,
event.id,
count=10,
input_data_dict=data_dict)
result = message.execute()
self.assertTrue(result)
self.assertEqual(
Job.objects.filter(job_type_id=job_type.id,
event_id=event.id).count(), 0)
# Should be no new messages
self.assertEqual(len(message.new_messages), 0)
def test_queue_job_timestamps(self):
"""Tests that job attributes are updated when a job is queued."""
data_dict = convert_data_to_v6_json(Data()).get_dict()
job = job_test_utils.create_job(num_exes=1,
status='CANCELED',
input=data_dict,
started=timezone.now(),
ended=timezone.now())
Job.objects.update_jobs_to_queued([job], timezone.now(), requeue=True)
job = Job.objects.get(pk=job.id)
self.assertEqual(job.status, 'QUEUED')
self.assertIsNotNone(job.queued)
self.assertIsNone(job.started)
self.assertIsNone(job.ended)
def execute(self):
"""See :meth:`messaging.messages.message.CommandMessage.execute`
"""
from ingest.models import Ingest
ingest_job_type = Ingest.objects.get_ingest_job_type()
# Grab the ingest object
ingest = Ingest.objects.get(pk=self.ingest_id)
when = ingest.transfer_ended if ingest.transfer_ended else now()
desc = {'file_name': ingest.file_name}
event = None
ingest_id = ingest.id
with transaction.atomic():
# Create the appropriate triggerevent
if self.create_ingest_type == STRIKE_JOB_TYPE:
desc['strike_id'] = self.strike_id
event = TriggerEvent.objects.create_trigger_event(
'STRIKE_TRANSFER', None, desc, when)
elif self.create_ingest_type == SCAN_JOB_TYPE:
ingest_id = Ingest.objects.get(scan_id=self.scan_id,
file_name=ingest.file_name).id
desc['scan_id'] = self.scan_id
event = TriggerEvent.objects.create_trigger_event(
'SCAN_TRANSFER', None, desc, when)
data = Data()
data.add_value(JsonValue('ingest_id', ingest_id))
data.add_value(JsonValue('workspace', ingest.workspace.name))
if ingest.new_workspace:
data.add_value(
JsonValue('new_workspace', ingest.new_workspace.name))
ingest_job = None
with transaction.atomic():
ingest_job = Queue.objects.queue_new_job_v6(
ingest_job_type, data, event)
ingest.job = ingest_job
ingest.status = 'QUEUED'
ingest.save()
return True
def test_json_no_recipe(self):
"""Tests converting a CreateJobs message (without a recipe) to and from JSON"""
manifest = {
'seedVersion': '1.0.0',
'job': {
'name': 'name',
'jobVersion': '1.0.0',
'packageVersion': '1.0.0',
'title': 'Title',
'description': 'This is a description',
'maintainer': {
'name': 'John Doe',
'email': '*****@*****.**'
},
'timeout': 10
}
}
job_type = job_test_utils.create_seed_job_type(manifest=manifest)
event = trigger_test_utils.create_trigger_event()
data_dict = convert_data_to_v6_json(Data()).get_dict()
count = 10
# Create message
message = create_jobs_message(job_type.name,
job_type.version,
job_type.revision_num,
event.id,
count=count,
input_data_dict=data_dict)
# Convert message to JSON and back, and then execute
message_json_dict = message.to_json()
new_message = CreateJobs.from_json(message_json_dict)
result = new_message.execute()
self.assertTrue(result)
self.assertEqual(
Job.objects.filter(job_type_id=job_type.id,
event_id=event.id).count(), count)
# Check for process_job_input messages
self.assertEqual(len(new_message.new_messages), count)
for msg in new_message.new_messages:
self.assertEqual(msg.type, 'process_job_input')
def test_json_forced_nodes(self):
"""Tests converting an UpdateRecipe message to and from JSON when forced nodes are provided"""
data_dict = convert_data_to_v6_json(Data()).get_dict()
job_completed = job_test_utils.create_job(status='COMPLETED', input=data_dict, output=data_dict)
sub_recipe_type = recipe_test_utils.create_recipe_type_v6()
definition = RecipeDefinition(Interface())
definition.add_job_node('job_completed', job_completed.job_type.name, job_completed.job_type.version,
job_completed.job_type_rev.revision_num)
definition.add_recipe_node('the_sub_recipe', sub_recipe_type.name, sub_recipe_type.revision_num)
definition.add_dependency('job_completed', 'the_sub_recipe')
definition_dict = convert_recipe_definition_to_v6_json(definition).get_dict()
recipe_type = recipe_test_utils.create_recipe_type_v6(definition=definition_dict)
recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type, input=data_dict)
recipe_test_utils.create_recipe_job(recipe=recipe, job_name='job_completed', job=job_completed)
forced_nodes = ForcedNodes()
sub_forced_nodes = ForcedNodes()
sub_forced_nodes.set_all_nodes()
forced_nodes.add_subrecipe('the_sub_recipe', sub_forced_nodes)
# Create message
message = create_update_recipe_message(recipe.id, forced_nodes=forced_nodes)
# Convert message to JSON and back, and then execute
message_json_dict = message.to_json()
new_message = UpdateRecipe.from_json(message_json_dict)
result = new_message.execute()
self.assertTrue(result)
# Check for message to create sub-recipe
self.assertEqual(len(new_message.new_messages), 1)
msg = new_message.new_messages[0]
self.assertEqual(msg.type, 'create_recipes')
self.assertEqual(msg.event_id, recipe.event_id)
msg_forced_nodes_dict = convert_forced_nodes_to_v6(msg.forced_nodes).get_dict()
expected_forced_nodes_dict = convert_forced_nodes_to_v6(forced_nodes).get_dict()
self.assertDictEqual(msg_forced_nodes_dict, expected_forced_nodes_dict)
self.assertEqual(msg.create_recipes_type, SUB_RECIPE_TYPE)
self.assertEqual(msg.recipe_id, recipe.id)
self.assertEqual(msg.root_recipe_id, recipe.root_superseded_recipe_id)
self.assertIsNone(msg.superseded_recipe_id)
sub = SubRecipe(sub_recipe_type.name, sub_recipe_type.revision_num, 'the_sub_recipe', True)
self.assertListEqual(msg.sub_recipes, [sub])
def generate_input_data(self, recipe_input_data, node_outputs):
"""Generates the input data for this node
:param recipe_input_data: The input data for the recipe
:type recipe_input_data: :class:`data.data.data.Data`
:param node_outputs: The RecipeNodeOutput tuples stored in a dict by node name
:type node_outputs: dict
:returns: The input data for this node
:rtype: :class:`data.data.data.Data`
:raises :class:`data.data.exceptions.InvalidData`: If there is a duplicate data value
"""
input_data = Data()
for connection in self.connections.values():
connection.add_value_to_data(input_data, recipe_input_data, node_outputs)
return input_data
def initialize_system():
"""Performs any necessary functions needed for initializing Scale"""
logger.info('Initializing system')
Scheduler.objects.initialize_scheduler()
# Make sure clock job has been created
clock_job_type = JobType.objects.get_clock_job_type()
count = Job.objects.filter(job_type_id=clock_job_type.id).count()
if not count:
logger.info('Queuing Scale Clock job')
with transaction.atomic():
init_event = TriggerEvent.objects.create_trigger_event(
'SCALE_INIT', None, {}, now())
job = Queue.objects.queue_new_job_v6(clock_job_type, Data(),
init_event)
CommandMessageManager().send_messages(
create_process_job_input_messages([job.id]))
def test_json(self):
"""Tests coverting a ProcessRecipeInput message to and from JSON"""
data_dict = convert_data_to_v6_json(Data()).get_dict()
recipe = recipe_test_utils.create_recipe(input=data_dict)
# Create message
message = ProcessRecipeInput()
message.recipe_id = recipe.id
# Convert message to JSON and back, and then execute
message_json_dict = message.to_json()
new_message = ProcessRecipeInput.from_json(message_json_dict)
result = new_message.execute()
self.assertTrue(result)
recipe = Recipe.objects.get(id=recipe.id)
self.assertEqual(len(new_message.new_messages), 1)
self.assertEqual(new_message.new_messages[0].type, 'update_recipes')
# Recipe should have input_file_size set to 0 (no input files)
self.assertEqual(recipe.input_file_size, 0.0)
def get_data(self):
"""Returns the data represented by this JSON
:returns: The data
:rtype: :class:`data.data.data.Data`:
"""
data = Data()
for name, file_ids in self._data['files'].items():
file_value = FileValue(name, file_ids)
data.add_value(file_value)
for name, json in self._data['json'].items():
json_value = JsonValue(name, json)
data.add_value(json_value)
return data
def test_execute_with_data(self):
"""Tests calling ProcessRecipeInput.execute() successfully when the recipe already has data populated"""
workspace = storage_test_utils.create_workspace()
file_1 = storage_test_utils.create_file(workspace=workspace,
file_size=10485760.0)
file_2 = storage_test_utils.create_file(workspace=workspace,
file_size=104857600.0)
file_3 = storage_test_utils.create_file(workspace=workspace,
file_size=987654321.0)
recipe_interface = Interface()
recipe_interface.add_parameter(FileParameter('input_a',
['text/plain']))
recipe_interface.add_parameter(
FileParameter('input_b', ['text/plain'], multiple=True))
definition = RecipeDefinition(recipe_interface)
definition_dict = convert_recipe_definition_to_v6_json(
definition).get_dict()
recipe_type = recipe_test_utils.create_recipe_type(
definition=definition_dict)
data = Data()
data.add_value(FileValue('input_a', [file_1.id]))
data.add_value(FileValue('input_b', [file_2.id, file_3.id]))
data_dict = convert_data_to_v6_json(data).get_dict()
recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type,
input=data_dict)
# Create message
message = ProcessRecipeInput()
message.recipe_id = recipe.id
# Execute message
result = message.execute()
self.assertTrue(result)
recipe = Recipe.objects.get(id=recipe.id)
# Check for update_recipes message
self.assertEqual(len(message.new_messages), 1)
self.assertEqual(message.new_messages[0].type, 'update_recipes')
# Check recipe for expected input_file_size
self.assertEqual(recipe.input_file_size, 1052.0)
# Make sure recipe input file models are created
recipe_input_files = RecipeInputFile.objects.filter(
recipe_id=recipe.id)
self.assertEqual(len(recipe_input_files), 3)
for recipe_input_file in recipe_input_files:
if recipe_input_file.input_file_id == file_1.id:
self.assertEqual(recipe_input_file.recipe_input, 'input_a')
elif recipe_input_file.input_file_id == file_2.id:
self.assertEqual(recipe_input_file.recipe_input, 'input_b')
elif recipe_input_file.input_file_id == file_3.id:
self.assertEqual(recipe_input_file.recipe_input, 'input_b')
else:
self.fail('Invalid input file ID: %s' %
recipe_input_file.input_file_id)
# Test executing message again
message_json_dict = message.to_json()
message = ProcessRecipeInput.from_json(message_json_dict)
result = message.execute()
self.assertTrue(result)
# Still should have update_recipes message
self.assertEqual(len(message.new_messages), 1)
self.assertEqual(message.new_messages[0].type, 'update_recipes')
# Make sure recipe input file models are unchanged
recipe_input_files = RecipeInputFile.objects.filter(
recipe_id=recipe.id)
self.assertEqual(len(recipe_input_files), 3)
def test_execute_with_recipe_legacy(self):
"""Tests calling ProcessRecipeInput.execute() successfully when a legacy sub-recipe has to get its data from its
recipe
"""
workspace = storage_test_utils.create_workspace()
file_1 = storage_test_utils.create_file(workspace=workspace,
file_size=104857600.0)
file_2 = storage_test_utils.create_file(workspace=workspace,
file_size=987654321.0)
file_3 = storage_test_utils.create_file(workspace=workspace,
file_size=65456.0)
file_4 = storage_test_utils.create_file(workspace=workspace,
file_size=24564165456.0)
manifest_a = {
'seedVersion': '1.0.0',
'job': {
'name': 'job-a',
'jobVersion': '1.0.0',
'packageVersion': '1.0.0',
'title': '',
'description': '',
'maintainer': {
'name': 'John Doe',
'email': '*****@*****.**'
},
'timeout': 10,
'interface': {
'command': '',
'inputs': {
'files': [],
'json': []
},
'outputs': {
'files': [{
'name': 'output_a',
'pattern': '*.png'
}]
}
}
}
}
job_type_a = job_test_utils.create_job_type(interface=manifest_a)
output_data_a = Data()
output_data_a.add_value(FileValue('output_a', [file_1.id]))
output_data_a_dict = convert_data_to_v6_json(output_data_a).get_dict()
manifest_b = {
'seedVersion': '1.0.0',
'job': {
'name': 'job-b',
'jobVersion': '1.0.0',
'packageVersion': '1.0.0',
'title': '',
'description': '',
'maintainer': {
'name': 'John Doe',
'email': '*****@*****.**'
},
'timeout': 10,
'interface': {
'command': '',
'inputs': {
'files': [],
'json': []
},
'outputs': {
'files': [{
'name': 'output_b',
'pattern': '*.png',
'multiple': True
}]
}
}
}
}
job_type_b = job_test_utils.create_job_type(interface=manifest_b)
output_data_b = Data()
output_data_b.add_value(
FileValue('output_b', [file_2.id, file_3.id, file_4.id]))
output_data_b_dict = convert_data_to_v6_json(output_data_b).get_dict()
job_a = job_test_utils.create_job(job_type=job_type_a,
num_exes=1,
status='COMPLETED',
output=output_data_a_dict)
job_b = job_test_utils.create_job(job_type=job_type_b,
num_exes=1,
status='COMPLETED',
output=output_data_b_dict)
sub_recipe_interface_c = Interface()
sub_recipe_interface_c.add_parameter(
FileParameter('input_a', ['image/png']))
sub_recipe_interface_c.add_parameter(
FileParameter('input_b', ['image/png'], multiple=True))
sub_recipe_def_c = RecipeDefinition(sub_recipe_interface_c)
sub_recipe_def_dict_c = convert_recipe_definition_to_v1_json(
sub_recipe_def_c).get_dict()
sub_recipe_type_c = recipe_test_utils.create_recipe_type(
definition=sub_recipe_def_dict_c)
sub_recipe_c = recipe_test_utils.create_recipe(
recipe_type=sub_recipe_type_c)
definition = RecipeDefinition(Interface())
definition.add_job_node('node_a', job_type_a.name, job_type_a.version,
job_type_a.revision_num)
definition.add_job_node('node_b', job_type_b.name, job_type_b.version,
job_type_b.revision_num)
definition.add_recipe_node('node_c', sub_recipe_type_c.name,
sub_recipe_type_c.revision_num)
definition.add_dependency('node_c', 'node_a')
definition.add_dependency_input_connection('node_c', 'input_a',
'node_a', 'output_a')
definition.add_dependency('node_c', 'node_b')
definition.add_dependency_input_connection('node_c', 'input_b',
'node_b', 'output_b')
def_dict = convert_recipe_definition_to_v6_json(definition).get_dict()
recipe_type = recipe_test_utils.create_recipe_type(definition=def_dict)
recipe_data_dict = {
'version': '1.0',
'input_data': [],
'workspace_id': workspace.id
}
recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type,
input=recipe_data_dict)
recipe_node_a = recipe_test_utils.create_recipe_node(
recipe=recipe, node_name='node_a', job=job_a)
recipe_node_b = recipe_test_utils.create_recipe_node(
recipe=recipe, node_name='node_b', job=job_b)
recipe_node_c = recipe_test_utils.create_recipe_node(
recipe=recipe, node_name='node_c', sub_recipe=sub_recipe_c)
RecipeNode.objects.bulk_create(
[recipe_node_a, recipe_node_b, recipe_node_c])
job_a.recipe = recipe
job_a.save()
job_b.recipe = recipe
job_b.save()
sub_recipe_c.recipe = recipe
sub_recipe_c.save()
# Create message
message = ProcessRecipeInput()
message.recipe_id = sub_recipe_c.id
# Execute message
result = message.execute()
self.assertTrue(result)
sub_recipe_c = Recipe.objects.get(id=sub_recipe_c.id)
# Check for update_recipes message
self.assertEqual(len(message.new_messages), 1)
self.assertEqual(message.new_messages[0].type, 'update_recipes')
# Check sub-recipe for expected input_file_size
self.assertEqual(sub_recipe_c.input_file_size, 24469.0)
# Check sub-recipe for expected input data
self.assertEqual(
sub_recipe_c.input['version'],
'1.0') # Should be legacy input data with workspace ID
self.assertEqual(sub_recipe_c.input['workspace_id'], workspace.id)
self.assertSetEqual(set(sub_recipe_c.get_input_data().values.keys()),
{'input_a', 'input_b'})
self.assertListEqual(
sub_recipe_c.get_input_data().values['input_a'].file_ids,
[file_1.id])
self.assertListEqual(
sub_recipe_c.get_input_data().values['input_b'].file_ids,
[file_2.id, file_3.id, file_4.id])
# Make sure sub-recipe input file models are created
input_files = RecipeInputFile.objects.filter(recipe_id=sub_recipe_c.id)
self.assertEqual(len(input_files), 4)
file_ids = {input_file.input_file_id for input_file in input_files}
self.assertSetEqual(file_ids,
{file_1.id, file_2.id, file_3.id, file_4.id})
# Test executing message again
message_json_dict = message.to_json()
message = ProcessRecipeInput.from_json(message_json_dict)
result = message.execute()
self.assertTrue(result)
# Still should have update_recipes message
self.assertEqual(len(message.new_messages), 1)
self.assertEqual(message.new_messages[0].type, 'update_recipes')
# Make sure recipe input file models are unchanged
input_files = RecipeInputFile.objects.filter(recipe_id=sub_recipe_c.id)
self.assertEqual(len(input_files), 4)
def test_convert_data_to_v6_json(self):
"""Tests calling convert_data_to_v6_json()"""
# Try interface with nothing set
data = Data()
json = convert_data_to_v6_json(data)
DataV6(data=json.get_dict(), do_validate=True) # Revalidate
# Try data with a variety of values
data = Data()
data.add_value(FileValue('input_a', [1234]))
data.add_value(FileValue('input_b', [1235, 1236]))
data.add_value(JsonValue('input_c', 'hello'))
data.add_value(JsonValue('input_d', 11.9))
json = convert_data_to_v6_json(data)
DataV6(data=json.get_dict(), do_validate=True) # Revalidate
self.assertSetEqual(set(json.get_data().values.keys()), {'input_a', 'input_b', 'input_c', 'input_d'})
def test_is_data_accepted(self):
"""Tests calling DataFilter.is_data_accepted()"""
data_filter = DataFilter(all=False)
data_filter.add_filter({'name': 'input_a', 'type': 'media-type', 'condition': '==', 'values': ['application/json']})
data_filter.add_filter({'name': 'input_b', 'type': 'string', 'condition': 'contains', 'values': ['abcde']})
data_filter.add_filter({'name': 'input_c', 'type': 'integer', 'condition': '>', 'values': ['0']})
data_filter.add_filter({'name': 'input_d', 'type': 'integer', 'condition': 'between', 'values': ['0', '100']})
data_filter.add_filter({'name': 'input_f', 'type': 'meta-data', 'condition': 'in', 'values': [['foo', 'baz']],
'fields': [['a', 'b']]})
data = Data()
file_value = FileValue('input_a', [self.file1.id])
data.add_value(file_value)
# first filter passes, so data is accepted if all is set to false
self.assertTrue(data_filter.is_data_accepted(data))
data_filter.all = True
# other filters fail so data is not accepted
self.assertFalse(data_filter.is_data_accepted(data))
# get other filters to pass
json_value = JsonValue('input_b', 'abcdefg')
data.add_value(json_value)
json_value = JsonValue('input_c', '10')
data.add_value(json_value)
json_value = JsonValue('input_d', 50)
data.add_value(json_value)
file_value = FileValue('input_f', [self.file2.id])
data.add_value(file_value)
self.assertTrue(data_filter.is_data_accepted(data))
def test_has_completed_false(self):
"""Tests calling Recipe.has_completed() when an entire recipe has not completed"""
data_dict = convert_data_to_v6_json(Data()).get_dict()
job_type = job_test_utils.create_seed_job_type()
sub_recipe_type = recipe_test_utils.create_recipe_type_v6()
definition = RecipeDefinition(Interface())
definition.add_job_node('A', job_type.name, job_type.version,
job_type.revision_num)
definition.add_recipe_node('B', sub_recipe_type.name,
sub_recipe_type.revision_num)
definition.add_job_node('C', job_type.name, job_type.version,
job_type.revision_num)
definition.add_job_node('D', job_type.name, job_type.version,
job_type.revision_num)
definition.add_job_node('E', job_type.name, job_type.version,
job_type.revision_num)
definition.add_job_node('F', job_type.name, job_type.version,
job_type.revision_num)
definition.add_recipe_node('G', sub_recipe_type.name,
sub_recipe_type.revision_num)
definition.add_job_node('H', job_type.name, job_type.version,
job_type.revision_num)
definition.add_dependency('A', 'C')
definition.add_dependency('A', 'E')
definition.add_dependency('A', 'H')
definition.add_dependency('C', 'D')
definition.add_dependency('G', 'H')
job_a = job_test_utils.create_job(job_type=job_type,
status='COMPLETED',
output=data_dict,
save=False)
job_c = job_test_utils.create_job(job_type=job_type,
status='COMPLETED',
output=data_dict,
save=False)
job_d = job_test_utils.create_job(job_type=job_type,
status='COMPLETED',
output=data_dict,
save=False)
job_e = job_test_utils.create_job(job_type=job_type,
status='COMPLETED',
output=data_dict,
save=False)
job_f = job_test_utils.create_job(job_type=job_type,
status='COMPLETED',
output=data_dict,
save=False)
job_h = job_test_utils.create_job(job_type=job_type,
status='COMPLETED',
output=data_dict,
save=False)
Job.objects.bulk_create([job_a, job_c, job_d, job_e, job_f, job_h])
recipe_b = recipe_test_utils.create_recipe(recipe_type=sub_recipe_type,
save=False)
recipe_b.is_completed = True
recipe_g = recipe_test_utils.create_recipe(recipe_type=sub_recipe_type,
save=False)
recipe_g.is_completed = False
Recipe.objects.bulk_create([recipe_b, recipe_g])
definition_json_dict = convert_recipe_definition_to_v6_json(
definition).get_dict()
recipe_type = recipe_test_utils.create_recipe_type_v6(
definition=definition_json_dict)
recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type)
recipe_node_a = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='A',
job=job_a,
save=False,
is_original=False)
recipe_node_c = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='C',
job=job_c,
save=False,
is_original=False)
recipe_node_d = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='D',
job=job_d,
save=False,
is_original=False)
recipe_node_e = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='E',
job=job_e,
save=False)
recipe_node_f = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='F',
job=job_f,
save=False)
recipe_node_h = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='H',
job=job_h,
save=False)
recipe_node_g = recipe_test_utils.create_recipe_node(
recipe=recipe,
node_name='G',
sub_recipe=recipe_g,
save=False,
is_original=False)
recipe_node_b = recipe_test_utils.create_recipe_node(
recipe=recipe, node_name='B', sub_recipe=recipe_b, save=False)
RecipeNode.objects.bulk_create([
recipe_node_a, recipe_node_b, recipe_node_c, recipe_node_d,
recipe_node_e, recipe_node_f, recipe_node_g, recipe_node_h
])
recipe_instance = Recipe.objects.get_recipe_instance(recipe.id)
self.assertFalse(recipe_instance.has_completed())
def test_validate(self):
"""Tests calling Data.validate()"""
interface = Interface()
data = Data()
interface.add_parameter(FileParameter('input_1', ['application/json']))
interface.add_parameter(JsonParameter('input_2', 'integer'))
data.add_value(FileValue('input_1', [123]))
data.add_value(JsonValue('input_2', 100))
data.add_value(JsonValue('extra_input_1', 'hello'))
data.add_value(JsonValue('extra_input_2', 'there'))
# Valid data
data.validate(interface)
# Ensure extra data values are removed
self.assertSetEqual(set(data.values.keys()), {'input_1', 'input_2'})
# Data is missing required input 3
interface.add_parameter(FileParameter('input_3', ['image/gif'], required=True))
with self.assertRaises(InvalidData) as context:
data.validate(interface)
self.assertEqual(context.exception.error.name, 'PARAM_REQUIRED')
data.add_value(FileValue('input_3', [999])) # Input 3 taken care of now
# Invalid data
interface.add_parameter(JsonParameter('input_4', 'string'))
mock_value = MagicMock()
mock_value.name = 'input_4'
mock_value.validate.side_effect = InvalidData('MOCK', '')
data.add_value(mock_value)
with self.assertRaises(InvalidData) as context:
data.validate(interface)
self.assertEqual(context.exception.error.name, 'MOCK')
def test_get_nodes_to_process_input(self):
"""Tests calling Recipe.get_nodes_to_process_input()"""
data_dict = convert_data_to_v6_json(Data()).get_dict()
job_type = job_test_utils.create_seed_job_type()
sub_recipe_type = recipe_test_utils.create_recipe_type_v6()
# Create recipe
definition = RecipeDefinition(Interface())
cond_interface_1 = Interface()
cond_interface_1.add_parameter(JsonParameter('cond_int', 'integer'))
definition.add_job_node('A', job_type.name, job_type.version,
job_type.revision_num)
df1 = DataFilter(filter_list=[{
'name': 'cond_int',
'type': 'integer',
'condition': '==',
'values': [0]
}, {
'name': 'cond_int',
'type': 'integer',
'condition': '!=',
'values': [0]
}],
all=False) #always True
df2 = DataFilter(filter_list=[{
'name': 'cond_int',
'type': 'integer',
'condition': '==',
'values': [0]
}, {
'name': 'cond_int',
'type': 'integer',
'condition': '!=',
'values': [0]
}],
all=True) #always False
definition.add_condition_node('B', cond_interface_1, df1) #True
definition.add_condition_node('C', cond_interface_1, df1) #True
definition.add_condition_node('D', cond_interface_1, df2) #False
definition.add_job_node('E', job_type.name, job_type.version,
job_type.revision_num)
definition.add_job_node('F', job_type.name, job_type.version,
job_type.revision_num)
definition.add_recipe_node('G', sub_recipe_type.name,
sub_recipe_type.revision_num)
definition.add_recipe_node('H', sub_recipe_type.name,
sub_recipe_type.revision_num)
definition.add_dependency('A', 'D')
definition.add_dependency('A', 'E')
definition.add_dependency('B', 'E')
definition.add_dependency('B', 'F')
definition.add_dependency('C', 'F')
definition.add_dependency('D', 'G')
definition.add_dependency('E', 'G')
definition.add_dependency('E', 'H')
definition_json_dict = convert_recipe_definition_to_v6_json(
definition).get_dict()
recipe_type = recipe_test_utils.create_recipe_type_v6(
definition=definition_json_dict)
recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type,
input=data_dict)
# Nodes A, B, and D already exist
job_a = job_test_utils.create_job(job_type=job_type,
status='COMPLETED',
input=data_dict,
output=data_dict,
save=True)
condition_b = recipe_test_utils.create_recipe_condition(
is_processed=True, is_accepted=True, save=False)
condition_d = recipe_test_utils.create_recipe_condition(
is_processed=True, is_accepted=False, save=False)
RecipeCondition.objects.bulk_create([condition_b, condition_d])
recipe_node_a = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='A',
job=job_a,
save=False)
recipe_node_b = recipe_test_utils.create_recipe_node(
recipe=recipe, node_name='B', condition=condition_b, save=False)
recipe_node_d = recipe_test_utils.create_recipe_node(
recipe=recipe, node_name='D', condition=condition_d, save=False)
RecipeNode.objects.bulk_create(
[recipe_node_a, recipe_node_b, recipe_node_d])
recipe_instance = Recipe.objects.get_recipe_instance(recipe.id)
nodes_to_process = recipe_instance.get_nodes_to_process_input()
self.assertSetEqual(set(nodes_to_process.keys()), {'C', 'E'})
def test_process_recipe_input(self):
"""Tests calling RecipeManager.process_recipe_input()"""
date_1 = now()
min_src_started_recipe_1 = date_1 - datetime.timedelta(days=200)
max_src_ended_recipe_1 = date_1 + datetime.timedelta(days=200)
date_2 = date_1 + datetime.timedelta(minutes=30)
date_3 = date_1 + datetime.timedelta(minutes=40)
date_4 = date_1 + datetime.timedelta(minutes=50)
min_src_started_recipe_2 = date_1 - datetime.timedelta(days=500)
max_src_ended_recipe_2 = date_1 + datetime.timedelta(days=500)
s_class = 'A'
s_sensor = '1'
collection = '12345'
task = 'abcd'
workspace = storage_test_utils.create_workspace()
file_1 = storage_test_utils.create_file(workspace=workspace, file_size=10485760.0,
source_sensor_class=s_class, source_sensor=s_sensor,
source_collection=collection, source_task=task)
file_2 = storage_test_utils.create_file(workspace=workspace, file_size=104857600.0,
source_started=date_2, source_ended=date_3,
source_sensor_class=s_class, source_sensor=s_sensor,
source_collection=collection, source_task=task)
file_3 = storage_test_utils.create_file(workspace=workspace, file_size=987654321.0,
source_started=min_src_started_recipe_1, source_ended=date_4)
file_4 = storage_test_utils.create_file(workspace=workspace, file_size=46546.0,
source_ended=max_src_ended_recipe_1)
file_5 = storage_test_utils.create_file(workspace=workspace, file_size=83457.0, source_started=date_2)
file_6 = storage_test_utils.create_file(workspace=workspace, file_size=42126588636633.0, source_ended=date_4)
file_7 = storage_test_utils.create_file(workspace=workspace, file_size=76645464662354.0)
file_8 = storage_test_utils.create_file(workspace=workspace, file_size=4654.0,
source_started=min_src_started_recipe_2)
file_9 = storage_test_utils.create_file(workspace=workspace, file_size=545.0, source_started=date_3,
source_ended=max_src_ended_recipe_2)
file_10 = storage_test_utils.create_file(workspace=workspace, file_size=0.154, source_ended=date_4,
source_sensor_class=s_class, source_sensor=s_sensor,
source_collection=collection, source_task=task)
recipe_interface = Interface()
recipe_interface.add_parameter(FileParameter('input_a', ['text/plain']))
recipe_interface.add_parameter(FileParameter('input_b', ['text/plain'], multiple=True))
definition = RecipeDefinition(recipe_interface)
definition_dict = convert_recipe_definition_to_v6_json(definition).get_dict()
recipe_type = recipe_test_utils.create_recipe_type_v6(definition=definition_dict)
data_1 = Data()
data_1.add_value(FileValue('input_a', [file_1.id]))
data_1.add_value(FileValue('input_b', [file_2.id, file_3.id, file_4.id, file_5.id]))
data_1_dict = convert_data_to_v6_json(data_1).get_dict()
data_2 = Data()
data_2.add_value(FileValue('input_a', [file_6.id]))
data_2.add_value(FileValue('input_b', [file_7.id, file_8.id, file_9.id, file_10.id]))
data_2_dict = convert_data_to_v6_json(data_2).get_dict()
data_3 = Data()
data_3_dict = convert_data_to_v6_json(data_3).get_dict()
recipe_1 = recipe_test_utils.create_recipe(recipe_type=recipe_type, input=data_1_dict)
recipe_2 = recipe_test_utils.create_recipe(recipe_type=recipe_type, input=data_2_dict)
recipe_3 = recipe_test_utils.create_recipe(recipe_type=recipe_type, input=data_3_dict)
# Execute method
Recipe.objects.process_recipe_input(recipe_1)
Recipe.objects.process_recipe_input(recipe_2)
Recipe.objects.process_recipe_input(recipe_3)
# Retrieve updated recipe models
recipes = Recipe.objects.filter(id__in=[recipe_1.id, recipe_2.id, recipe_3.id]).order_by('id')
recipe_1 = recipes[0]
recipe_2 = recipes[1]
recipe_3 = recipes[2]
# Check recipes for expected fields
self.assertEqual(recipe_1.input_file_size, 1053.0)
self.assertEqual(recipe_1.source_started, min_src_started_recipe_1)
self.assertEqual(recipe_1.source_ended, max_src_ended_recipe_1)
self.assertEqual(recipe_1.source_sensor_class, s_class)
self.assertEqual(recipe_1.source_sensor, s_sensor)
self.assertEqual(recipe_1.source_collection, collection)
self.assertEqual(recipe_1.source_task, task)
self.assertEqual(recipe_2.input_file_size, 113269857.0)
self.assertEqual(recipe_2.source_started, min_src_started_recipe_2)
self.assertEqual(recipe_2.source_ended, max_src_ended_recipe_2)
self.assertEqual(recipe_2.source_sensor_class, s_class)
self.assertEqual(recipe_2.source_sensor, s_sensor)
self.assertEqual(recipe_2.source_collection, collection)
self.assertEqual(recipe_2.source_task, task)
self.assertEqual(recipe_3.input_file_size, 0.0)
self.assertIsNone(recipe_3.source_started)
self.assertIsNone(recipe_3.source_ended)
# Make sure recipe input file models are created
recipe_input_files = RecipeInputFile.objects.filter(recipe_id=recipe_1.id)
self.assertEqual(len(recipe_input_files), 5)
input_files_dict = {'input_a': set(), 'input_b': set()}
for recipe_input_file in recipe_input_files:
input_files_dict[recipe_input_file.recipe_input].add(recipe_input_file.input_file_id)
self.assertDictEqual(input_files_dict, {'input_a': {file_1.id}, 'input_b': {file_2.id, file_3.id, file_4.id,
file_5.id}})
recipe_input_files = RecipeInputFile.objects.filter(recipe_id=recipe_2.id)
self.assertEqual(len(recipe_input_files), 5)
input_files_dict = {'input_a': set(), 'input_b': set()}
for recipe_input_file in recipe_input_files:
input_files_dict[recipe_input_file.recipe_input].add(recipe_input_file.input_file_id)
self.assertDictEqual(input_files_dict, {'input_a': {file_6.id}, 'input_b': {file_7.id, file_8.id, file_9.id,
file_10.id}})
self.assertEqual(RecipeInputFile.objects.filter(recipe_id=recipe_3.id).count(), 0)
def test_has_completed_true(self):
"""Tests calling Recipe.has_completed() when an entire recipe has completed"""
data_dict = convert_data_to_v6_json(Data()).get_dict()
job_type = job_test_utils.create_seed_job_type()
sub_recipe_type = recipe_test_utils.create_recipe_type_v6()
cond_interface_1 = Interface()
cond_interface_1.add_parameter(JsonParameter('cond_int', 'integer'))
df2 = DataFilter(filter_list=[{
'name': 'cond_int',
'type': 'integer',
'condition': '==',
'values': [0]
}, {
'name': 'cond_int',
'type': 'integer',
'condition': '!=',
'values': [0]
}],
all=True) #always False
definition = RecipeDefinition(Interface())
definition.add_job_node('A', job_type.name, job_type.version,
job_type.revision_num)
definition.add_recipe_node('B', sub_recipe_type.name,
sub_recipe_type.revision_num)
definition.add_job_node('C', job_type.name, job_type.version,
job_type.revision_num)
definition.add_job_node('D', job_type.name, job_type.version,
job_type.revision_num)
definition.add_job_node('E', job_type.name, job_type.version,
job_type.revision_num)
definition.add_job_node('F', job_type.name, job_type.version,
job_type.revision_num)
definition.add_recipe_node('G', sub_recipe_type.name,
sub_recipe_type.revision_num)
definition.add_job_node('H', job_type.name, job_type.version,
job_type.revision_num)
definition.add_condition_node('I', cond_interface_1, df2) #False
definition.add_job_node('J', job_type.name, job_type.version,
job_type.revision_num)
definition.add_dependency('A', 'C')
definition.add_dependency('A', 'E')
definition.add_dependency('A', 'H')
definition.add_dependency('C', 'D')
definition.add_dependency('G', 'H')
definition.add_dependency('A', 'I')
definition.add_dependency('I', 'J')
job_a = job_test_utils.create_job(job_type=job_type,
status='COMPLETED',
output=data_dict,
save=False)
job_c = job_test_utils.create_job(job_type=job_type,
status='COMPLETED',
output=data_dict,
save=False)
job_d = job_test_utils.create_job(job_type=job_type,
status='COMPLETED',
output=data_dict,
save=False)
job_e = job_test_utils.create_job(job_type=job_type,
status='COMPLETED',
output=data_dict,
save=False)
job_f = job_test_utils.create_job(job_type=job_type,
status='COMPLETED',
output=data_dict,
save=False)
job_h = job_test_utils.create_job(job_type=job_type,
status='COMPLETED',
output=data_dict,
save=False)
Job.objects.bulk_create([job_a, job_c, job_d, job_e, job_f, job_h])
condition_i = recipe_test_utils.create_recipe_condition(
is_processed=True, is_accepted=False, save=True)
recipe_b = recipe_test_utils.create_recipe(recipe_type=sub_recipe_type,
save=False)
recipe_b.is_completed = True
recipe_g = recipe_test_utils.create_recipe(recipe_type=sub_recipe_type,
save=False)
recipe_g.is_completed = True
Recipe.objects.bulk_create([recipe_b, recipe_g])
definition_json_dict = convert_recipe_definition_to_v6_json(
definition).get_dict()
recipe_type = recipe_test_utils.create_recipe_type_v6(
definition=definition_json_dict)
recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type)
recipe_node_a = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='A',
job=job_a,
save=False,
is_original=False)
recipe_node_c = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='C',
job=job_c,
save=False,
is_original=False)
recipe_node_d = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='D',
job=job_d,
save=False,
is_original=False)
recipe_node_e = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='E',
job=job_e,
save=False)
recipe_node_f = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='F',
job=job_f,
save=False)
recipe_node_h = recipe_test_utils.create_recipe_node(recipe=recipe,
node_name='H',
job=job_h,
save=False)
recipe_node_i = recipe_test_utils.create_recipe_node(
recipe=recipe, node_name='I', condition=condition_i, save=False)
recipe_node_g = recipe_test_utils.create_recipe_node(
recipe=recipe,
node_name='G',
sub_recipe=recipe_g,
save=False,
is_original=False)
recipe_node_b = recipe_test_utils.create_recipe_node(
recipe=recipe, node_name='B', sub_recipe=recipe_b, save=False)
RecipeNode.objects.bulk_create([
recipe_node_a, recipe_node_b, recipe_node_c, recipe_node_d,
recipe_node_e, recipe_node_f, recipe_node_g, recipe_node_h,
recipe_node_i
])
recipe_instance = Recipe.objects.get_recipe_instance(recipe.id)
self.assertTrue(recipe_instance.has_completed())
def execute(self):
"""See :meth:`messaging.messages.message.CommandMessage.execute`
"""
# Check to see if a force stop was placed on this purge process
results = PurgeResults.objects.get(trigger_event=self.trigger_id)
if results.force_stop_purge:
return True
files_to_delete = ScaleFile.objects.filter_files(job_ids=[self.job_id])
if files_to_delete:
# Construct input data
files = []
workspaces = []
for f in files_to_delete:
files.append({
'id': f.id,
'file_path': f.file_path,
'workspace': f.workspace.name
})
if f.workspace.name not in [
k for wrkspc in workspaces for k in wrkspc.keys()
]:
workspaces.append(
{f.workspace.name: f.workspace.json_config})
inputs = Data()
inputs.add_value(JsonValue('job_id', str(self.job_id)))
inputs.add_value(JsonValue('trigger_id', str(self.trigger_id)))
inputs.add_value(
JsonValue('source_file_id', str(self.source_file_id)))
inputs.add_value(JsonValue('purge', str(self.purge)))
inputs.add_value(JsonValue('files', json.dumps(files)))
inputs.add_value(JsonValue('workspaces', json.dumps(workspaces)))
# Send message to create system job to delete files
msg = create_jobs_message(job_type_name="scale-delete-files",
job_type_version="1.0.0",
event_id=self.trigger_id,
job_type_rev_num=1,
input_data=inputs)
self.new_messages.append(msg)
return True
