def test_json_forced_nodes(self):
"""Tests coverting a ProcessRecipeInput message to and from JSON with forced nodes provided"""
data_dict = convert_data_to_v6_json(Data()).get_dict()
recipe = recipe_test_utils.create_recipe(input=data_dict)
forced_nodes = ForcedNodes()
forced_nodes.set_all_nodes()
forced_nodes_dict = convert_forced_nodes_to_v6(forced_nodes).get_dict()
# Create message
message = create_process_recipe_input_messages(
[recipe.id], forced_nodes=forced_nodes)[0]
# Convert message to JSON and back, and then execute
message_json_dict = message.to_json()
new_message = ProcessRecipeInput.from_json(message_json_dict)
result = new_message.execute()
self.assertTrue(result)
recipe = Recipe.objects.get(id=recipe.id)
self.assertEqual(len(new_message.new_messages), 1)
msg = new_message.new_messages[0]
self.assertEqual(msg.type, 'update_recipe')
self.assertEqual(msg.root_recipe_id, recipe.id)
self.assertDictEqual(
convert_forced_nodes_to_v6(msg.forced_nodes).get_dict(),
forced_nodes_dict)
# Recipe should have input_file_size set to 0 (no input files)
self.assertEqual(recipe.input_file_size, 0.0)
def convert_definition_to_v6(definition):
"""Returns the v6 definition JSON for the given batch definition
:param definition: The batch definition
:type definition: :class:`batch.definition.definition.BatchDefinition`
:returns: The v6 definition JSON
:rtype: :class:`batch.definition.json.definition_v6.BatchDefinitionV6`
"""
json_dict = {'version': SCHEMA_VERSION}
if definition.root_batch_id is not None:
prev_batch_dict = {'root_batch_id': definition.root_batch_id}
if definition.forced_nodes:
prev_batch_dict['forced_nodes'] = convert_forced_nodes_to_v6(
definition.forced_nodes).get_dict()
json_dict['previous_batch'] = prev_batch_dict
if definition.dataset:
json_dict['dataset'] = definition.dataset
if definition.supersedes is not None:
json_dict['supersedes'] = definition.supersedes
if definition.forced_nodes:
json_dict['forced_nodes'] = convert_forced_nodes_to_v6(
definition.forced_nodes).get_dict()
return BatchDefinitionV6(definition=json_dict, do_validate=False)
def to_json(self):
"""See :meth:`messaging.messages.message.CommandMessage.to_json`
"""
json_dict = {'recipe_id': self.recipe_id}
if self.forced_nodes:
json_dict['forced_nodes'] = convert_forced_nodes_to_v6(self.forced_nodes).get_dict()
return json_dict
def test_json_forced_nodes(self):
"""Tests converting an UpdateRecipe message to and from JSON when forced nodes are provided"""
data_dict = convert_data_to_v6_json(Data()).get_dict()
job_completed = job_test_utils.create_job(status='COMPLETED', input=data_dict, output=data_dict)
sub_recipe_type = recipe_test_utils.create_recipe_type_v6()
definition = RecipeDefinition(Interface())
definition.add_job_node('job_completed', job_completed.job_type.name, job_completed.job_type.version,
job_completed.job_type_rev.revision_num)
definition.add_recipe_node('the_sub_recipe', sub_recipe_type.name, sub_recipe_type.revision_num)
definition.add_dependency('job_completed', 'the_sub_recipe')
definition_dict = convert_recipe_definition_to_v6_json(definition).get_dict()
recipe_type = recipe_test_utils.create_recipe_type_v6(definition=definition_dict)
recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type, input=data_dict)
recipe_test_utils.create_recipe_job(recipe=recipe, job_name='job_completed', job=job_completed)
forced_nodes = ForcedNodes()
sub_forced_nodes = ForcedNodes()
sub_forced_nodes.set_all_nodes()
forced_nodes.add_subrecipe('the_sub_recipe', sub_forced_nodes)
# Create message
message = create_update_recipe_message(recipe.id, forced_nodes=forced_nodes)
# Convert message to JSON and back, and then execute
message_json_dict = message.to_json()
new_message = UpdateRecipe.from_json(message_json_dict)
result = new_message.execute()
self.assertTrue(result)
# Check for message to create sub-recipe
self.assertEqual(len(new_message.new_messages), 1)
msg = new_message.new_messages[0]
self.assertEqual(msg.type, 'create_recipes')
self.assertEqual(msg.event_id, recipe.event_id)
msg_forced_nodes_dict = convert_forced_nodes_to_v6(msg.forced_nodes).get_dict()
expected_forced_nodes_dict = convert_forced_nodes_to_v6(forced_nodes).get_dict()
self.assertDictEqual(msg_forced_nodes_dict, expected_forced_nodes_dict)
self.assertEqual(msg.create_recipes_type, SUB_RECIPE_TYPE)
self.assertEqual(msg.recipe_id, recipe.id)
self.assertEqual(msg.root_recipe_id, recipe.root_superseded_recipe_id)
self.assertIsNone(msg.superseded_recipe_id)
sub = SubRecipe(sub_recipe_type.name, sub_recipe_type.revision_num, 'the_sub_recipe', True)
self.assertListEqual(msg.sub_recipes, [sub])
def to_json(self):
"""See :meth:`messaging.messages.message.CommandMessage.to_json`
"""
json_dict = {
'event_id': self.event_id,
'create_recipes_type': self.create_recipes_type
}
if self.batch_id:
json_dict['batch_id'] = self.batch_id
if self.forced_nodes:
json_dict['forced_nodes'] = convert_forced_nodes_to_v6(
self.forced_nodes).get_dict()
if self.create_recipes_type == NEW_RECIPE_TYPE:
if self.ingest_event_id:
json_dict['ingest_event_id'] = self.ingest_event_id
json_dict['recipe_input_data'] = self.recipe_input_data
json_dict['recipe_type_name'] = self.recipe_type_name
json_dict['recipe_type_rev_num'] = self.recipe_type_rev_num
json_dict['recipe_configuration'] = self.configuration
json_dict['new_recipes'] = self.new_recipes
elif self.create_recipes_type == REPROCESS_TYPE:
json_dict['recipe_type_name'] = self.recipe_type_name
json_dict['recipe_type_rev_num'] = self.recipe_type_rev_num
json_dict['root_recipe_ids'] = self.root_recipe_ids
elif self.create_recipes_type == SUB_RECIPE_TYPE:
json_dict['recipe_id'] = self.recipe_id
if self.root_recipe_id:
json_dict['root_recipe_id'] = self.root_recipe_id
if self.superseded_recipe_id:
json_dict['superseded_recipe_id'] = self.superseded_recipe_id
sub_recipes = []
for sub_recipe in self.sub_recipes:
sub_recipes.append({
'recipe_type_name': sub_recipe.recipe_type_name,
'recipe_type_rev_num': sub_recipe.recipe_type_rev_num,
'node_name': sub_recipe.node_name,
'process_input': sub_recipe.process_input
})
json_dict['sub_recipes'] = sub_recipes
return json_dict
def test_convert_forced_nodes_to_v6_full(self):
"""Tests calling convert_forced_nodes_to_v6() with a full diff with all types (deleted, new, changed, etc) of nodes"""
recipe_d_forced_nodes = ForcedNodes()
recipe_d_forced_nodes.add_node('1')
recipe_d_forced_nodes.add_node('2')
top_forced_nodes = ForcedNodes()
top_forced_nodes.add_node('C')
top_forced_nodes.add_subrecipe('D', recipe_d_forced_nodes)
v6 = convert_forced_nodes_to_v6(top_forced_nodes)
full = {
'version': '7',
'all': False,
'nodes': [u'C', u'D'],
'sub_recipes': {
'D': {
'version': '7',
'all': False,
'nodes': ['1', '2']
}
}
}
self.assertDictEqual(v6.get_dict(), full)
def test_execute(self):
"""Tests calling CompletedJobs.execute() successfully"""
from recipe.test import utils as recipe_test_utils
recipe_1 = recipe_test_utils.create_recipe()
job_1 = job_test_utils.create_job(num_exes=1, status='QUEUED')
job_test_utils.create_job_exe(job=job_1)
job_2 = job_test_utils.create_job(num_exes=1,
status='RUNNING',
recipe=recipe_1)
job_test_utils.create_job_exe(job=job_2, output=JobResults())
job_3 = job_test_utils.create_job(num_exes=0, status='PENDING')
job_ids = [job_1.id, job_2.id, job_3.id]
recipe_test_utils.create_recipe_job(recipe=recipe_1, job=job_2)
when_ended = now()
# Add jobs to message
message = CompletedJobs()
message.ended = when_ended
if message.can_fit_more():
message.add_completed_job(CompletedJob(job_1.id, job_1.num_exes))
if message.can_fit_more():
message.add_completed_job(CompletedJob(job_2.id, job_2.num_exes))
if message.can_fit_more():
message.add_completed_job(CompletedJob(job_3.id, job_3.num_exes))
# Execute message
result = message.execute()
self.assertTrue(result)
from recipe.diff.forced_nodes import ForcedNodes
from recipe.diff.json.forced_nodes_v6 import convert_forced_nodes_to_v6
forced_nodes = ForcedNodes()
forced_nodes.set_all_nodes()
forced_nodes_dict = convert_forced_nodes_to_v6(forced_nodes).get_dict()
jobs = Job.objects.filter(id__in=job_ids).order_by('id')
self.assertEqual(len(message.new_messages), 3)
update_recipe_metrics_msg = None
update_recipe_msg = None
publish_job_msg = None
for msg in message.new_messages:
if msg.type == 'update_recipe':
update_recipe_msg = msg
elif msg.type == 'publish_job':
publish_job_msg = msg
elif msg.type == 'update_recipe_metrics':
update_recipe_metrics_msg = msg
self.assertIsNotNone(update_recipe_msg)
self.assertIsNotNone(publish_job_msg)
self.assertIsNotNone(update_recipe_metrics_msg)
self.assertEqual(publish_job_msg.job_id, job_2.id)
# Job 1 should be completed
self.assertEqual(jobs[0].status, 'COMPLETED')
self.assertEqual(jobs[0].num_exes, 1)
self.assertEqual(jobs[0].ended, when_ended)
# Job 2 should be completed and has output, so should be in update_recipe message
self.assertEqual(jobs[1].status, 'COMPLETED')
self.assertEqual(jobs[1].num_exes, 1)
self.assertEqual(jobs[1].ended, when_ended)
self.assertEqual(update_recipe_msg.root_recipe_id, recipe_1.id)
self.assertDictEqual(
convert_forced_nodes_to_v6(
update_recipe_msg.forced_nodes).get_dict(), forced_nodes_dict)
# Job 3 should ignore update
self.assertEqual(jobs[2].status, 'PENDING')
self.assertEqual(jobs[2].num_exes, 0)
# Test executing message again
new_ended = when_ended + datetime.timedelta(minutes=5)
message_json_dict = message.to_json()
message = CompletedJobs.from_json(message_json_dict)
message.ended = new_ended
result = message.execute()
self.assertTrue(result)
# Should have the same messages as before
jobs = Job.objects.filter(id__in=job_ids).order_by('id')
self.assertEqual(len(message.new_messages), 3)
update_recipe_metrics_msg = None
update_recipe_msg = None
publish_job_msg = None
for msg in message.new_messages:
if msg.type == 'update_recipe':
update_recipe_msg = msg
elif msg.type == 'publish_job':
publish_job_msg = msg
elif msg.type == 'update_recipe_metrics':
update_recipe_metrics_msg = msg
self.assertIsNotNone(update_recipe_msg)
self.assertIsNotNone(publish_job_msg)
self.assertIsNotNone(update_recipe_metrics_msg)
self.assertEqual(publish_job_msg.job_id, job_2.id)
# Job 1 should be completed
self.assertEqual(jobs[0].status, 'COMPLETED')
self.assertEqual(jobs[0].num_exes, 1)
self.assertEqual(jobs[0].ended, when_ended)
# Job 2 should be completed and has output, so should be in update_recipe message
self.assertEqual(jobs[1].status, 'COMPLETED')
self.assertEqual(jobs[1].num_exes, 1)
self.assertEqual(jobs[1].ended, when_ended)
self.assertEqual(update_recipe_msg.root_recipe_id, recipe_1.id)
self.assertDictEqual(
convert_forced_nodes_to_v6(
update_recipe_msg.forced_nodes).get_dict(), forced_nodes_dict)
# Job 3 should ignore update
self.assertEqual(jobs[2].status, 'PENDING')
self.assertEqual(jobs[2].num_exes, 0)
def test_execute(self):
"""Tests calling CancelJobs.execute() successfully"""
when = now()
data = JobData()
from recipe.test import utils as recipe_test_utils
recipe = recipe_test_utils.create_recipe()
job_type = job_test_utils.create_seed_job_type()
job_1 = job_test_utils.create_job(job_type=job_type,
num_exes=3,
status='FAILED',
input=data.get_dict(),
recipe=recipe)
job_2 = job_test_utils.create_job(job_type=job_type,
num_exes=3,
status='CANCELED',
input=data.get_dict(),
recipe=recipe)
job_3 = job_test_utils.create_job(job_type=job_type,
num_exes=1,
status='COMPLETED',
input=data.get_dict(),
recipe=recipe)
job_4 = job_test_utils.create_job(job_type=job_type,
num_exes=0,
status='PENDING',
recipe=recipe)
job_ids = [job_1.id, job_2.id, job_3.id, job_4.id]
recipe_test_utils.create_recipe_job(recipe=recipe,
job_name='job_1',
job=job_1)
recipe_test_utils.create_recipe_job(recipe=recipe,
job_name='job_2',
job=job_2)
recipe_test_utils.create_recipe_job(recipe=recipe,
job_name='job_3',
job=job_3)
recipe_test_utils.create_recipe_job(recipe=recipe,
job_name='job_4',
job=job_4)
# Add jobs to message
message = CancelJobs()
message.when = when
if message.can_fit_more():
message.add_job(job_1.id)
if message.can_fit_more():
message.add_job(job_2.id)
if message.can_fit_more():
message.add_job(job_3.id)
if message.can_fit_more():
message.add_job(job_4.id)
# Execute message
result = message.execute()
self.assertTrue(result)
jobs = Job.objects.filter(id__in=job_ids).order_by('id')
# Job 1 should have been canceled
self.assertEqual(jobs[0].status, 'CANCELED')
self.assertEqual(jobs[0].last_status_change, when)
# Job 2 was already canceled
self.assertEqual(jobs[1].status, 'CANCELED')
self.assertNotEqual(jobs[1].last_status_change, when)
# Job 3 was already COMPLETED, so can't be canceled
self.assertEqual(jobs[2].status, 'COMPLETED')
self.assertNotEqual(jobs[2].last_status_change, when)
# Job 4 should have been canceled
self.assertEqual(jobs[3].status, 'CANCELED')
self.assertEqual(jobs[3].last_status_change, when)
from recipe.diff.forced_nodes import ForcedNodes
from recipe.diff.json.forced_nodes_v6 import convert_forced_nodes_to_v6
forced_nodes = ForcedNodes()
forced_nodes.set_all_nodes()
forced_nodes_dict = convert_forced_nodes_to_v6(forced_nodes).get_dict()
# Should be messages to update recipe and update recipe metrics after canceling jobs
self.assertEqual(len(message.new_messages), 2)
update_recipe_msg = None
update_recipe_metrics_msg = None
for msg in message.new_messages:
if msg.type == 'update_recipe':
update_recipe_msg = msg
elif msg.type == 'update_recipe_metrics':
update_recipe_metrics_msg = msg
self.assertIsNotNone(update_recipe_msg)
self.assertIsNotNone(update_recipe_metrics_msg)
self.assertEqual(update_recipe_msg.root_recipe_id, recipe.id)
self.assertDictEqual(
convert_forced_nodes_to_v6(
update_recipe_msg.forced_nodes).get_dict(), forced_nodes_dict)
self.assertListEqual(update_recipe_metrics_msg._recipe_ids,
[recipe.id])
# Test executing message again
message.new_messages = []
result = message.execute()
self.assertTrue(result)
# All results should be the same
jobs = Job.objects.filter(id__in=job_ids).order_by('id')
# Job 1 should have been canceled
self.assertEqual(jobs[0].status, 'CANCELED')
self.assertEqual(jobs[0].last_status_change, when)
# Job 2 was already canceled
self.assertEqual(jobs[1].status, 'CANCELED')
self.assertNotEqual(jobs[1].last_status_change, when)
# Job 3 was already COMPLETED, so can't be canceled
self.assertEqual(jobs[2].status, 'COMPLETED')
self.assertNotEqual(jobs[2].last_status_change, when)
# Job 4 should have been canceled
self.assertEqual(jobs[3].status, 'CANCELED')
self.assertEqual(jobs[3].last_status_change, when)
# Should be messages to update recipe and update recipe metrics after canceling jobs
self.assertEqual(len(message.new_messages), 2)
update_recipe_msg = None
update_recipe_metrics_msg = None
for msg in message.new_messages:
if msg.type == 'update_recipe':
update_recipe_msg = msg
elif msg.type == 'update_recipe_metrics':
update_recipe_metrics_msg = msg
self.assertIsNotNone(update_recipe_msg)
self.assertIsNotNone(update_recipe_metrics_msg)
self.assertEqual(update_recipe_msg.root_recipe_id, recipe.id)
self.assertDictEqual(
convert_forced_nodes_to_v6(
update_recipe_msg.forced_nodes).get_dict(), forced_nodes_dict)
self.assertListEqual(update_recipe_metrics_msg._recipe_ids,
[recipe.id])
def test_execute_with_top_level_recipe(self):
"""Tests calling UpdateRecipeMetrics.execute() successfully where messages need to be sent to update a top-level
recipe
"""
batch = batch_test_utils.create_batch()
top_recipe = recipe_test_utils.create_recipe(batch=batch)
recipe = recipe_test_utils.create_recipe(batch=batch)
recipe.recipe = top_recipe
recipe.root_recipe = top_recipe
recipe.save()
recipe_node_1 = recipe_test_utils.create_recipe_node(recipe=top_recipe,
sub_recipe=recipe)
forced_nodes = ForcedNodes()
forced_nodes.set_all_nodes()
forced_nodes_dict = convert_forced_nodes_to_v6(forced_nodes).get_dict()
# Recipe jobs
job_1 = job_test_utils.create_job(status='FAILED', save=False)
job_2 = job_test_utils.create_job(status='CANCELED', save=False)
job_3 = job_test_utils.create_job(status='BLOCKED', save=False)
job_4 = job_test_utils.create_job(status='BLOCKED', save=False)
job_5 = job_test_utils.create_job(status='COMPLETED', save=False)
Job.objects.bulk_create([job_1, job_2, job_3, job_4, job_5])
# Recipe nodes
recipe_node_2 = recipe_test_utils.create_recipe_node(recipe=recipe,
job=job_1)
recipe_node_3 = recipe_test_utils.create_recipe_node(recipe=recipe,
job=job_2)
recipe_node_4 = recipe_test_utils.create_recipe_node(recipe=recipe,
job=job_3)
recipe_node_5 = recipe_test_utils.create_recipe_node(recipe=recipe,
job=job_4)
recipe_node_6 = recipe_test_utils.create_recipe_node(recipe=recipe,
job=job_5)
RecipeNode.objects.bulk_create([
recipe_node_1, recipe_node_2, recipe_node_3, recipe_node_4,
recipe_node_5, recipe_node_6
])
# Add recipes to message
message = UpdateRecipeMetrics()
if message.can_fit_more():
message.add_recipe(recipe.id)
# Execute message
result = message.execute()
self.assertTrue(result)
recipe = Recipe.objects.get(id=recipe.id)
self.assertEqual(recipe.jobs_total, 5)
self.assertEqual(recipe.jobs_pending, 0)
self.assertEqual(recipe.jobs_blocked, 2)
self.assertEqual(recipe.jobs_queued, 0)
self.assertEqual(recipe.jobs_running, 0)
self.assertEqual(recipe.jobs_failed, 1)
self.assertEqual(recipe.jobs_completed, 1)
self.assertEqual(recipe.jobs_canceled, 1)
self.assertEqual(recipe.sub_recipes_total, 0)
self.assertEqual(recipe.sub_recipes_completed, 0)
# Make sure message is created to update top-level recipe and recipe metrics
# There should be no message to update batch metrics since we did not update a top-level recipe
self.assertEqual(len(message.new_messages), 2)
update_recipe_metrics_msg = message.new_messages[0]
update_recipe_msg = message.new_messages[1]
self.assertEqual(update_recipe_metrics_msg.type,
'update_recipe_metrics')
self.assertListEqual(update_recipe_metrics_msg._recipe_ids,
[top_recipe.id])
self.assertEqual(update_recipe_msg.type, 'update_recipe')
self.assertEqual(update_recipe_msg.root_recipe_id, top_recipe.id)
self.assertDictEqual(
convert_forced_nodes_to_v6(
update_recipe_msg.forced_nodes).get_dict(), forced_nodes_dict)
# Test executing message again
message_json_dict = message.to_json()
message = UpdateRecipeMetrics.from_json(message_json_dict)
result = message.execute()
self.assertTrue(result)
recipe = Recipe.objects.get(id=recipe.id)
self.assertEqual(recipe.jobs_total, 5)
self.assertEqual(recipe.jobs_pending, 0)
self.assertEqual(recipe.jobs_blocked, 2)
self.assertEqual(recipe.jobs_queued, 0)
self.assertEqual(recipe.jobs_running, 0)
self.assertEqual(recipe.jobs_failed, 1)
self.assertEqual(recipe.jobs_completed, 1)
self.assertEqual(recipe.jobs_canceled, 1)
self.assertEqual(recipe.sub_recipes_total, 0)
self.assertEqual(recipe.sub_recipes_completed, 0)
# Make sure message is created to update top-level recipe and recipe metrics
# There should be no message to update batch metrics since we did not update a top-level recipe
self.assertEqual(len(message.new_messages), 2)
update_recipe_metrics_msg = message.new_messages[0]
update_recipe_msg = message.new_messages[1]
self.assertEqual(update_recipe_metrics_msg.type,
'update_recipe_metrics')
self.assertListEqual(update_recipe_metrics_msg._recipe_ids,
[top_recipe.id])
self.assertEqual(update_recipe_msg.type, 'update_recipe')
self.assertEqual(update_recipe_msg.root_recipe_id, top_recipe.id)
self.assertDictEqual(
convert_forced_nodes_to_v6(
update_recipe_msg.forced_nodes).get_dict(), forced_nodes_dict)
def test_execute(self):
"""Tests calling UncancelJobs.execute() successfully"""
old_when = now()
when = old_when + datetime.timedelta(minutes=60)
recipe = recipe_test_utils.create_recipe()
job_1 = job_test_utils.create_job(num_exes=0,
status='PENDING',
last_status_change=old_when)
job_2 = job_test_utils.create_job(num_exes=0,
status='CANCELED',
last_status_change=old_when,
recipe=recipe)
job_3 = job_test_utils.create_job(num_exes=1,
status='CANCELED',
last_status_change=old_when)
job_4 = job_test_utils.create_job(num_exes=1,
status='FAILED',
last_status_change=old_when)
job_ids = [job_1.id, job_2.id, job_3.id, job_4.id]
recipe_test_utils.create_recipe_job(recipe=recipe, job=job_2)
# Add jobs to message
message = UncancelJobs()
message.when = when
if message.can_fit_more():
message.add_job(job_1.id)
if message.can_fit_more():
message.add_job(job_2.id)
if message.can_fit_more():
message.add_job(job_3.id)
if message.can_fit_more():
message.add_job(job_4.id)
# Execute message
result = message.execute()
self.assertTrue(result)
from recipe.diff.forced_nodes import ForcedNodes
from recipe.diff.json.forced_nodes_v6 import convert_forced_nodes_to_v6
forced_nodes = ForcedNodes()
forced_nodes.set_all_nodes()
forced_nodes_dict = convert_forced_nodes_to_v6(forced_nodes).get_dict()
jobs = Job.objects.filter(id__in=job_ids).order_by('id')
# Job 1 should not be updated because it was not CANCELED
self.assertEqual(jobs[0].status, 'PENDING')
self.assertEqual(jobs[0].last_status_change, old_when)
# Job 2 should be uncanceled
self.assertEqual(jobs[1].status, 'PENDING')
self.assertEqual(jobs[1].last_status_change, when)
# Job 3 should not be updated since it has already been queued
self.assertEqual(jobs[2].status, 'CANCELED')
self.assertEqual(jobs[2].last_status_change, old_when)
# Job 4 should not be updated because it was not CANCELED
self.assertEqual(jobs[3].status, 'FAILED')
self.assertEqual(jobs[3].last_status_change, old_when)
# Make sure update_recipe and update_recipe_metrics messages were created
self.assertEqual(len(message.new_messages), 2)
update_recipe_msg = None
update_recipe_metrics_msg = None
for msg in message.new_messages:
if msg.type == 'update_recipe':
update_recipe_msg = msg
elif msg.type == 'update_recipe_metrics':
update_recipe_metrics_msg = msg
self.assertIsNotNone(update_recipe_msg)
self.assertIsNotNone(update_recipe_metrics_msg)
self.assertEqual(update_recipe_msg.root_recipe_id, recipe.id)
self.assertDictEqual(
convert_forced_nodes_to_v6(
update_recipe_msg.forced_nodes).get_dict(), forced_nodes_dict)
self.assertListEqual(update_recipe_metrics_msg._recipe_ids,
[recipe.id])
# Test executing message again
newer_when = when + datetime.timedelta(minutes=60)
message_json_dict = message.to_json()
message = UncancelJobs.from_json(message_json_dict)
message.when = newer_when
result = message.execute()
self.assertTrue(result)
jobs = Job.objects.filter(id__in=job_ids).order_by('id')
# Job 1 should not be updated because it was not CANCELED
self.assertEqual(jobs[0].status, 'PENDING')
self.assertEqual(jobs[0].last_status_change, old_when)
# Job 2 should not be updated since it already was last mexxage execution
self.assertEqual(jobs[1].status, 'PENDING')
self.assertEqual(jobs[1].last_status_change, when)
# Job 3 should not be updated since it has already been queued
self.assertEqual(jobs[2].status, 'CANCELED')
self.assertEqual(jobs[2].last_status_change, old_when)
# Job 4 should not be updated because it was not CANCELED
self.assertEqual(jobs[3].status, 'FAILED')
self.assertEqual(jobs[3].last_status_change, old_when)
# Make sure update_recipe and update_recipe_metrics messages were created
self.assertEqual(len(message.new_messages), 2)
update_recipe_msg = None
update_recipe_metrics_msg = None
for msg in message.new_messages:
if msg.type == 'update_recipe':
update_recipe_msg = msg
elif msg.type == 'update_recipe_metrics':
update_recipe_metrics_msg = msg
self.assertIsNotNone(update_recipe_msg)
self.assertIsNotNone(update_recipe_metrics_msg)
self.assertEqual(update_recipe_msg.root_recipe_id, recipe.id)
self.assertDictEqual(
convert_forced_nodes_to_v6(
update_recipe_msg.forced_nodes).get_dict(), forced_nodes_dict)
self.assertListEqual(update_recipe_metrics_msg._recipe_ids,
[recipe.id])
def test_convert_forced_nodes_to_v6_empty(self):
"""Tests calling convert_forced_nodes_to_v6() with an empty forced nodes object"""
empty = ForcedNodes()
v6 = convert_forced_nodes_to_v6(empty)
self.assertDictEqual(v6.get_dict(), {'version': '7', 'all': False})
def test_execute(self):
"""Tests calling UpdateRecipe.execute() successfully"""
data_dict = convert_data_to_v6_json(Data()).get_dict()
job_a = job_test_utils.create_job(status='COMPLETED', input=data_dict, output=data_dict)
recipe_b = recipe_test_utils.create_recipe()
job_c = job_test_utils.create_job(status='PENDING')
job_d = job_test_utils.create_job(status='BLOCKED')
recipe_type_e = recipe_test_utils.create_recipe_type_v6()
job_type_f = job_test_utils.create_seed_job_type()
job_type_k = job_test_utils.create_seed_job_type()
job_g = job_test_utils.create_job(status='FAILED', input=data_dict)
job_h = job_test_utils.create_job(status='PENDING')
condition_i = recipe_test_utils.create_recipe_condition(save=True)
definition = RecipeDefinition(Interface())
definition.add_job_node('node_a', job_a.job_type.name, job_a.job_type.version, job_a.job_type_rev.revision_num)
definition.add_recipe_node('node_b', recipe_b.recipe_type.name, recipe_b.recipe_type.revision_num)
definition.add_job_node('node_c', job_c.job_type.name, job_c.job_type.version, job_c.job_type_rev.revision_num)
definition.add_job_node('node_d', job_d.job_type.name, job_d.job_type.version, job_d.job_type_rev.revision_num)
definition.add_recipe_node('node_e', recipe_type_e.name, recipe_type_e.revision_num)
definition.add_job_node('node_f', job_type_f.name, job_type_f.version, job_type_f.revision_num)
definition.add_job_node('node_g', job_g.job_type.name, job_g.job_type.version, job_g.job_type_rev.revision_num)
definition.add_job_node('node_h', job_h.job_type.name, job_h.job_type.version, job_h.job_type_rev.revision_num)
definition.add_condition_node('node_i', Interface(), DataFilter()) #True
definition.add_condition_node('node_j', Interface(), DataFilter()) #True
definition.add_job_node('node_k', job_type_k.name, job_type_k.version, job_type_k.revision_num)
definition.add_dependency('node_a', 'node_c')
definition.add_dependency('node_a', 'node_e')
definition.add_dependency('node_a', 'node_g')
definition.add_dependency('node_c', 'node_d')
definition.add_dependency('node_e', 'node_f')
definition.add_dependency('node_g', 'node_h')
definition.add_dependency('node_a', 'node_i')
definition.add_dependency('node_a', 'node_j')
definition.add_dependency('node_i', 'node_k')
definition.add_dependency('node_j', 'node_k')
definition_dict = convert_recipe_definition_to_v6_json(definition).get_dict()
recipe_type = recipe_test_utils.create_recipe_type_v6(definition=definition_dict)
recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type, input=data_dict)
node_a = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='node_a', job=job_a, save=False)
node_b = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='node_b', sub_recipe=recipe_b,
save=False)
node_c = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='node_c', job=job_c, save=False)
node_d = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='node_d', job=job_d, save=False)
node_g = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='node_g', job=job_g, save=False)
node_h = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='node_h', job=job_h, save=False)
node_i = recipe_test_utils.create_recipe_node(recipe=recipe, node_name='node_i', condition=condition_i,
save=False)
RecipeNode.objects.bulk_create([node_a, node_b, node_c, node_d, node_g, node_h, node_i])
forced_nodes = ForcedNodes()
forced_nodes_e = ForcedNodes()
forced_nodes_e.set_all_nodes()
forced_nodes.add_subrecipe('node_e', forced_nodes_e)
# Create and execute message
message = create_update_recipe_message(recipe.id, forced_nodes=forced_nodes)
result = message.execute()
self.assertTrue(result)
self.assertEqual(len(message.new_messages), 8)
# Check messages
blocked_jobs_msg = None
pending_jobs_msg = None
create_cond_msg = None
create_jobs_msg = None
create_recipes_msg = None
process_condition_msg = None
process_job_input_msg = None
process_recipe_input_msg = None
for msg in message.new_messages:
if msg.type == 'blocked_jobs':
blocked_jobs_msg = msg
elif msg.type == 'pending_jobs':
pending_jobs_msg = msg
elif msg.type == 'create_conditions':
create_cond_msg = msg
elif msg.type == 'create_jobs':
create_jobs_msg = msg
elif msg.type == 'create_recipes':
create_recipes_msg = msg
elif msg.type == 'process_condition':
process_condition_msg = msg
elif msg.type == 'process_job_input':
process_job_input_msg = msg
elif msg.type == 'process_recipe_input':
process_recipe_input_msg = msg
self.assertIsNotNone(blocked_jobs_msg)
self.assertIsNotNone(pending_jobs_msg)
self.assertIsNotNone(create_cond_msg)
self.assertIsNotNone(create_jobs_msg)
self.assertIsNotNone(create_recipes_msg)
self.assertIsNotNone(process_condition_msg)
self.assertIsNotNone(process_job_input_msg)
self.assertIsNotNone(process_recipe_input_msg)
# Check message to change jobs to BLOCKED
self.assertListEqual(blocked_jobs_msg._blocked_job_ids, [job_h.id])
# Check message to change jobs to PENDING
self.assertListEqual(pending_jobs_msg._pending_job_ids, [job_d.id])
# Check message to create conditions
self.assertEqual(create_cond_msg.recipe_id, recipe.id)
self.assertEqual(create_cond_msg.root_recipe_id, recipe.root_superseded_recipe_id)
condition = Condition('node_j', True)
self.assertListEqual(create_cond_msg.conditions, [condition])
# Check message to create jobs
self.assertEqual(create_jobs_msg.event_id, recipe.event_id)
self.assertEqual(create_jobs_msg.create_jobs_type, RECIPE_TYPE)
self.assertEqual(create_jobs_msg.recipe_id, recipe.id)
self.assertEqual(create_jobs_msg.root_recipe_id, recipe.root_superseded_recipe_id)
self.assertIsNone(create_jobs_msg.superseded_recipe_id)
recipe_job = RecipeJob(job_type_f.name, job_type_f.version, job_type_f.revision_num, 'node_f', False)
self.assertListEqual(create_jobs_msg.recipe_jobs, [recipe_job])
# Check message to create sub-recipes
self.assertEqual(create_recipes_msg.event_id, recipe.event_id)
msg_forced_nodes_dict = convert_forced_nodes_to_v6(create_recipes_msg.forced_nodes).get_dict()
expected_forced_nodes_dict = convert_forced_nodes_to_v6(forced_nodes).get_dict()
self.assertDictEqual(msg_forced_nodes_dict, expected_forced_nodes_dict)
self.assertEqual(create_recipes_msg.create_recipes_type, SUB_RECIPE_TYPE)
self.assertEqual(create_recipes_msg.recipe_id, recipe.id)
self.assertEqual(create_recipes_msg.root_recipe_id, recipe.root_superseded_recipe_id)
self.assertIsNone(create_recipes_msg.superseded_recipe_id)
sub = SubRecipe(recipe_type_e.name, recipe_type_e.revision_num, 'node_e', True)
self.assertListEqual(create_recipes_msg.sub_recipes, [sub])
# Check message to process condition
self.assertEqual(process_condition_msg.condition_id, condition_i.id)
# Check message to process job input
self.assertEqual(process_job_input_msg.job_id, job_c.id)
# Check message to process recipe input
self.assertEqual(process_recipe_input_msg.recipe_id, recipe_b.id)
# Test executing message again
message_json_dict = message.to_json()
message = UpdateRecipe.from_json(message_json_dict)
result = message.execute()
self.assertTrue(result)
# Make sure the same messages are returned
self.assertEqual(len(message.new_messages), 8)
blocked_jobs_msg = None
pending_jobs_msg = None
create_cond_msg = None
create_jobs_msg = None
create_recipes_msg = None
process_condition_msg = None
process_job_input_msg = None
process_recipe_input_msg = None
for msg in message.new_messages:
if msg.type == 'blocked_jobs':
blocked_jobs_msg = msg
elif msg.type == 'pending_jobs':
pending_jobs_msg = msg
elif msg.type == 'create_conditions':
create_cond_msg = msg
elif msg.type == 'create_jobs':
create_jobs_msg = msg
elif msg.type == 'create_recipes':
create_recipes_msg = msg
elif msg.type == 'process_condition':
process_condition_msg = msg
elif msg.type == 'process_job_input':
process_job_input_msg = msg
elif msg.type == 'process_recipe_input':
process_recipe_input_msg = msg
self.assertIsNotNone(blocked_jobs_msg)
self.assertIsNotNone(pending_jobs_msg)
self.assertIsNotNone(create_cond_msg)
self.assertIsNotNone(create_jobs_msg)
self.assertIsNotNone(create_recipes_msg)
self.assertIsNotNone(process_condition_msg)
self.assertIsNotNone(process_job_input_msg)
self.assertIsNotNone(process_recipe_input_msg)
# Check message to change jobs to BLOCKED
self.assertListEqual(blocked_jobs_msg._blocked_job_ids, [job_h.id])
# Check message to change jobs to PENDING
self.assertListEqual(pending_jobs_msg._pending_job_ids, [job_d.id])
# Check message to create conditions
self.assertEqual(create_cond_msg.recipe_id, recipe.id)
self.assertEqual(create_cond_msg.root_recipe_id, recipe.root_superseded_recipe_id)
condition = Condition('node_j', True)
self.assertListEqual(create_cond_msg.conditions, [condition])
# Check message to create jobs
self.assertEqual(create_jobs_msg.event_id, recipe.event_id)
self.assertEqual(create_jobs_msg.create_jobs_type, RECIPE_TYPE)
self.assertEqual(create_jobs_msg.recipe_id, recipe.id)
self.assertEqual(create_jobs_msg.root_recipe_id, recipe.root_superseded_recipe_id)
self.assertIsNone(create_jobs_msg.superseded_recipe_id)
recipe_job = RecipeJob(job_type_f.name, job_type_f.version, job_type_f.revision_num, 'node_f', False)
self.assertListEqual(create_jobs_msg.recipe_jobs, [recipe_job])
# Check message to create sub-recipes
self.assertEqual(create_recipes_msg.event_id, recipe.event_id)
msg_forced_nodes_dict = convert_forced_nodes_to_v6(create_recipes_msg.forced_nodes).get_dict()
expected_forced_nodes_dict = convert_forced_nodes_to_v6(forced_nodes).get_dict()
self.assertDictEqual(msg_forced_nodes_dict, expected_forced_nodes_dict)
self.assertEqual(create_recipes_msg.create_recipes_type, SUB_RECIPE_TYPE)
self.assertEqual(create_recipes_msg.recipe_id, recipe.id)
self.assertEqual(create_recipes_msg.root_recipe_id, recipe.root_superseded_recipe_id)
self.assertIsNone(create_recipes_msg.superseded_recipe_id)
sub = SubRecipe(recipe_type_e.name, recipe_type_e.revision_num, 'node_e', True)
self.assertListEqual(create_recipes_msg.sub_recipes, [sub])
# Check message to process condition
self.assertEqual(process_condition_msg.condition_id, condition_i.id)
# Check message to process job input
self.assertEqual(process_job_input_msg.job_id, job_c.id)
# Check message to process recipe input
self.assertEqual(process_recipe_input_msg.recipe_id, recipe_b.id)
