def setUp(self):
django.setup()
self.source = source_test_utils.create_source()
try:
import ingest.test.utils as ingest_test_utils
self.ingest = ingest_test_utils.create_ingest(
source_file=self.source)
except:
self.ingest = None
try:
import product.test.utils as product_test_utils
self.product1 = product_test_utils.create_product(
is_superseded=True)
product_test_utils.create_file_link(ancestor=self.source,
descendant=self.product1)
except:
self.product1 = None
try:
import product.test.utils as product_test_utils
self.product2 = product_test_utils.create_product()
product_test_utils.create_file_link(ancestor=self.source,
descendant=self.product2)
except:
self.product2 = None
def setUp(self):
django.setup()
self.src_file_1 = source_test_utils.create_source()
self.src_file_2 = source_test_utils.create_source()
self.src_file_3 = source_test_utils.create_source()
self.src_file_4 = source_test_utils.create_source()
self.job_exe_1 = job_test_utils.create_job_exe()
self.recipe_job_1 = recipe_test_utils.create_recipe_job(job=self.job_exe_1.job)
self.product_1 = prod_test_utils.create_product(self.job_exe_1, has_been_published=True)
self.product_2 = prod_test_utils.create_product(self.job_exe_1, has_been_published=True)
FileAncestryLink.objects.create(ancestor=self.src_file_1, descendant=self.product_1, job_exe=self.job_exe_1,
job=self.job_exe_1.job, recipe=self.recipe_job_1.recipe)
FileAncestryLink.objects.create(ancestor=self.src_file_1, descendant=self.product_2, job_exe=self.job_exe_1,
job=self.job_exe_1.job, recipe=self.recipe_job_1.recipe)
FileAncestryLink.objects.create(ancestor=self.src_file_2, descendant=self.product_1, job_exe=self.job_exe_1,
job=self.job_exe_1.job, recipe=self.recipe_job_1.recipe)
FileAncestryLink.objects.create(ancestor=self.src_file_2, descendant=self.product_2, job_exe=self.job_exe_1,
job=self.job_exe_1.job, recipe=self.recipe_job_1.recipe)
self.job_exe_2 = job_test_utils.create_job_exe()
self.recipe_job_2 = recipe_test_utils.create_recipe_job(job=self.job_exe_2.job)
self.product_3 = prod_test_utils.create_product(self.job_exe_2, has_been_published=True)
FileAncestryLink.objects.create(ancestor=self.src_file_3, descendant=self.product_3, job_exe=self.job_exe_2,
job=self.job_exe_2.job, recipe=self.recipe_job_2.recipe)
FileAncestryLink.objects.create(ancestor=self.src_file_4, descendant=self.product_3, job_exe=self.job_exe_2,
job=self.job_exe_2.job, recipe=self.recipe_job_2.recipe)
def setUp(self):
django.setup()
self.source = source_test_utils.create_source()
try:
import ingest.test.utils as ingest_test_utils
self.ingest = ingest_test_utils.create_ingest(source_file=self.source)
except:
self.ingest = None
try:
import product.test.utils as product_test_utils
self.product1 = product_test_utils.create_product(is_superseded=True)
product_test_utils.create_file_link(ancestor=self.source, descendant=self.product1)
except:
self.product1 = None
try:
import product.test.utils as product_test_utils
self.product2 = product_test_utils.create_product()
product_test_utils.create_file_link(ancestor=self.source, descendant=self.product2)
except:
self.product2 = None
def test_successful_with_partial_recipe(self):
"""Tests calling QueueManager.handle_job_completion() successfully with a job in a recipe."""
# Queue the recipe
recipe_id = Queue.objects.queue_new_recipe(self.recipe_type, self.data, self.event)
# Fake out completing Job 1
job_1 = RecipeJob.objects.select_related('job').get(recipe_id=recipe_id, job_name='Job 1').job
job_exe_1 = JobExecution.objects.get(job_id=job_1.id)
output_file_1 = product_test_utils.create_product(job_exe=job_exe_1, workspace=self.workspace)
output_file_2 = product_test_utils.create_product(job_exe=job_exe_1, workspace=self.workspace)
results = JobResults()
results.add_file_list_parameter('Test Output 1', [output_file_1.id, output_file_2.id])
JobExecution.objects.post_steps_results(job_exe_1.id, results, ResultsManifest())
Job.objects.filter(pk=job_1.id).update(status='RUNNING')
JobExecution.objects.filter(pk=job_exe_1.id).update(status='RUNNING')
# Call method to test
Queue.objects.handle_job_completion(job_exe_1.id, now())
# Make sure processor was called
self.assertTrue(self.mock_processor.process_completed.called)
# Make sure Job 2 in the recipe is successfully queued
recipe_job_2 = RecipeJob.objects.select_related('job', 'recipe').get(recipe_id=recipe_id, job_name='Job 2')
self.assertEqual(recipe_job_2.job.status, 'QUEUED')
self.assertIsNone(recipe_job_2.recipe.completed)
def setUp(self):
django.setup()
from batch.test import utils as batch_test_utils
from product.test import utils as product_test_utils
self.country = storage_test_utils.create_country()
self.src_file = source_test_utils.create_source()
self.job_type1 = job_test_utils.create_job_type(name='test1', category='test-1', is_operational=True)
self.job1 = job_test_utils.create_job(job_type=self.job_type1)
self.job_exe1 = job_test_utils.create_job_exe(job=self.job1)
self.product1 = product_test_utils.create_product(job_exe=self.job_exe1, has_been_published=True,
is_published=True, file_name='test.txt',
countries=[self.country])
product_test_utils.create_file_link(ancestor=self.src_file, descendant=self.product1, job=self.job1,
job_exe=self.job_exe1)
self.batch = batch_test_utils.create_batch()
self.job_type2 = job_test_utils.create_job_type(name='test2', category='test-2', is_operational=False)
self.job2 = job_test_utils.create_job(job_type=self.job_type2)
self.job_exe2 = job_test_utils.create_job_exe(job=self.job2)
self.product2a = product_test_utils.create_product(job_exe=self.job_exe2, has_been_published=True,
is_published=False, countries=[self.country])
product_test_utils.create_file_link(ancestor=self.src_file, descendant=self.product2a, job=self.job2,
job_exe=self.job_exe2, batch=self.batch)
self.product2b = product_test_utils.create_product(job_exe=self.job_exe2, has_been_published=True,
is_published=True, is_superseded=True,
countries=[self.country])
product_test_utils.create_file_link(ancestor=self.src_file, descendant=self.product2b, job=self.job2,
job_exe=self.job_exe2, batch=self.batch)
self.product2c = product_test_utils.create_product(job_exe=self.job_exe2, has_been_published=True,
is_published=True, countries=[self.country])
product_test_utils.create_file_link(ancestor=self.src_file, descendant=self.product2c, job=self.job2,
job_exe=self.job_exe2, batch=self.batch)
def setUp(self):
django.setup()
self.src_file_1 = source_test_utils.create_source()
self.src_file_2 = source_test_utils.create_source()
self.src_file_3 = source_test_utils.create_source()
self.src_file_4 = source_test_utils.create_source()
self.job_exe_1 = job_test_utils.create_job_exe()
self.recipe_job_1 = recipe_test_utils.create_recipe_job(job=self.job_exe_1.job)
self.product_1 = prod_test_utils.create_product(self.job_exe_1, has_been_published=True)
self.product_2 = prod_test_utils.create_product(self.job_exe_1, has_been_published=True)
FileAncestryLink.objects.create(ancestor=self.src_file_1, descendant=self.product_1, job_exe=self.job_exe_1,
job=self.job_exe_1.job, recipe=self.recipe_job_1.recipe)
FileAncestryLink.objects.create(ancestor=self.src_file_1, descendant=self.product_2, job_exe=self.job_exe_1,
job=self.job_exe_1.job, recipe=self.recipe_job_1.recipe)
FileAncestryLink.objects.create(ancestor=self.src_file_2, descendant=self.product_1, job_exe=self.job_exe_1,
job=self.job_exe_1.job, recipe=self.recipe_job_1.recipe)
FileAncestryLink.objects.create(ancestor=self.src_file_2, descendant=self.product_2, job_exe=self.job_exe_1,
job=self.job_exe_1.job, recipe=self.recipe_job_1.recipe)
self.job_exe_2 = job_test_utils.create_job_exe()
self.recipe_job_2 = recipe_test_utils.create_recipe_job(job=self.job_exe_2.job)
self.product_3 = prod_test_utils.create_product(self.job_exe_2, has_been_published=True)
FileAncestryLink.objects.create(ancestor=self.src_file_3, descendant=self.product_3, job_exe=self.job_exe_2,
job=self.job_exe_2.job, recipe=self.recipe_job_2.recipe)
FileAncestryLink.objects.create(ancestor=self.src_file_4, descendant=self.product_3, job_exe=self.job_exe_2,
job=self.job_exe_2.job, recipe=self.recipe_job_2.recipe)
def test_get_source_products(self):
"""Tests calling get_source_products()"""
from batch.test import utils as batch_test_utils
from product.test import utils as product_test_utils
job_exe_1 = job_utils.create_job_exe()
job_exe_2 = job_utils.create_job_exe()
product_1 = product_test_utils.create_product(job_exe=job_exe_1,
has_been_published=True,
workspace=self.workspace)
product_2 = product_test_utils.create_product(job_exe=job_exe_2,
has_been_published=True,
workspace=self.workspace)
batch_1 = batch_test_utils.create_batch()
batch_2 = batch_test_utils.create_batch()
product_test_utils.create_file_link(ancestor=self.src_file,
descendant=product_1,
job=job_exe_1.job,
job_exe=job_exe_1,
batch=batch_1)
product_test_utils.create_file_link(ancestor=self.src_file,
descendant=product_2,
job=job_exe_2.job,
job_exe=job_exe_2,
batch=batch_2)
products = SourceFile.objects.get_source_products(
self.src_file.id, batch_ids=[batch_1.id])
self.assertEqual(len(products), 1)
self.assertEqual(products[0].id, product_1.id)
def test_json(self):
"""Tests coverting an UnpublishJobs message to and from JSON"""
when = now()
job_exe_1 = job_test_utils.create_job_exe(status='COMPLETED')
job_exe_2 = job_test_utils.create_job_exe(status='COMPLETED')
product_1 = product_test_utils.create_product(job_exe=job_exe_1,
is_published=True)
product_2 = product_test_utils.create_product(job_exe=job_exe_2,
is_published=True)
# Add jobs to message
message = UnpublishJobs()
message.when = when
if message.can_fit_more():
message.add_job(job_exe_1.job_id)
if message.can_fit_more():
message.add_job(job_exe_2.job_id)
# Convert message to JSON and back, and then execute
message_json_dict = message.to_json()
new_message = UnpublishJobs.from_json(message_json_dict)
result = new_message.execute()
self.assertTrue(result)
products = ProductFile.objects.filter(
id__in=[product_1.id, product_2.id])
self.assertEqual(len(products), 2)
self.assertFalse(products[0].is_published)
self.assertEqual(products[0].unpublished, when)
self.assertFalse(products[1].is_published)
self.assertEqual(products[1].unpublished, when)
def test_execute(self):
"""Tests calling UnpublishJobs.execute() successfully"""
when = now()
job_exe_1 = job_test_utils.create_job_exe(status='COMPLETED')
job_exe_2 = job_test_utils.create_job_exe(status='COMPLETED')
product_1 = product_test_utils.create_product(job_exe=job_exe_1,
is_published=True)
product_2 = product_test_utils.create_product(job_exe=job_exe_2,
is_published=True)
# Add jobs to message
message = UnpublishJobs()
message.when = when
if message.can_fit_more():
message.add_job(job_exe_1.job_id)
if message.can_fit_more():
message.add_job(job_exe_2.job_id)
# Execute message
result = message.execute()
self.assertTrue(result)
# Check that products are unpublished
products = ProductFile.objects.filter(
id__in=[product_1.id, product_2.id])
self.assertEqual(len(products), 2)
self.assertFalse(products[0].is_published)
self.assertEqual(products[0].unpublished, when)
self.assertFalse(products[1].is_published)
self.assertEqual(products[1].unpublished, when)
def test_successful_with_full_recipe(self):
"""Tests calling QueueManager.handle_job_completion() successfully with all jobs in a recipe."""
# Queue the recipe
handler = Queue.objects.queue_new_recipe(self.recipe_type, self.data,
self.event)
# Fake out completing Job 1
job_1 = RecipeJob.objects.select_related('job').get(
recipe_id=handler.recipe.id, job_name='Job 1').job
job_exe_1 = job_test_utils.create_job_exe(job=job_1, status='RUNNING')
output_file_1 = product_test_utils.create_product(
job_exe=job_exe_1, workspace=self.workspace)
output_file_2 = product_test_utils.create_product(
job_exe=job_exe_1, workspace=self.workspace)
results = JobResults()
results.add_file_list_parameter('Test Output 1',
[output_file_1.id, output_file_2.id])
job_exe_output_1 = JobExecutionOutput()
job_exe_output_1.job_exe_id = job_exe_1.id
job_exe_output_1.job_id = job_exe_1.job_id
job_exe_output_1.job_type_id = job_exe_1.job_type_id
job_exe_output_1.exe_num = job_exe_1.exe_num
job_exe_output_1.output = results.get_dict()
job_exe_output_1.save()
Job.objects.filter(pk=job_1.id).update(status='RUNNING')
Queue.objects.handle_job_completion(job_1.id, job_1.num_exes, now())
# Fake out completing Job 2
job_2 = RecipeJob.objects.select_related('job').get(
recipe_id=handler.recipe.id, job_name='Job 2').job
job_exe_2 = job_test_utils.create_job_exe(job=job_2, status='RUNNING')
output_file_1 = product_test_utils.create_product(
job_exe=job_exe_2, workspace=self.workspace)
output_file_2 = product_test_utils.create_product(
job_exe=job_exe_2, workspace=self.workspace)
results = JobResults()
results.add_file_list_parameter('Test Output 2',
[output_file_1.id, output_file_2.id])
job_exe_output_2 = JobExecutionOutput()
job_exe_output_2.job_exe_id = job_exe_2.id
job_exe_output_2.job_id = job_exe_2.job_id
job_exe_output_2.job_type_id = job_exe_2.job_type_id
job_exe_output_2.exe_num = job_exe_2.exe_num
job_exe_output_2.output = results.get_dict()
job_exe_output_2.save()
Job.objects.filter(pk=job_2.id).update(status='RUNNING')
# Call method to test
Queue.objects.handle_job_completion(job_2.id, job_2.num_exes, now())
# Make sure final recipe attributes are updated
recipe = Recipe.objects.get(pk=handler.recipe.id)
self.assertIsNotNone(recipe.completed)
def setUp(self):
django.setup()
self.job_exe = job_test_utils.create_job_exe()
self.product_1 = prod_test_utils.create_product(job_exe=self.job_exe)
self.product_2 = prod_test_utils.create_product(job_exe=self.job_exe)
self.product_3 = prod_test_utils.create_product(job_exe=self.job_exe)
def setUp(self):
django.setup()
self.job_exe = job_test_utils.create_job_exe()
self.product_1 = prod_test_utils.create_product(job_exe=self.job_exe)
self.product_2 = prod_test_utils.create_product(job_exe=self.job_exe)
self.product_3 = prod_test_utils.create_product(job_exe=self.job_exe)
def setUp(self):
django.setup()
self.job_type1 = job_test_utils.create_job_type(name='test1', category='test-1', is_operational=True)
self.job1 = job_test_utils.create_job(job_type=self.job_type1)
self.job_exe1 = job_test_utils.create_job_exe(job=self.job1)
self.product1 = product_test_utils.create_product(job_exe=self.job_exe1, has_been_published=True,
file_name='test.txt')
self.job_type2 = job_test_utils.create_job_type(name='test2', category='test-2', is_operational=False)
self.job2 = job_test_utils.create_job(job_type=self.job_type2)
self.job_exe2 = job_test_utils.create_job_exe(job=self.job2)
self.product2 = product_test_utils.create_product(job_exe=self.job_exe2, has_been_published=True)
def setUp(self):
django.setup()
self.source = source_test_utils.create_source()
self.ancestor = product_test_utils.create_product(file_name='test_ancestor.txt')
self.descendant = product_test_utils.create_product(file_name='test_descendant.txt')
self.product = product_test_utils.create_product(file_name='test_product.txt')
product_test_utils.create_file_link(ancestor=self.source, descendant=self.ancestor)
product_test_utils.create_file_link(ancestor=self.source, descendant=self.product)
product_test_utils.create_file_link(ancestor=self.source, descendant=self.descendant)
product_test_utils.create_file_link(ancestor=self.ancestor, descendant=self.product)
product_test_utils.create_file_link(ancestor=self.ancestor, descendant=self.descendant)
product_test_utils.create_file_link(ancestor=self.product, descendant=self.descendant)
def setUp(self):
django.setup()
self.job_exe_1 = job_test_utils.create_job_exe()
self.job_type_1_id = self.job_exe_1.job.job_type.id
self.job_exe_2 = job_test_utils.create_job_exe()
self.job_type_2_id = self.job_exe_2.job.job_type.id
self.product_1 = prod_test_utils.create_product()
self.product_2 = prod_test_utils.create_product(
has_been_published=True)
self.product_3 = prod_test_utils.create_product(
self.job_exe_2, has_been_published=True)
time.sleep(0.001)
self.last_modified_start = now()
self.product_4 = prod_test_utils.create_product()
self.product_5 = prod_test_utils.create_product(self.job_exe_2)
self.product_6 = prod_test_utils.create_product(
self.job_exe_2, has_been_published=True)
time.sleep(0.001)
self.product_7 = prod_test_utils.create_product(
self.job_exe_1, has_been_published=True)
time.sleep(0.001)
self.product_8 = prod_test_utils.create_product(
has_been_published=True)
self.last_modified_end = now()
def setUp(self):
django.setup()
self.source = source_test_utils.create_source()
self.ancestor = product_test_utils.create_product(file_name='test_ancestor.txt')
self.descendant = product_test_utils.create_product(file_name='test_descendant.txt')
self.product = product_test_utils.create_product(file_name='test_product.txt')
product_test_utils.create_file_link(ancestor=self.source, descendant=self.ancestor)
product_test_utils.create_file_link(ancestor=self.source, descendant=self.product)
product_test_utils.create_file_link(ancestor=self.source, descendant=self.descendant)
product_test_utils.create_file_link(ancestor=self.ancestor, descendant=self.product)
product_test_utils.create_file_link(ancestor=self.ancestor, descendant=self.descendant)
product_test_utils.create_file_link(ancestor=self.product, descendant=self.descendant)
def setUp(self):
django.setup()
self.country = storage_test_utils.create_country()
self.job_type1 = job_test_utils.create_job_type(name='test1', category='test-1', is_operational=True)
self.job1 = job_test_utils.create_job(job_type=self.job_type1)
self.job_exe1 = job_test_utils.create_job_exe(job=self.job1)
self.product1 = product_test_utils.create_product(job_exe=self.job_exe1, has_been_published=True,
file_name='test.txt', countries=[self.country])
self.job_type2 = job_test_utils.create_job_type(name='test2', category='test-2', is_operational=False)
self.job2 = job_test_utils.create_job(job_type=self.job_type2)
self.job_exe2 = job_test_utils.create_job_exe(job=self.job2)
self.product2 = product_test_utils.create_product(job_exe=self.job_exe2, has_been_published=True,
countries=[self.country])
def setUp(self):
django.setup()
from product.test import utils as product_test_utils
self.country = storage_test_utils.create_country()
self.job_type1 = job_test_utils.create_job_type(name='test1',
category='test-1',
is_operational=True)
self.job1 = job_test_utils.create_job(job_type=self.job_type1)
self.job_exe1 = job_test_utils.create_job_exe(job=self.job1)
self.src_file = source_test_utils.create_source(
data_started='2016-01-01T00:00:00Z',
data_ended='2016-01-01T00:00:00Z',
file_name='test.txt',
is_parsed=True)
self.product1 = product_test_utils.create_product(
job_exe=self.job_exe1,
has_been_published=True,
is_published=True,
file_name='test_prod.txt',
countries=[self.country])
product_test_utils.create_file_link(ancestor=self.src_file,
descendant=self.product1,
job=self.job1,
job_exe=self.job_exe1)
def setUp(self):
django.setup()
# Generation 1
self.file_1 = storage_test_utils.create_file()
self.file_2 = storage_test_utils.create_file()
# Generation 2
job_exe_1 = job_test_utils.create_job_exe()
recipe_job_1 = recipe_test_utils.create_recipe_job(job=job_exe_1.job)
self.file_3 = prod_test_utils.create_product(job_exe=job_exe_1)
self.file_4 = prod_test_utils.create_product(job_exe=job_exe_1)
self.file_5 = prod_test_utils.create_product(job_exe=job_exe_1)
# Generation 3
job_exe_2 = job_test_utils.create_job_exe()
recipe_job_2 = recipe_test_utils.create_recipe_job(job=job_exe_2.job)
self.file_6 = prod_test_utils.create_product(job_exe=job_exe_2)
# Stand alone file
self.file_7 = prod_test_utils.create_product()
# First job links generation 1 to 2
FileAncestryLink.objects.create(ancestor=self.file_1, descendant=self.file_3, job_exe=job_exe_1,
job=job_exe_1.job, recipe=recipe_job_1.recipe)
FileAncestryLink.objects.create(ancestor=self.file_1, descendant=self.file_4, job_exe=job_exe_1,
job=job_exe_1.job, recipe=recipe_job_1.recipe)
FileAncestryLink.objects.create(ancestor=self.file_1, descendant=self.file_5, job_exe=job_exe_1,
job=job_exe_1.job, recipe=recipe_job_1.recipe)
FileAncestryLink.objects.create(ancestor=self.file_2, descendant=self.file_3, job_exe=job_exe_1,
job=job_exe_1.job, recipe=recipe_job_1.recipe)
FileAncestryLink.objects.create(ancestor=self.file_2, descendant=self.file_4, job_exe=job_exe_1,
job=job_exe_1.job, recipe=recipe_job_1.recipe)
FileAncestryLink.objects.create(ancestor=self.file_2, descendant=self.file_5, job_exe=job_exe_1,
job=job_exe_1.job, recipe=recipe_job_1.recipe)
# Second job links generation 2 to 3
FileAncestryLink.objects.create(ancestor=self.file_3, descendant=self.file_6, job_exe=job_exe_2,
job=job_exe_2.job, recipe=recipe_job_2.recipe)
FileAncestryLink.objects.create(ancestor=self.file_1, descendant=self.file_6, job_exe=job_exe_2,
job=job_exe_2.job, recipe=recipe_job_2.recipe,
ancestor_job_exe=job_exe_1, ancestor_job=job_exe_1.job)
FileAncestryLink.objects.create(ancestor=self.file_2, descendant=self.file_6, job_exe=job_exe_2,
job=job_exe_2.job, recipe=recipe_job_2.recipe,
ancestor_job_exe=job_exe_1, ancestor_job=job_exe_1.job)
def setUp(self):
django.setup()
# Generation 1
self.file_1 = storage_test_utils.create_file()
self.file_2 = storage_test_utils.create_file()
# Generation 2
job_exe_1 = job_test_utils.create_job_exe()
recipe_job_1 = recipe_test_utils.create_recipe_job(job=job_exe_1.job)
self.file_3 = prod_test_utils.create_product(job_exe=job_exe_1)
self.file_4 = prod_test_utils.create_product(job_exe=job_exe_1)
self.file_5 = prod_test_utils.create_product(job_exe=job_exe_1)
# Generation 3
job_exe_2 = job_test_utils.create_job_exe()
recipe_job_2 = recipe_test_utils.create_recipe_job(job=job_exe_2.job)
self.file_6 = prod_test_utils.create_product(job_exe=job_exe_2)
# Stand alone file
self.file_7 = prod_test_utils.create_product()
# First job links generation 1 to 2
FileAncestryLink.objects.create(ancestor=self.file_1, descendant=self.file_3, job_exe=job_exe_1,
job=job_exe_1.job, recipe=recipe_job_1.recipe)
FileAncestryLink.objects.create(ancestor=self.file_1, descendant=self.file_4, job_exe=job_exe_1,
job=job_exe_1.job, recipe=recipe_job_1.recipe)
FileAncestryLink.objects.create(ancestor=self.file_1, descendant=self.file_5, job_exe=job_exe_1,
job=job_exe_1.job, recipe=recipe_job_1.recipe)
FileAncestryLink.objects.create(ancestor=self.file_2, descendant=self.file_3, job_exe=job_exe_1,
job=job_exe_1.job, recipe=recipe_job_1.recipe)
FileAncestryLink.objects.create(ancestor=self.file_2, descendant=self.file_4, job_exe=job_exe_1,
job=job_exe_1.job, recipe=recipe_job_1.recipe)
FileAncestryLink.objects.create(ancestor=self.file_2, descendant=self.file_5, job_exe=job_exe_1,
job=job_exe_1.job, recipe=recipe_job_1.recipe)
# Second job links generation 2 to 3
FileAncestryLink.objects.create(ancestor=self.file_3, descendant=self.file_6, job_exe=job_exe_2,
job=job_exe_2.job, recipe=recipe_job_2.recipe)
FileAncestryLink.objects.create(ancestor=self.file_1, descendant=self.file_6, job_exe=job_exe_2,
job=job_exe_2.job, recipe=recipe_job_2.recipe,
ancestor_job_exe=job_exe_1, ancestor_job=job_exe_1.job)
FileAncestryLink.objects.create(ancestor=self.file_2, descendant=self.file_6, job_exe=job_exe_2,
job=job_exe_2.job, recipe=recipe_job_2.recipe,
ancestor_job_exe=job_exe_1, ancestor_job=job_exe_1.job)
def test_publish_products_supersede_products(self):
"""Tests calling ProductFileManager.publish_products() where products with existing UUIDs get superseded
"""
# Create existing products with known UUIDs
builder = hashlib.md5()
builder.update('test')
uuid_1 = builder.hexdigest()
builder.update('hello')
uuid_2 = builder.hexdigest()
builder.update('again')
uuid_3 = builder.hexdigest()
product_a = prod_test_utils.create_product(uuid=uuid_1,
has_been_published=True,
is_published=True)
product_b = prod_test_utils.create_product(uuid=uuid_2,
has_been_published=True,
is_published=True)
product_c = prod_test_utils.create_product(uuid=uuid_3,
has_been_published=True,
is_published=True)
# Set the new products with the same UUIDs
ScaleFile.objects.filter(id=self.product_1.id).update(uuid=uuid_1)
ScaleFile.objects.filter(id=self.product_2.id).update(uuid=uuid_2)
ScaleFile.objects.filter(id=self.product_3.id).update(uuid=uuid_3)
# Publish new products
when = now()
ProductFile.objects.publish_products(self.job_exe, self.job_exe.job,
when)
# Check old products to make sure they are superseded
product_a = ScaleFile.objects.get(id=product_a.id)
product_b = ScaleFile.objects.get(id=product_b.id)
product_c = ScaleFile.objects.get(id=product_c.id)
self.assertFalse(product_a.is_published)
self.assertTrue(product_a.is_superseded)
self.assertEqual(product_a.superseded, when)
self.assertFalse(product_b.is_published)
self.assertTrue(product_b.is_superseded)
self.assertEqual(product_b.superseded, when)
self.assertFalse(product_c.is_published)
self.assertTrue(product_c.is_superseded)
self.assertEqual(product_c.superseded, when)
def test_get_source_jobs(self):
"""Tests calling get_source_jobs()"""
from product.test import utils as product_test_utils
job_exe = job_utils.create_job_exe()
product_1 = product_test_utils.create_product(job_exe=job_exe, has_been_published=True,
workspace=self.workspace)
product_2 = product_test_utils.create_product(job_exe=job_exe, has_been_published=True,
workspace=self.workspace)
product_test_utils.create_file_link(ancestor=self.src_file, descendant=product_1, job=job_exe.job,
job_exe=job_exe)
product_test_utils.create_file_link(ancestor=self.src_file, descendant=product_2, job=job_exe.job,
job_exe=job_exe)
jobs = SourceFile.objects.get_source_jobs(self.src_file.id)
# Should only return one job despite two file_ancestry_link models
self.assertEqual(len(jobs), 1)
self.assertEqual(jobs[0].id, job_exe.job.id)
def test_successful_with_full_recipe(self):
"""Tests calling QueueManager.handle_job_completion() successfully with all jobs in a recipe."""
# Queue the recipe
recipe_id = Queue.objects.queue_new_recipe(self.recipe_type, self.data, self.event)
# Fake out completing Job 1
job_1 = RecipeJob.objects.select_related("job").get(recipe_id=recipe_id, job_name="Job 1").job
job_exe_1 = JobExecution.objects.get(job_id=job_1.id)
output_file_1 = product_test_utils.create_product(job_exe=job_exe_1, workspace=self.workspace)
output_file_2 = product_test_utils.create_product(job_exe=job_exe_1, workspace=self.workspace)
results = JobResults()
results.add_file_list_parameter("Test Output 1", [output_file_1.id, output_file_2.id])
JobExecution.objects.post_steps_results(job_exe_1.id, results, ResultsManifest())
Job.objects.filter(pk=job_1.id).update(status="RUNNING")
JobExecution.objects.filter(pk=job_exe_1.id).update(status="RUNNING")
Queue.objects.handle_job_completion(job_exe_1.id, now())
# Fake out completing Job 2
job_2 = RecipeJob.objects.select_related("job").get(recipe_id=recipe_id, job_name="Job 2").job
job_exe_2 = JobExecution.objects.get(job_id=job_2.id)
output_file_1 = product_test_utils.create_product(job_exe=job_exe_2, workspace=self.workspace)
output_file_2 = product_test_utils.create_product(job_exe=job_exe_2, workspace=self.workspace)
results = JobResults()
results.add_file_list_parameter("Test Output 2", [output_file_1.id, output_file_2.id])
JobExecution.objects.post_steps_results(job_exe_2.id, results, ResultsManifest())
Job.objects.filter(pk=job_2.id).update(status="RUNNING")
JobExecution.objects.filter(pk=job_exe_2.id).update(status="RUNNING")
# Call method to test
Queue.objects.handle_job_completion(job_exe_2.id, now())
# Make sure processor was called
self.assertEqual(self.mock_processor.process_completed.call_count, 2)
# Make sure final recipe attributes are updated
recipe = Recipe.objects.get(pk=recipe_id)
self.assertIsNotNone(recipe.completed)
def test_execute(self):
"""Tests calling CompletedJobs.execute() successfully"""
job_exe = job_test_utils.create_job_exe(status='COMPLETED')
job = job_exe.job
product_1 = product_test_utils.create_product(job_exe=job_exe)
product_2 = product_test_utils.create_product(job_exe=job_exe)
# Add job to message
message = PublishJob()
message.job_id = job.id
# Execute message
result = message.execute()
self.assertTrue(result)
# Check that products are published
for scale_file in ScaleFile.objects.filter(id__in=[product_1.id, product_2.id]):
self.assertTrue(scale_file.is_published)
def setUp(self):
django.setup()
self.country = storage_test_utils.create_country()
self.job_type1 = job_test_utils.create_job_type(name='test1', category='test-1', is_operational=True)
self.job1 = job_test_utils.create_job(job_type=self.job_type1)
self.job_exe1 = job_test_utils.create_job_exe(job=self.job1)
self.product1 = product_test_utils.create_product(job_exe=self.job_exe1, has_been_published=True,
is_published=True, file_name='test.txt',
countries=[self.country])
self.job_type2 = job_test_utils.create_job_type(name='test2', category='test-2', is_operational=False)
self.job2 = job_test_utils.create_job(job_type=self.job_type2)
self.job_exe2 = job_test_utils.create_job_exe(job=self.job2)
self.product2a = product_test_utils.create_product(job_exe=self.job_exe2, has_been_published=True,
is_published=False, countries=[self.country])
self.product2b = product_test_utils.create_product(job_exe=self.job_exe2, has_been_published=True,
is_published=True, is_superseded=True,
countries=[self.country])
self.product2c = product_test_utils.create_product(job_exe=self.job_exe2, has_been_published=True,
is_published=True, countries=[self.country])
def test_json(self):
"""Tests coverting a PublishJob message to and from JSON"""
job_exe = job_test_utils.create_job_exe(status='COMPLETED')
job = job_exe.job
product_1 = product_test_utils.create_product(job_exe=job_exe)
product_2 = product_test_utils.create_product(job_exe=job_exe)
# Add job to message
message = PublishJob()
message.job_id = job.id
# Convert message to JSON and back, and then execute
message_json_dict = message.to_json()
new_message = PublishJob.from_json(message_json_dict)
result = new_message.execute()
self.assertTrue(result)
for scale_file in ScaleFile.objects.filter(id__in=[product_1.id, product_2.id]):
self.assertTrue(scale_file.is_published)
def test_publish_products_unpublish_superseded(self):
"""Tests calling ProductFileManager.publish_products() where the job has superseded job products that must be
unpublished
"""
# Job 1 is superseded by Job 2 and Job 2 is superseded by Job 3
job_exe_1 = job_test_utils.create_job_exe()
product_1_a = prod_test_utils.create_product(job_exe=job_exe_1, has_been_published=True, is_published=True)
product_1_b = prod_test_utils.create_product(job_exe=job_exe_1, has_been_published=True, is_published=True)
job_type = job_test_utils.create_seed_job_type()
event = trigger_test_utils.create_trigger_event()
job_type_rev = JobTypeRevision.objects.get_by_natural_key(job_type, job_type.revision_num)
job_2 = Job.objects.create_job_v6(job_type_rev, event_id=event.id, superseded_job=job_exe_1.job)
job_2.save()
job_exe_2 = job_test_utils.create_job_exe(job=job_2)
Job.objects.supersede_jobs([job_exe_1.job.id], now())
product_2_a = prod_test_utils.create_product(job_exe=job_exe_2, has_been_published=True, is_published=True)
product_2_b = prod_test_utils.create_product(job_exe=job_exe_2, has_been_published=True, is_published=True)
job_3 = Job.objects.create_job_v6(job_type_rev, event_id=event.id, superseded_job=job_exe_2.job)
job_3.save()
job_exe_3 = job_test_utils.create_job_exe(job=job_3)
Job.objects.supersede_jobs([job_2.id], now())
product_3_a = prod_test_utils.create_product(job_exe=job_exe_3)
product_3_b = prod_test_utils.create_product(job_exe=job_exe_3)
when = now()
ProductFile.objects.publish_products(job_exe_3, job_3, when)
# Make sure products from Job 1 and Job 2 are unpublished
product_1_a = ScaleFile.objects.get(id=product_1_a.id)
product_1_b = ScaleFile.objects.get(id=product_1_b.id)
product_2_a = ScaleFile.objects.get(id=product_2_a.id)
product_2_b = ScaleFile.objects.get(id=product_2_b.id)
self.assertTrue(product_1_a.has_been_published)
self.assertFalse(product_1_a.is_published)
self.assertEqual(product_1_a.unpublished, when)
self.assertTrue(product_1_b.has_been_published)
self.assertFalse(product_1_b.is_published)
self.assertEqual(product_1_b.unpublished, when)
self.assertTrue(product_2_a.has_been_published)
self.assertFalse(product_2_a.is_published)
self.assertEqual(product_2_a.unpublished, when)
self.assertTrue(product_2_b.has_been_published)
self.assertFalse(product_2_b.is_published)
self.assertEqual(product_2_b.unpublished, when)
# Make sure Job 3 products are published
product_3_a = ScaleFile.objects.get(id=product_3_a.id)
product_3_b = ScaleFile.objects.get(id=product_3_b.id)
self.assertTrue(product_3_a.has_been_published)
self.assertTrue(product_3_a.is_published)
self.assertFalse(product_3_a.is_superseded)
self.assertEqual(product_3_a.published, when)
self.assertIsNone(product_3_a.superseded)
self.assertTrue(product_3_b.has_been_published)
self.assertTrue(product_3_b.is_published)
self.assertFalse(product_3_b.is_superseded)
self.assertEqual(product_3_b.published, when)
self.assertIsNone(product_3_b.superseded)
def test_publish_products_unpublish_superseded(self):
"""Tests calling ProductFileManager.publish_products() where the job has superseded job products that must be
unpublished
"""
# Job 1 is superseded by Job 2 and Job 2 is superseded by Job 3
job_exe_1 = job_test_utils.create_job_exe()
product_1_a = prod_test_utils.create_product(job_exe=job_exe_1, has_been_published=True, is_published=True)
product_1_b = prod_test_utils.create_product(job_exe=job_exe_1, has_been_published=True, is_published=True)
job_type = job_test_utils.create_job_type()
event = trigger_test_utils.create_trigger_event()
job_2 = Job.objects.create_job(job_type=job_type, event=event, superseded_job=job_exe_1.job)
job_2.save()
job_exe_2 = job_test_utils.create_job_exe(job=job_2)
Job.objects.supersede_jobs([job_exe_1.job], now())
product_2_a = prod_test_utils.create_product(job_exe=job_exe_2, has_been_published=True, is_published=True)
product_2_b = prod_test_utils.create_product(job_exe=job_exe_2, has_been_published=True, is_published=True)
job_3 = Job.objects.create_job(job_type=job_type, event=event, superseded_job=job_exe_2.job)
job_3.save()
job_exe_3 = job_test_utils.create_job_exe(job=job_3)
Job.objects.supersede_jobs([job_2], now())
product_3_a = prod_test_utils.create_product(job_exe=job_exe_3)
product_3_b = prod_test_utils.create_product(job_exe=job_exe_3)
when = now()
ProductFile.objects.publish_products(job_exe_3, when)
# Make sure products from Job 1 and Job 2 are unpublished
product_1_a = ProductFile.objects.get(id=product_1_a.id)
product_1_b = ProductFile.objects.get(id=product_1_b.id)
product_2_a = ProductFile.objects.get(id=product_2_a.id)
product_2_b = ProductFile.objects.get(id=product_2_b.id)
self.assertTrue(product_1_a.has_been_published)
self.assertFalse(product_1_a.is_published)
self.assertEqual(product_1_a.unpublished, when)
self.assertTrue(product_1_b.has_been_published)
self.assertFalse(product_1_b.is_published)
self.assertEqual(product_1_b.unpublished, when)
self.assertTrue(product_2_a.has_been_published)
self.assertFalse(product_2_a.is_published)
self.assertEqual(product_2_a.unpublished, when)
self.assertTrue(product_2_b.has_been_published)
self.assertFalse(product_2_b.is_published)
self.assertEqual(product_2_b.unpublished, when)
# Make sure Job 3 products are published
product_3_a = ProductFile.objects.get(id=product_3_a.id)
product_3_b = ProductFile.objects.get(id=product_3_b.id)
self.assertTrue(product_3_a.has_been_published)
self.assertTrue(product_3_a.is_published)
self.assertFalse(product_3_a.is_superseded)
self.assertEqual(product_3_a.published, when)
self.assertIsNone(product_3_a.superseded)
self.assertTrue(product_3_b.has_been_published)
self.assertTrue(product_3_b.is_published)
self.assertFalse(product_3_b.is_superseded)
self.assertEqual(product_3_b.published, when)
self.assertIsNone(product_3_b.superseded)
def test_publish_products_supersede_products(self):
"""Tests calling ProductFileManager.publish_products() where products with existing UUIDs get superseded
"""
# Create existing products with known UUIDs
builder = hashlib.md5()
builder.update('test')
uuid_1 = builder.hexdigest()
builder.update('hello')
uuid_2 = builder.hexdigest()
builder.update('again')
uuid_3 = builder.hexdigest()
product_a = prod_test_utils.create_product(uuid=uuid_1, has_been_published=True, is_published=True)
product_b = prod_test_utils.create_product(uuid=uuid_2, has_been_published=True, is_published=True)
product_c = prod_test_utils.create_product(uuid=uuid_3, has_been_published=True, is_published=True)
# Set the new products with the same UUIDs
ProductFile.objects.filter(id=self.product_1.id).update(uuid=uuid_1)
ProductFile.objects.filter(id=self.product_2.id).update(uuid=uuid_2)
ProductFile.objects.filter(id=self.product_3.id).update(uuid=uuid_3)
# Publish new products
when = now()
ProductFile.objects.publish_products(self.job_exe, when)
# Check old products to make sure they are superseded
product_a = ProductFile.objects.get(id=product_a.id)
product_b = ProductFile.objects.get(id=product_b.id)
product_c = ProductFile.objects.get(id=product_c.id)
self.assertFalse(product_a.is_published)
self.assertTrue(product_a.is_superseded)
self.assertEqual(product_a.superseded, when)
self.assertFalse(product_b.is_published)
self.assertTrue(product_b.is_superseded)
self.assertEqual(product_b.superseded, when)
self.assertFalse(product_c.is_published)
self.assertTrue(product_c.is_superseded)
self.assertEqual(product_c.superseded, when)
def setUp(self):
django.setup()
self.job_exe_1 = job_test_utils.create_job_exe()
self.job_type_1_id = self.job_exe_1.job.job_type.id
self.job_exe_2 = job_test_utils.create_job_exe()
self.job_type_2_id = self.job_exe_2.job.job_type.id
self.product_1 = prod_test_utils.create_product()
self.product_2 = prod_test_utils.create_product(has_been_published=True)
self.product_3 = prod_test_utils.create_product(self.job_exe_2, has_been_published=True)
time.sleep(0.001)
self.last_modified_start = now()
self.product_4 = prod_test_utils.create_product()
self.product_5 = prod_test_utils.create_product(self.job_exe_2)
self.product_6 = prod_test_utils.create_product(self.job_exe_2, has_been_published=True)
time.sleep(0.001)
self.product_7 = prod_test_utils.create_product(self.job_exe_1, has_been_published=True)
time.sleep(0.001)
self.product_8 = prod_test_utils.create_product(has_been_published=True)
self.last_modified_end = now()
def setUp(self):
django.setup()
self.file = storage_test_utils.create_file()
self.job = job_test_utils.create_job(data={"input_data": [{"name": "input_file", "file_id": self.file.id}]})
# Attempt to stage related models
self.job_exe = job_test_utils.create_job_exe(job=self.job)
try:
import recipe.test.utils as recipe_test_utils
self.recipe = recipe_test_utils.create_recipe()
self.recipe_job = recipe_test_utils.create_recipe_job(recipe, job=self.job)
except:
self.recipe = None
self.receip_job = None
try:
import product.test.utils as product_test_utils
self.product = product_test_utils.create_product(job_exe=self.job_exe)
except:
self.product = None
def test_successful_supersede_different_recipe_type(self):
"""Tests calling RecipeManager.create_recipe() to supersede a recipe with a different recipe type version that
has one identical node, and deletes another node to replace it with a new one.
"""
interface_3 = {
'version': '1.0',
'command': 'my_command',
'command_arguments': 'args',
'input_data': [{
'name': 'Test Input 3',
'type': 'files',
'media_types': ['image/tiff'],
}],
'output_data': [{
'name': 'Test Output 3',
'type': 'file',
}]}
job_type_3 = job_test_utils.create_job_type(interface=interface_3)
new_definition = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input',
'type': 'file',
'media_types': ['text/plain'],
}],
'jobs': [{
'name': 'Job 1',
'job_type': {
'name': self.job_type_1.name,
'version': self.job_type_1.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input',
'job_input': 'Test Input 1',
}]
}, {
'name': 'Job 3',
'job_type': {
'name': job_type_3.name,
'version': job_type_3.version,
},
'dependencies': [{
'name': 'Job 1',
'connections': [{
'output': 'Test Output 1',
'input': 'Test Input 3',
}]
}]
}]
}
new_recipe_type = recipe_test_utils.create_recipe_type(name=self.recipe_type.name, definition=new_definition)
event = trigger_test_utils.create_trigger_event()
handler = Recipe.objects.create_recipe(recipe_type=self.recipe_type, event=event, data=RecipeData(self.data))
recipe = Recipe.objects.get(id=handler.recipe.id)
recipe_job_1 = RecipeJob.objects.select_related('job').get(recipe_id=handler.recipe.id, job_name='Job 1')
recipe_job_2 = RecipeJob.objects.select_related('job').get(recipe_id=handler.recipe.id, job_name='Job 2')
job_exe_2 = job_test_utils.create_job_exe(job=recipe_job_2.job)
try:
from product.models import ProductFile
from product.test import utils as product_test_utils
product = product_test_utils.create_product(job_exe=job_exe_2, has_been_published=True, is_published=True)
except ImportError:
product = None
superseded_jobs = {'Job 1': recipe_job_1.job, 'Job 2': recipe_job_2.job}
# Create a new recipe with a different version
graph_a = self.recipe_type.get_recipe_definition().get_graph()
graph_b = new_recipe_type.get_recipe_definition().get_graph()
delta = RecipeGraphDelta(graph_a, graph_b)
new_handler = Recipe.objects.create_recipe(recipe_type=new_recipe_type, event=event, data=None,
superseded_recipe=recipe, delta=delta,
superseded_jobs=superseded_jobs)
# Check that old recipe and job 2 are superseded, job 1 should be copied (not superseded)
recipe = Recipe.objects.get(id=recipe.id)
job_1 = Job.objects.get(id=recipe_job_1.job_id)
job_2 = Job.objects.get(id=recipe_job_2.job_id)
self.assertTrue(recipe.is_superseded)
self.assertFalse(job_1.is_superseded)
self.assertTrue(job_2.is_superseded)
# Check that product of job 2 (which was superseded with no new job) was unpublished
if product:
product = ProductFile.objects.get(id=product.id)
self.assertFalse(product.is_published)
self.assertIsNotNone(product.unpublished)
# Check that new recipe supersedes the old one, job 1 is copied from old recipe, and job 2 is new and does not
# supersede anything
new_recipe = Recipe.objects.get(id=new_handler.recipe.id)
new_recipe_job_1 = RecipeJob.objects.select_related('job').get(recipe_id=new_handler.recipe.id,
job_name='Job 1')
new_recipe_job_2 = RecipeJob.objects.select_related('job').get(recipe_id=new_handler.recipe.id,
job_name='Job 3')
self.assertEqual(new_recipe.superseded_recipe_id, recipe.id)
self.assertEqual(new_recipe.root_superseded_recipe_id, recipe.id)
self.assertDictEqual(new_recipe.data, recipe.data)
self.assertEqual(new_recipe_job_1.job.id, job_1.id)
self.assertFalse(new_recipe_job_1.is_original)
self.assertIsNone(new_recipe_job_1.job.superseded_job)
self.assertIsNone(new_recipe_job_1.job.root_superseded_job)
self.assertNotEqual(new_recipe_job_2.job.id, job_2.id)
self.assertTrue(new_recipe_job_2.is_original)
self.assertIsNone(new_recipe_job_2.job.superseded_job_id)
self.assertIsNone(new_recipe_job_2.job.root_superseded_job_id)
def test_successful_supersede(self):
"""Tests calling QueueManager.queue_new_recipe() successfully when superseding a recipe."""
# Queue initial recipe and complete its first job
handler = Queue.objects.queue_new_recipe(self.recipe_type, self.data,
self.event)
recipe = Recipe.objects.get(id=handler.recipe.id)
recipe_job_1 = RecipeNode.objects.select_related('job')
recipe_job_1 = recipe_job_1.get(recipe_id=handler.recipe.id,
node_name='Job 1')
Job.objects.update_jobs_to_running([recipe_job_1.job], now())
results = JobResults()
results.add_file_list_parameter(
'Test Output 1', [product_test_utils.create_product().id])
job_test_utils.create_job_exe(job=recipe_job_1.job,
status='COMPLETED',
output=results)
Job.objects.update_jobs_to_completed([recipe_job_1.job], now())
Job.objects.process_job_output([recipe_job_1.job_id], now())
# Create a new recipe type that has a new version of job 2 (job 1 is identical)
new_job_type_2 = job_test_utils.create_job_type(
name=self.job_type_2.name,
version='New Version',
interface=self.job_type_2.manifest)
new_definition = {
'version':
'1.0',
'input_data': [{
'name': 'Recipe Input',
'type': 'file',
'media_types': ['text/plain'],
}],
'jobs': [{
'name':
'New Job 1',
'job_type': {
'name': self.job_type_1.name,
'version': self.job_type_1.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input',
'job_input': 'Test Input 1',
}]
}, {
'name':
'New Job 2',
'job_type': {
'name': new_job_type_2.name,
'version': new_job_type_2.version,
},
'dependencies': [{
'name':
'New Job 1',
'connections': [{
'output': 'Test Output 1',
'input': 'Test Input 2',
}]
}]
}]
}
new_recipe_type = recipe_test_utils.create_recipe_type(
name=self.recipe_type.name, definition=new_definition)
event = trigger_test_utils.create_trigger_event()
recipe_job_1 = RecipeNode.objects.select_related('job').get(
recipe_id=handler.recipe.id, node_name='Job 1')
recipe_job_2 = RecipeNode.objects.select_related('job').get(
recipe_id=handler.recipe.id, node_name='Job 2')
superseded_jobs = {
'Job 1': recipe_job_1.job,
'Job 2': recipe_job_2.job
}
graph_a = self.recipe_type.get_recipe_definition().get_graph()
graph_b = new_recipe_type.get_recipe_definition().get_graph()
delta = RecipeGraphDelta(graph_a, graph_b)
# Queue new recipe that supersedes the old recipe
new_handler = Queue.objects.queue_new_recipe(
new_recipe_type,
None,
event,
superseded_recipe=recipe,
delta=delta,
superseded_jobs=superseded_jobs)
# Ensure old recipe is superseded
recipe = Recipe.objects.get(id=handler.recipe.id)
self.assertTrue(recipe.is_superseded)
# Ensure new recipe supersedes old recipe
new_recipe = Recipe.objects.get(id=new_handler.recipe.id)
self.assertEqual(new_recipe.superseded_recipe_id, handler.recipe.id)
# Ensure that job 1 is already completed (it was copied from original recipe) and that job 2 is queued
new_recipe_job_1 = RecipeNode.objects.select_related('job').get(
recipe_id=new_handler.recipe.id, node_name='New Job 1')
new_recipe_job_2 = RecipeNode.objects.select_related('job').get(
recipe_id=new_handler.recipe.id, node_name='New Job 2')
self.assertEqual(new_recipe_job_1.job.status, 'COMPLETED')
self.assertFalse(new_recipe_job_1.is_original)
self.assertEqual(new_recipe_job_2.job.status, 'QUEUED')
self.assertTrue(new_recipe_job_2.is_original)
def test_successful_supersede_different_recipe_type(self):
"""Tests calling RecipeManager.create_recipe() to supersede a recipe with a different recipe type version that
has one identical node, and deletes another node to replace it with a new one.
"""
interface_3 = {
'version':
'1.0',
'command':
'my_command',
'command_arguments':
'args',
'input_data': [{
'name': 'Test Input 3',
'type': 'files',
'media_types': ['image/tiff'],
}],
'output_data': [{
'name': 'Test Output 3',
'type': 'file',
}]
}
job_type_3 = job_test_utils.create_job_type(interface=interface_3)
new_definition = {
'version':
'1.0',
'input_data': [{
'name': 'Recipe Input',
'type': 'file',
'media_types': ['text/plain'],
}],
'jobs': [{
'name':
'Job 1',
'job_type': {
'name': self.job_type_1.name,
'version': self.job_type_1.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input',
'job_input': 'Test Input 1',
}]
}, {
'name':
'Job 3',
'job_type': {
'name': job_type_3.name,
'version': job_type_3.version,
},
'dependencies': [{
'name':
'Job 1',
'connections': [{
'output': 'Test Output 1',
'input': 'Test Input 3',
}]
}]
}]
}
new_recipe_type = recipe_test_utils.create_recipe_type(
name=self.recipe_type.name, definition=new_definition)
event = trigger_test_utils.create_trigger_event()
handler = Recipe.objects.create_recipe(recipe_type=self.recipe_type,
event=event,
data=RecipeData(self.data))
recipe = Recipe.objects.get(id=handler.recipe.id)
recipe_job_1 = RecipeJob.objects.select_related('job').get(
recipe_id=handler.recipe.id, job_name='Job 1')
recipe_job_2 = RecipeJob.objects.select_related('job').get(
recipe_id=handler.recipe.id, job_name='Job 2')
job_exe_2 = job_test_utils.create_job_exe(job=recipe_job_2.job)
try:
from product.models import ProductFile
from product.test import utils as product_test_utils
product = product_test_utils.create_product(
job_exe=job_exe_2, has_been_published=True, is_published=True)
except ImportError:
product = None
superseded_jobs = {
'Job 1': recipe_job_1.job,
'Job 2': recipe_job_2.job
}
# Create a new recipe with a different version
graph_a = self.recipe_type.get_recipe_definition().get_graph()
graph_b = new_recipe_type.get_recipe_definition().get_graph()
delta = RecipeGraphDelta(graph_a, graph_b)
new_handler = Recipe.objects.create_recipe(
recipe_type=new_recipe_type,
event=event,
data=None,
superseded_recipe=recipe,
delta=delta,
superseded_jobs=superseded_jobs)
# Check that old recipe and job 2 are superseded, job 1 should be copied (not superseded)
recipe = Recipe.objects.get(id=recipe.id)
job_1 = Job.objects.get(id=recipe_job_1.job_id)
job_2 = Job.objects.get(id=recipe_job_2.job_id)
self.assertTrue(recipe.is_superseded)
self.assertFalse(job_1.is_superseded)
self.assertTrue(job_2.is_superseded)
# Check that product of job 2 (which was superseded with no new job) was unpublished
if product:
product = ProductFile.objects.get(id=product.id)
self.assertFalse(product.is_published)
self.assertIsNotNone(product.unpublished)
# Check that new recipe supersedes the old one, job 1 is copied from old recipe, and job 2 is new and does not
# supersede anything
new_recipe = Recipe.objects.get(id=new_handler.recipe.id)
new_recipe_job_1 = RecipeJob.objects.select_related('job').get(
recipe_id=new_handler.recipe.id, job_name='Job 1')
new_recipe_job_2 = RecipeJob.objects.select_related('job').get(
recipe_id=new_handler.recipe.id, job_name='Job 3')
self.assertEqual(new_recipe.superseded_recipe_id, recipe.id)
self.assertEqual(new_recipe.root_superseded_recipe_id, recipe.id)
self.assertDictEqual(new_recipe.data, recipe.data)
self.assertEqual(new_recipe_job_1.job.id, job_1.id)
self.assertFalse(new_recipe_job_1.is_original)
self.assertIsNone(new_recipe_job_1.job.superseded_job)
self.assertIsNone(new_recipe_job_1.job.root_superseded_job)
self.assertNotEqual(new_recipe_job_2.job.id, job_2.id)
self.assertTrue(new_recipe_job_2.is_original)
self.assertIsNone(new_recipe_job_2.job.superseded_job_id)
self.assertIsNone(new_recipe_job_2.job.root_superseded_job_id)
def test_successful_supersede_mixed(self):
"""Tests calling QueueManager.queue_new_recipe() successfully when superseding a recipe where the results of a
Seed job get passed to the input of a legacy job
"""
workspace = storage_test_utils.create_workspace()
source_file = source_test_utils.create_source(workspace=workspace)
event = trigger_test_utils.create_trigger_event()
interface_1 = {
'seedVersion': '1.0.0',
'job': {
'name': 'job-type-a',
'jobVersion': '1.0.0',
'packageVersion': '1.0.0',
'title': 'Job Type 1',
'description': 'This is a description',
'maintainer': {
'name': 'John Doe',
'email': '*****@*****.**'
},
'timeout': 10,
'interface': {
'command': '',
'inputs': {
'files': [{
'name': 'test-input-a'
}]
},
'outputs': {
'files': [{
'name': 'test-output-a',
'pattern': '*.png'
}]
}
}
}
}
job_type_1 = job_test_utils.create_seed_job_type(manifest=interface_1)
interface_2 = {
'version':
'1.0',
'command':
'test_command',
'command_arguments':
'test_arg',
'input_data': [{
'name': 'Test Input 2',
'type': 'file',
'media_types': ['image/png', 'image/tiff'],
}],
'output_data': [{
'name': 'Test Output 2',
'type': 'file',
}]
}
job_type_2 = job_test_utils.create_job_type(interface=interface_2)
definition = {
'version':
'1.0',
'input_data': [{
'name': 'Recipe Input',
'type': 'file',
'media_types': ['text/plain'],
}],
'jobs': [{
'name':
'Job 1',
'job_type': {
'name': job_type_1.name,
'version': job_type_1.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input',
'job_input': 'test-input-a',
}]
}, {
'name':
'Job 2',
'job_type': {
'name': job_type_2.name,
'version': job_type_2.version,
},
'dependencies': [{
'name':
'Job 1',
'connections': [{
'output': 'test-output-a',
'input': 'Test Input 2',
}]
}]
}]
}
recipe_definition = RecipeDefinition(definition)
recipe_definition.validate_job_interfaces()
recipe_type = recipe_test_utils.create_recipe_type(
definition=definition)
data = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input',
'file_id': source_file.id,
}],
'workspace_id': workspace.id,
}
data = LegacyRecipeData(data)
# Queue initial recipe and complete its first job
handler = Queue.objects.queue_new_recipe(recipe_type, data, event)
recipe = Recipe.objects.get(id=handler.recipe.id)
recipe_job_1 = RecipeNode.objects.select_related('job')
recipe_job_1 = recipe_job_1.get(recipe_id=handler.recipe.id,
node_name='Job 1')
Job.objects.update_jobs_to_running([recipe_job_1.job], now())
results = JobResults()
results.add_file_parameter('test-output-a',
product_test_utils.create_product().id)
job_test_utils.create_job_exe(job=recipe_job_1.job,
status='COMPLETED',
output=results)
Job.objects.update_jobs_to_completed([recipe_job_1.job], now())
Job.objects.process_job_output([recipe_job_1.job_id], now())
# Create a new recipe type that has a new version of job 2 (job 1 is identical)
new_job_type_2 = job_test_utils.create_job_type(
name=job_type_2.name,
version='New Version',
interface=job_type_2.manifest)
new_definition = {
'version':
'1.0',
'input_data': [{
'name': 'Recipe Input',
'type': 'file',
'media_types': ['text/plain'],
}],
'jobs': [{
'name':
'New Job 1',
'job_type': {
'name': job_type_1.name,
'version': job_type_1.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input',
'job_input': 'test-input-a',
}]
}, {
'name':
'New Job 2',
'job_type': {
'name': new_job_type_2.name,
'version': new_job_type_2.version,
},
'dependencies': [{
'name':
'New Job 1',
'connections': [{
'output': 'test-output-a',
'input': 'Test Input 2',
}]
}]
}]
}
new_recipe_type = recipe_test_utils.create_recipe_type(
name=recipe_type.name, definition=new_definition)
event = trigger_test_utils.create_trigger_event()
recipe_job_1 = RecipeNode.objects.select_related('job').get(
recipe_id=handler.recipe.id, node_name='Job 1')
recipe_job_2 = RecipeNode.objects.select_related('job').get(
recipe_id=handler.recipe.id, node_name='Job 2')
superseded_jobs = {
'Job 1': recipe_job_1.job,
'Job 2': recipe_job_2.job
}
graph_a = recipe_type.get_recipe_definition().get_graph()
graph_b = new_recipe_type.get_recipe_definition().get_graph()
delta = RecipeGraphDelta(graph_a, graph_b)
# Queue new recipe that supersedes the old recipe
new_handler = Queue.objects.queue_new_recipe(
new_recipe_type,
None,
event,
superseded_recipe=recipe,
delta=delta,
superseded_jobs=superseded_jobs)
# Ensure old recipe is superseded
recipe = Recipe.objects.get(id=handler.recipe.id)
self.assertTrue(recipe.is_superseded)
# Ensure new recipe supersedes old recipe
new_recipe = Recipe.objects.get(id=new_handler.recipe.id)
self.assertEqual(new_recipe.superseded_recipe_id, handler.recipe.id)
# Ensure that job 1 is already completed (it was copied from original recipe) and that job 2 is queued
new_recipe_job_1 = RecipeNode.objects.select_related('job').get(
recipe_id=new_handler.recipe.id, node_name='New Job 1')
new_recipe_job_2 = RecipeNode.objects.select_related('job').get(
recipe_id=new_handler.recipe.id, node_name='New Job 2')
self.assertEqual(new_recipe_job_1.job.status, 'COMPLETED')
self.assertFalse(new_recipe_job_1.is_original)
self.assertEqual(new_recipe_job_2.job.status, 'QUEUED')
self.assertTrue(new_recipe_job_2.is_original)
def test_successful_supersede(self):
"""Tests calling QueueManager.queue_new_recipe() successfully when superseding a recipe."""
# Queue initial recipe and complete its first job
node = node_test_utils.create_node()
recipe_id = Queue.objects.queue_new_recipe(self.recipe_type, self.data, self.event)
recipe = Recipe.objects.get(id=recipe_id)
recipe_job_1 = RecipeJob.objects.select_related('job__job_exe').get(recipe_id=recipe_id, job_name='Job 1')
job_exe_1 = JobExecution.objects.get(job_id=recipe_job_1.job_id)
queued_job_exe = QueuedJobExecution(Queue.objects.get(job_exe_id=job_exe_1.id))
queued_job_exe.accepted(node, JobResources(cpus=10, mem=1000, disk_in=1000, disk_out=1000, disk_total=2000))
Queue.objects.schedule_job_executions('123', [queued_job_exe], {})
results = JobResults()
results.add_file_list_parameter('Test Output 1', [product_test_utils.create_product().file_id])
JobExecution.objects.filter(id=job_exe_1.id).update(results=results.get_dict())
Queue.objects.handle_job_completion(job_exe_1.id, now())
# Create a new recipe type that has a new version of job 2 (job 1 is identical)
new_job_type_2 = job_test_utils.create_job_type(name=self.job_type_2.name, version='New Version',
interface=self.job_type_2.interface)
new_definition = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input',
'type': 'file',
'media_types': ['text/plain'],
}],
'jobs': [{
'name': 'New Job 1',
'job_type': {
'name': self.job_type_1.name,
'version': self.job_type_1.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input',
'job_input': 'Test Input 1',
}]
}, {
'name': 'New Job 2',
'job_type': {
'name': new_job_type_2.name,
'version': new_job_type_2.version,
},
'dependencies': [{
'name': 'New Job 1',
'connections': [{
'output': 'Test Output 1',
'input': 'Test Input 2',
}]
}]
}]
}
new_recipe_type = recipe_test_utils.create_recipe_type(name=self.recipe_type.name, definition=new_definition)
event = trigger_test_utils.create_trigger_event()
recipe_job_1 = RecipeJob.objects.select_related('job').get(recipe_id=recipe_id, job_name='Job 1')
recipe_job_2 = RecipeJob.objects.select_related('job').get(recipe_id=recipe_id, job_name='Job 2')
superseded_jobs = {'Job 1': recipe_job_1.job, 'Job 2': recipe_job_2.job}
graph_a = self.recipe_type.get_recipe_definition().get_graph()
graph_b = new_recipe_type.get_recipe_definition().get_graph()
delta = RecipeGraphDelta(graph_a, graph_b)
# Queue new recipe that supersedes the old recipe
new_recipe_id = Queue.objects.queue_new_recipe(new_recipe_type, None, event, recipe, delta, superseded_jobs)
# Ensure old recipe is superseded
recipe = Recipe.objects.get(id=recipe_id)
self.assertTrue(recipe.is_superseded)
# Ensure new recipe supersedes old recipe
new_recipe = Recipe.objects.get(id=new_recipe_id)
self.assertEqual(new_recipe.superseded_recipe_id, recipe_id)
# Ensure that job 1 is already completed (it was copied from original recipe) and that job 2 is queued
new_recipe_job_1 = RecipeJob.objects.select_related('job').get(recipe_id=new_recipe_id, job_name='New Job 1')
new_recipe_job_2 = RecipeJob.objects.select_related('job').get(recipe_id=new_recipe_id, job_name='New Job 2')
self.assertEqual(new_recipe_job_1.job.status, 'COMPLETED')
self.assertFalse(new_recipe_job_1.is_original)
self.assertEqual(new_recipe_job_2.job.status, 'QUEUED')
self.assertTrue(new_recipe_job_2.is_original)
# Complete both the old and new job 2 and check that only the new recipe completes
job_exe_2 = JobExecution.objects.get(job_id=recipe_job_2.job_id)
queued_job_exe_2 = QueuedJobExecution(Queue.objects.get(job_exe_id=job_exe_2.id))
queued_job_exe_2.accepted(node, JobResources(cpus=10, mem=1000, disk_in=1000, disk_out=1000, disk_total=2000))
Queue.objects.schedule_job_executions('123', [queued_job_exe_2], {})
Queue.objects.handle_job_completion(job_exe_2.id, now())
new_job_exe_2 = JobExecution.objects.get(job_id=new_recipe_job_2.job_id)
new_queued_job_exe_2 = QueuedJobExecution(Queue.objects.get(job_exe_id=new_job_exe_2.id))
new_queued_job_exe_2.accepted(node, JobResources(cpus=10, mem=1000, disk_in=1000, disk_out=1000,
disk_total=2000))
Queue.objects.schedule_job_executions('123', [new_queued_job_exe_2], {})
Queue.objects.handle_job_completion(new_job_exe_2.id, now())
recipe = Recipe.objects.get(id=recipe.id)
new_recipe = Recipe.objects.get(id=new_recipe.id)
self.assertIsNone(recipe.completed)
self.assertIsNotNone(new_recipe.completed)
