def create_recipe_handler(recipe_type=None,
data=None,
event=None,
superseded_recipe=None,
delta=None,
superseded_jobs=None):
"""Creates a recipe along with its declared jobs for unit testing
:returns: The recipe handler with created recipe and jobs
:rtype: :class:`recipe.handlers.handler.RecipeHandler`
"""
if not recipe_type:
recipe_type = create_recipe_type()
if not data:
data = {}
if not isinstance(data, LegacyRecipeData):
data = LegacyRecipeData(data)
if not event:
event = trigger_test_utils.create_trigger_event()
if superseded_recipe and not delta:
delta = RecipeGraphDelta(RecipeGraph(), RecipeGraph())
return Recipe.objects.create_recipe_old(
recipe_type,
data,
event,
superseded_recipe=superseded_recipe,
delta=delta,
superseded_jobs=superseded_jobs)
def create_job(job_type=None, event=None, status='PENDING', error=None, data=None, num_exes=0, queued=None,
started=None, ended=None, last_status_change=None, priority=100, results=None):
'''Creates a job model for unit testing
:returns: The job model
:rtype: :class:`job.models.Job`
'''
if not job_type:
job_type = create_job_type()
if not event:
event = trigger_test_utils.create_trigger_event()
if not last_status_change:
last_status_change = timezone.now()
if not data:
data = {
'version': '1.0',
'input_data': [],
'output_data': [],
}
job = Job.objects.create_job(job_type, event)
job.priority = priority
job.data = data
job.status = status
job.num_exes = num_exes
job.queued = queued
job.started = started
job.ended = ended
job.last_status_change = last_status_change
job.error = error
job.results = results
job.save()
return job
def create_recipe(recipe_type=None, data=None, event=None, is_superseded=False, superseded=None):
"""Creates a recipe for unit testing
:returns: The recipe model
:rtype: :class:`recipe.models.Recipe`
"""
if not recipe_type:
recipe_type = create_recipe_type()
if not data:
data = {}
if not event:
event = trigger_test_utils.create_trigger_event()
if is_superseded and not superseded:
superseded = timezone.now()
recipe = Recipe()
recipe.recipe_type = recipe_type
recipe.recipe_type_rev = RecipeTypeRevision.objects.get_revision(recipe_type.id, recipe_type.revision_num)
recipe.event = event
recipe.data = data
recipe.is_superseded = is_superseded
recipe.superseded = superseded
recipe.save()
return recipe
def test_execute_with_job(self):
"""Tests calling PurgeSourceFile.execute() successfully"""
# Create a file
source_file = storage_test_utils.create_file(file_type='SOURCE')
trigger = trigger_test_utils.create_trigger_event()
PurgeResults.objects.create(source_file_id=source_file.id,
trigger_event=trigger)
# Create a job and other models
job = job_test_utils.create_job()
job_test_utils.create_input_file(job=job, input_file=source_file)
# Create message
message = create_purge_source_file_message(
source_file_id=source_file.id, trigger_id=trigger.id)
# Execute message
result = message.execute()
self.assertTrue(result)
# Test to see that a message to purge the job was created
self.assertEqual(len(message.new_messages), 1)
for msg in message.new_messages:
self.assertEqual(msg.job_id, job.id)
self.assertEqual(msg.type, 'spawn_delete_files_job')
def create_job(job_type=None,
event=None,
status='PENDING',
error=None,
data=None,
num_exes=0,
queued=None,
started=None,
ended=None,
last_status_change=None,
priority=100,
results=None,
superseded_job=None,
delete_superseded=True,
is_superseded=False,
superseded=None):
"""Creates a job model for unit testing
:returns: The job model
:rtype: :class:`job.models.Job`
"""
if not job_type:
job_type = create_job_type()
if not event:
event = trigger_test_utils.create_trigger_event()
if not last_status_change:
last_status_change = timezone.now()
if not data:
data = {
'version': '1.0',
'input_data': [],
'output_data': [],
}
if not results:
results = dict()
if superseded_job and not superseded_job.is_superseded:
Job.objects.supersede_jobs([superseded_job], timezone.now())
if is_superseded and not superseded:
superseded = timezone.now()
job = Job.objects.create_job(job_type,
event,
superseded_job=superseded_job,
delete_superseded=delete_superseded)
job.priority = priority
job.data = data
job.status = status
job.num_exes = num_exes
job.queued = queued
job.started = started
job.ended = ended
job.last_status_change = last_status_change
job.error = error
job.results = results
job.is_superseded = is_superseded
job.superseded = superseded
job.save()
return job
def test_execute_force_stop(self):
"""Tests calling PurgeRecipe.execute() successfully"""
# Create PurgeResults entry
file_2 = storage_test_utils.create_file(file_type='SOURCE')
trigger = trigger_test_utils.create_trigger_event()
PurgeResults.objects.create(source_file_id=file_2.id,
trigger_event=trigger,
force_stop_purge=True)
self.assertEqual(
PurgeResults.objects.values_list(
'num_recipes_deleted',
flat=True).get(trigger_event=trigger.id), 0)
# Create recipes
recipe_type = recipe_test_utils.create_recipe_type()
recipe = recipe_test_utils.create_recipe(recipe_type=recipe_type)
# Create message
message = create_purge_recipe_message(recipe_id=recipe.id,
trigger_id=trigger.id,
source_file_id=file_2.id)
# Execute message
result = message.execute()
self.assertTrue(result)
# Check results are accurate
self.assertEqual(
PurgeResults.objects.values_list(
'num_recipes_deleted',
flat=True).get(source_file_id=file_2.id), 0)
def create_recipe(recipe_type=None, data=None, event=None):
'''Creates a job type model for unit testing
:param recipe_type: The associated recipe type
:type recipe_type: :class:'recipe.models.RecipeType'
:param data: The associated data for the recipe
:type data: dict
:param event: The associated event
:type event: :class:'trigger.models.TriggerEvent'
:returns: The recipe model
:rtype: :class:`recipe.models.Recipe`
'''
if not data:
data = {}
if not recipe_type:
recipe_type = create_recipe_type()
if not event:
event = trigger_test_utils.create_trigger_event()
recipe = Recipe()
recipe.recipe_type = recipe_type
recipe.recipe_type_rev = RecipeTypeRevision.objects.get_revision(
recipe_type.id, recipe_type.revision_num)
recipe.event = event
recipe.data = data
recipe.save()
return recipe
def setUp(self):
django.setup()
cpus = 50
mem = 500
disk = 50
self.job_type_1 = job_test_utils.create_job_type(priority=1, cpus=cpus, mem=mem, disk=disk)
self.job_type_2 = job_test_utils.create_job_type(priority=2, cpus=cpus, mem=mem, disk=disk)
self.job_type_3 = job_test_utils.create_job_type(priority=3, cpus=cpus, mem=mem, disk=disk)
self.job_type_4 = job_test_utils.create_job_type(priority=4, cpus=cpus, mem=mem, disk=disk)
self.job_type_5 = job_test_utils.create_job_type(priority=5, cpus=cpus, mem=mem, disk=disk)
resource_1 = shared_resource_test_utils.create_resource()
resource_2 = shared_resource_test_utils.create_resource(is_global=False)
resource_3 = shared_resource_test_utils.create_resource(limit=1000)
shared_resource_test_utils.create_requirement(job_type=self.job_type_1, shared_resource=resource_1)
shared_resource_test_utils.create_requirement(job_type=self.job_type_2, shared_resource=resource_2)
shared_resource_test_utils.create_requirement(job_type=self.job_type_3, shared_resource=resource_3, usage=400)
shared_resource_test_utils.create_requirement(job_type=self.job_type_4, shared_resource=resource_3, usage=200)
shared_resource_test_utils.create_requirement(job_type=self.job_type_5, shared_resource=resource_3, usage=100)
self.trigger_event_1 = trigger_test_utils.create_trigger_event()
self.node_1 = node_test_utils.create_node()
self.node_2 = node_test_utils.create_node()
self.node_3 = node_test_utils.create_node()
def create_recipe(recipe_type=None, data=None, event=None):
'''Creates a job type model for unit testing
:param recipe_type: The associated recipe type
:type recipe_type: :class:'recipe.models.RecipeType'
:param data: The associated data for the recipe
:type data: dict
:param event: The associated event
:type event: :class:'trigger.models.TriggerEvent'
:returns: The recipe model
:rtype: :class:`recipe.models.Recipe`
'''
if not data:
data = {}
if not recipe_type:
recipe_type = create_recipe_type()
if not event:
event = trigger_test_utils.create_trigger_event()
recipe = Recipe()
recipe.recipe_type = recipe_type
recipe.recipe_type_rev = RecipeTypeRevision.objects.get_revision(recipe_type.id, recipe_type.revision_num)
recipe.event = event
recipe.data = data
recipe.save()
return recipe
def test_create_message(self):
"""
Tests calling the create message function for DeleteFiles
"""
job = job_test_utils.create_job()
job_exe = job_test_utils.create_job_exe(job=job)
trigger = trigger_test_utils.create_trigger_event()
file_path_1 = os.path.join('my_dir', 'my_file.txt')
file_path_2 = os.path.join('my_dir', 'my_file1.json')
file_path_3 = os.path.join('my_dir', 'my_file2.json')
file_path_4 = os.path.join('my_dir', 'my_file3.json')
file_1 = storage_test_utils.create_file(file_path=file_path_1,
job_exe=job_exe)
file_2 = storage_test_utils.create_file(file_path=file_path_2,
job_exe=job_exe)
file_3 = storage_test_utils.create_file(file_path=file_path_3,
job_exe=job_exe)
file_4 = storage_test_utils.create_file(file_path=file_path_4,
job_exe=job_exe)
files = [file_1, file_2, file_3, file_4]
# No exception is success
create_delete_files_messages(files=files,
job_id=job.id,
trigger_id=trigger.id,
source_file_id=self.source_file.id,
purge=True)
def test_json(self):
"""Tests coverting a PurgeJobs message to and from JSON"""
input_file = storage_test_utils.create_file(file_type='SOURCE')
trigger = trigger_test_utils.create_trigger_event()
PurgeResults.objects.create(source_file_id=input_file.id, trigger_event=trigger)
job_exe = job_test_utils.create_job_exe(status='COMPLETED')
job = job_exe.job
# Add job to message
message = PurgeJobs()
message._purge_job_ids = [job.id]
message.trigger_id = trigger.id
message.source_file_id = input_file.id
message.status_change = timezone.now()
# Convert message to JSON and back, and then execute
message_json_dict = message.to_json()
new_message = PurgeJobs.from_json(message_json_dict)
result = new_message.execute()
self.assertTrue(result)
# Check that job is deleted
self.assertEqual(Job.objects.filter(id=job.id).count(), 0)
self.assertEqual(JobExecution.objects.filter(id=job_exe.id).count(), 0)
def test_execute_with_recipe(self):
"""Tests calling PurgeJobs.execute() successfully with job as part of recipe"""
recipe = recipe_test_utils.create_recipe()
job_exe = job_test_utils.create_job_exe(status='COMPLETED')
job = job_exe.job
recipe_test_utils.create_recipe_node(recipe=recipe, node_name='A', job=job, save=True)
source_file = storage_test_utils.create_file(file_type='SOURCE')
trigger = trigger_test_utils.create_trigger_event()
PurgeResults.objects.create(source_file_id=source_file.id, trigger_event=trigger)
# Add job to message
message = PurgeJobs()
message._purge_job_ids = [job.id]
message.trigger_id = trigger.id
message.source_file_id = source_file.id
message.status_change = timezone.now()
# Execute message
result = message.execute()
self.assertTrue(result)
# Check that a new message to purge source file was created
msgs = [msg for msg in message.new_messages if msg.type == 'purge_recipe']
self.assertEqual(len(msgs), 1)
for msg in msgs:
self.assertEqual(msg.recipe_id, recipe.id)
def setUp(self):
django.setup()
self.source_file = storage_test_utils.create_file(file_type='SOURCE')
trigger = trigger_test_utils.create_trigger_event()
PurgeResults.objects.create(source_file_id=self.source_file.id,
trigger_event=trigger)
def test_execute_force_stop_purge(self):
"""Tests calling PurgeJobs.execute() successfully with force_stop_purge set (no action should be completed)"""
# Create PurgeResults entry
source_file = storage_test_utils.create_file()
trigger = trigger_test_utils.create_trigger_event()
PurgeResults.objects.create(source_file_id=source_file.id, trigger_event=trigger, force_stop_purge=True)
self.assertEqual(PurgeResults.objects.values_list('num_jobs_deleted', flat=True).get(
source_file_id=source_file.id), 0)
job_exe_1 = job_test_utils.create_job_exe(status='COMPLETED')
job_exe_2 = job_test_utils.create_job_exe(status='COMPLETED')
job_exe_3 = job_test_utils.create_job_exe(status='COMPLETED')
job_1 = job_exe_1.job
job_2 = job_exe_2.job
job_3 = job_exe_3.job
# Add job to message
message = PurgeJobs()
message.source_file_id = source_file.id
message.trigger_id = trigger.id
message._purge_job_ids = [job_1.id, job_2.id, job_3.id]
message.status_change = timezone.now()
# Execute message
result = message.execute()
self.assertTrue(result)
# Check results are accurate
self.assertEqual(PurgeResults.objects.values_list('num_jobs_deleted', flat=True).get(
source_file_id=source_file.id), 0)
def create_recipe(recipe_type=None, data=None, event=None, is_superseded=False, superseded=None):
"""Creates a recipe for unit testing
:returns: The recipe model
:rtype: :class:`recipe.models.Recipe`
"""
if not recipe_type:
recipe_type = create_recipe_type()
if not data:
data = {}
if not event:
event = trigger_test_utils.create_trigger_event()
if is_superseded and not superseded:
superseded = timezone.now()
recipe = Recipe()
recipe.recipe_type = recipe_type
recipe.recipe_type_rev = RecipeTypeRevision.objects.get_revision(recipe_type.id, recipe_type.revision_num)
recipe.event = event
recipe.data = data
recipe.is_superseded = is_superseded
recipe.superseded = superseded
recipe.save()
return recipe
def test_execute_force_stop(self):
"""Tests calling PurgeSourceFile.execute() successfully"""
# Create a file
source_file = storage_test_utils.create_file(file_type='SOURCE')
# Create PurgeResults entry
trigger = trigger_test_utils.create_trigger_event()
PurgeResults.objects.create(source_file_id=source_file.id,
trigger_event=trigger,
force_stop_purge=True)
self.assertIsNone(
PurgeResults.objects.values_list(
'purge_completed', flat=True).get(trigger_event=trigger))
# Create message
message = create_purge_source_file_message(
source_file_id=source_file.id, trigger_id=trigger.id)
# Execute message
result = message.execute()
self.assertTrue(result)
# Test to see that the PurgeResults was completed
self.assertIsNone(
PurgeResults.objects.values_list(
'purge_completed', flat=True).get(trigger_event=trigger))
def test_execute_with_recipe(self):
"""Tests calling PurgeSourceFile.execute() successfully"""
# Create a file
source_file = storage_test_utils.create_file(file_type='SOURCE')
trigger = trigger_test_utils.create_trigger_event()
PurgeResults.objects.create(source_file_id=source_file.id,
trigger_event=trigger)
# Create a recipe and other models
recipe = recipe_test_utils.create_recipe()
recipe_test_utils.create_input_file(recipe=recipe,
input_file=source_file)
# Create message
message = create_purge_source_file_message(
source_file_id=source_file.id, trigger_id=trigger.id)
# Execute message
result = message.execute()
self.assertTrue(result)
# Test to see that a message to purge the recipe was created
self.assertEqual(len(message.new_messages), 1)
for msg in message.new_messages:
self.assertEqual(msg.recipe_id, recipe.id)
self.assertEqual(msg.type, 'purge_recipe')
def test_publish_products_unpublish_superseded(self):
"""Tests calling ProductFileManager.publish_products() where the job has superseded job products that must be
unpublished
"""
# Job 1 is superseded by Job 2 and Job 2 is superseded by Job 3
job_exe_1 = job_test_utils.create_job_exe()
product_1_a = prod_test_utils.create_product(job_exe=job_exe_1, has_been_published=True, is_published=True)
product_1_b = prod_test_utils.create_product(job_exe=job_exe_1, has_been_published=True, is_published=True)
job_type = job_test_utils.create_seed_job_type()
event = trigger_test_utils.create_trigger_event()
job_type_rev = JobTypeRevision.objects.get_by_natural_key(job_type, job_type.revision_num)
job_2 = Job.objects.create_job_v6(job_type_rev, event_id=event.id, superseded_job=job_exe_1.job)
job_2.save()
job_exe_2 = job_test_utils.create_job_exe(job=job_2)
Job.objects.supersede_jobs([job_exe_1.job.id], now())
product_2_a = prod_test_utils.create_product(job_exe=job_exe_2, has_been_published=True, is_published=True)
product_2_b = prod_test_utils.create_product(job_exe=job_exe_2, has_been_published=True, is_published=True)
job_3 = Job.objects.create_job_v6(job_type_rev, event_id=event.id, superseded_job=job_exe_2.job)
job_3.save()
job_exe_3 = job_test_utils.create_job_exe(job=job_3)
Job.objects.supersede_jobs([job_2.id], now())
product_3_a = prod_test_utils.create_product(job_exe=job_exe_3)
product_3_b = prod_test_utils.create_product(job_exe=job_exe_3)
when = now()
ProductFile.objects.publish_products(job_exe_3, job_3, when)
# Make sure products from Job 1 and Job 2 are unpublished
product_1_a = ScaleFile.objects.get(id=product_1_a.id)
product_1_b = ScaleFile.objects.get(id=product_1_b.id)
product_2_a = ScaleFile.objects.get(id=product_2_a.id)
product_2_b = ScaleFile.objects.get(id=product_2_b.id)
self.assertTrue(product_1_a.has_been_published)
self.assertFalse(product_1_a.is_published)
self.assertEqual(product_1_a.unpublished, when)
self.assertTrue(product_1_b.has_been_published)
self.assertFalse(product_1_b.is_published)
self.assertEqual(product_1_b.unpublished, when)
self.assertTrue(product_2_a.has_been_published)
self.assertFalse(product_2_a.is_published)
self.assertEqual(product_2_a.unpublished, when)
self.assertTrue(product_2_b.has_been_published)
self.assertFalse(product_2_b.is_published)
self.assertEqual(product_2_b.unpublished, when)
# Make sure Job 3 products are published
product_3_a = ScaleFile.objects.get(id=product_3_a.id)
product_3_b = ScaleFile.objects.get(id=product_3_b.id)
self.assertTrue(product_3_a.has_been_published)
self.assertTrue(product_3_a.is_published)
self.assertFalse(product_3_a.is_superseded)
self.assertEqual(product_3_a.published, when)
self.assertIsNone(product_3_a.superseded)
self.assertTrue(product_3_b.has_been_published)
self.assertTrue(product_3_b.is_published)
self.assertFalse(product_3_b.is_superseded)
self.assertEqual(product_3_b.published, when)
self.assertIsNone(product_3_b.superseded)
def test_execute_input_data(self):
"""Tests calling CreateJobs.execute() with input data"""
manifest = {
'seedVersion': '1.0.0',
'job': {
'name': 'name',
'jobVersion': '1.0.0',
'packageVersion': '1.0.0',
'title': 'Title',
'description': 'This is a description',
'maintainer': {
'name': 'John Doe',
'email': '*****@*****.**'
},
'timeout': 10,
'interface': {
'command': 'the command'
}
}
}
job_type = job_test_utils.create_seed_job_type(manifest=manifest)
event = trigger_test_utils.create_trigger_event()
data = Data()
# Create and execute message
message = create_jobs_message(job_type.name,
job_type.version,
job_type.revision_num,
event.id,
input_data=data)
result = message.execute()
self.assertTrue(result)
# Check for job creation
self.assertEqual(
Job.objects.filter(job_type_id=job_type.id,
event_id=event.id).count(), 1)
# Check for process_job_input message
self.assertEqual(len(message.new_messages), 1)
msg = message.new_messages[0]
self.assertEqual(msg.type, 'process_job_input')
# Test executing message again
message_json_dict = message.to_json()
message = CreateJobs.from_json(message_json_dict)
result = message.execute()
self.assertTrue(result)
# Check that a second job is not created
self.assertEqual(
Job.objects.filter(job_type_id=job_type.id,
event_id=event.id).count(), 1)
# Check for process_job_input message
self.assertEqual(len(message.new_messages), 1)
msg = message.new_messages[0]
self.assertEqual(msg.type, 'process_job_input')
def setUp(self):
django.setup()
workspace = storage_test_utils.create_workspace()
source_file = source_test_utils.create_source(workspace=workspace)
self.event = trigger_test_utils.create_trigger_event()
interface_1 = {
"version": "1.0",
"command": "test_command",
"command_arguments": "test_arg",
"input_data": [{"name": "Test Input 1", "type": "file", "media_types": ["text/plain"]}],
"output_data": [{"name": "Test Output 1", "type": "files", "media_type": "image/png"}],
}
self.job_type_1 = job_test_utils.create_job_type(interface=interface_1)
interface_2 = {
"version": "1.0",
"command": "test_command",
"command_arguments": "test_arg",
"input_data": [{"name": "Test Input 2", "type": "files", "media_types": ["image/png", "image/tiff"]}],
"output_data": [{"name": "Test Output 2", "type": "file"}],
}
self.job_type_2 = job_test_utils.create_job_type(interface=interface_2)
definition = {
"version": "1.0",
"input_data": [{"name": "Recipe Input", "type": "file", "media_types": ["text/plain"]}],
"jobs": [
{
"name": "Job 1",
"job_type": {"name": self.job_type_1.name, "version": self.job_type_1.version},
"recipe_inputs": [{"recipe_input": "Recipe Input", "job_input": "Test Input 1"}],
},
{
"name": "Job 2",
"job_type": {"name": self.job_type_2.name, "version": self.job_type_2.version},
"dependencies": [
{"name": "Job 1", "connections": [{"output": "Test Output 1", "input": "Test Input 2"}]}
],
},
],
}
recipe_definition = RecipeDefinition(definition)
recipe_definition.validate_job_interfaces()
self.recipe_type = recipe_test_utils.create_recipe_type(definition=definition)
self.data = {
"version": "1.0",
"input_data": [{"name": "Recipe Input", "file_id": source_file.id}],
"workspace_id": workspace.id,
}
# Register a fake processor
self.mock_processor = MagicMock(QueueEventProcessor)
Queue.objects.register_processor(lambda: self.mock_processor)
def test_publish_products_unpublish_superseded(self):
"""Tests calling ProductFileManager.publish_products() where the job has superseded job products that must be
unpublished
"""
# Job 1 is superseded by Job 2 and Job 2 is superseded by Job 3
job_exe_1 = job_test_utils.create_job_exe()
product_1_a = prod_test_utils.create_product(job_exe=job_exe_1, has_been_published=True, is_published=True)
product_1_b = prod_test_utils.create_product(job_exe=job_exe_1, has_been_published=True, is_published=True)
job_type = job_test_utils.create_job_type()
event = trigger_test_utils.create_trigger_event()
job_2 = Job.objects.create_job(job_type=job_type, event=event, superseded_job=job_exe_1.job)
job_2.save()
job_exe_2 = job_test_utils.create_job_exe(job=job_2)
Job.objects.supersede_jobs([job_exe_1.job], now())
product_2_a = prod_test_utils.create_product(job_exe=job_exe_2, has_been_published=True, is_published=True)
product_2_b = prod_test_utils.create_product(job_exe=job_exe_2, has_been_published=True, is_published=True)
job_3 = Job.objects.create_job(job_type=job_type, event=event, superseded_job=job_exe_2.job)
job_3.save()
job_exe_3 = job_test_utils.create_job_exe(job=job_3)
Job.objects.supersede_jobs([job_2], now())
product_3_a = prod_test_utils.create_product(job_exe=job_exe_3)
product_3_b = prod_test_utils.create_product(job_exe=job_exe_3)
when = now()
ProductFile.objects.publish_products(job_exe_3, when)
# Make sure products from Job 1 and Job 2 are unpublished
product_1_a = ProductFile.objects.get(id=product_1_a.id)
product_1_b = ProductFile.objects.get(id=product_1_b.id)
product_2_a = ProductFile.objects.get(id=product_2_a.id)
product_2_b = ProductFile.objects.get(id=product_2_b.id)
self.assertTrue(product_1_a.has_been_published)
self.assertFalse(product_1_a.is_published)
self.assertEqual(product_1_a.unpublished, when)
self.assertTrue(product_1_b.has_been_published)
self.assertFalse(product_1_b.is_published)
self.assertEqual(product_1_b.unpublished, when)
self.assertTrue(product_2_a.has_been_published)
self.assertFalse(product_2_a.is_published)
self.assertEqual(product_2_a.unpublished, when)
self.assertTrue(product_2_b.has_been_published)
self.assertFalse(product_2_b.is_published)
self.assertEqual(product_2_b.unpublished, when)
# Make sure Job 3 products are published
product_3_a = ProductFile.objects.get(id=product_3_a.id)
product_3_b = ProductFile.objects.get(id=product_3_b.id)
self.assertTrue(product_3_a.has_been_published)
self.assertTrue(product_3_a.is_published)
self.assertFalse(product_3_a.is_superseded)
self.assertEqual(product_3_a.published, when)
self.assertIsNone(product_3_a.superseded)
self.assertTrue(product_3_b.has_been_published)
self.assertTrue(product_3_b.is_published)
self.assertFalse(product_3_b.is_superseded)
self.assertEqual(product_3_b.published, when)
self.assertIsNone(product_3_b.superseded)
def setUp(self):
django.setup()
self.job_1 = job_test_utils.create_job()
self.trigger_1 = trigger_test_utils.create_trigger_event()
self.job_exe_1 = job_test_utils.create_job_exe(job=self.job_1)
self.file_1 = storage_test_utils.create_file(job_exe=self.job_exe_1)
self.source_file = storage_test_utils.create_file(file_type='SOURCE')
self.workspace = storage_test_utils.create_workspace()
def setUp(self):
django.setup()
self.job_type_1 = job_test_utils.create_job_type(priority=1)
self.job_type_2 = job_test_utils.create_job_type(priority=1)
self.job_type_3 = job_test_utils.create_job_type(priority=3)
self.job_type_4 = job_test_utils.create_job_type(priority=4)
self.job_type_5 = job_test_utils.create_job_type(priority=4)
self.trigger_event_1 = trigger_test_utils.create_trigger_event()
def test_execute_force_stop(self):
"""Tests calling DeleteFile.execute() successfully"""
# Create PurgeResults entry
source_file = storage_test_utils.create_file()
trigger = trigger_test_utils.create_trigger_event()
PurgeResults.objects.create(source_file_id=source_file.id,
trigger_event=trigger,
force_stop_purge=True)
self.assertEqual(
PurgeResults.objects.values_list(
'num_products_deleted',
flat=True).get(source_file_id=source_file.id), 0)
job = job_test_utils.create_job()
job_exe = job_test_utils.create_job_exe(job=job)
file_path_1 = os.path.join('my_dir', 'my_file.txt')
file_path_2 = os.path.join('my_dir', 'my_file1.json')
file_path_3 = os.path.join('my_dir', 'my_file2.json')
file_path_4 = os.path.join('my_dir', 'my_file3.json')
file_1 = storage_test_utils.create_file(file_path=file_path_1,
job_exe=job_exe)
file_2 = storage_test_utils.create_file(file_path=file_path_2,
job_exe=job_exe)
file_3 = storage_test_utils.create_file(file_path=file_path_3,
job_exe=job_exe)
file_4 = storage_test_utils.create_file(file_path=file_path_4,
job_exe=job_exe)
# Add files to message
message = DeleteFiles()
message.purge = True
message.job_id = job.id
message.source_file_id = source_file.id
message.trigger_id = trigger.id
if message.can_fit_more():
message.add_file(file_1.id)
if message.can_fit_more():
message.add_file(file_2.id)
if message.can_fit_more():
message.add_file(file_3.id)
if message.can_fit_more():
message.add_file(file_4.id)
# Execute message
result = message.execute()
self.assertTrue(result)
# Check results are accurate
self.assertEqual(
PurgeResults.objects.values_list(
'num_products_deleted',
flat=True).get(source_file_id=source_file.id), 0)
def test_successful(self):
'''Tests calling RecipeManager.create_recipe() successfully.'''
event = trigger_test_utils.create_trigger_event()
recipe = Recipe.objects.create_recipe(recipe_type=self.recipe_type, event=event, data=self.data)
# Make sure the recipe jobs get created with the correct job types
recipe_job_1 = RecipeJob.objects.get(recipe_id=recipe.id, job_name='Job 1')
recipe_job_2 = RecipeJob.objects.get(recipe_id=recipe.id, job_name='Job 2')
self.assertEqual(recipe_job_1.job.job_type.id, self.job_type_1.id)
self.assertEqual(recipe_job_2.job.job_type.id, self.job_type_2.id)
def test_successful(self):
"""Tests calling QueueManager.queue_new_job() successfully with a Seed job type"""
workspace = storage_test_utils.create_workspace()
source_file = source_test_utils.create_source(workspace=workspace)
event = trigger_test_utils.create_trigger_event()
manifest = {
'seedVersion': '1.0.0',
'job': {
'name': 'test-job',
'jobVersion': '1.0.0',
'packageVersion': '1.0.0',
'title': 'Test Job',
'description': 'This is a test job',
'maintainer': {
'name': 'John Doe',
'email': '*****@*****.**'
},
'timeout': 10,
'interface': {
'command': '',
'inputs': {
'files': [{
'name': 'input_a'
}]
},
'outputs': {
'files': [{
'name': 'output_a',
'multiple': True,
'pattern': '*.png'
}]
}
}
}
}
job_type = job_test_utils.create_seed_job_type(manifest=manifest)
data_dict = {
'version': '1.0',
'input_data': [{
'name': 'input_a',
'file_id': source_file.id,
}],
'output_data': [{
'name': 'output_a',
'workspace_id': workspace.id
}]
}
data = JobData(data_dict)
job = Queue.objects.queue_new_job(job_type, data, event)
self.assertEqual(job.status, 'QUEUED')
def test_successful_supersede_same_recipe_type(self):
"""Tests calling RecipeManager.create_recipe() to supersede a recipe with the same recipe type."""
event = trigger_test_utils.create_trigger_event()
handler = Recipe.objects.create_recipe(recipe_type=self.recipe_type,
event=event,
data=RecipeData(self.data))
recipe = Recipe.objects.get(id=handler.recipe.id)
recipe_job_1 = RecipeJob.objects.select_related('job').get(
recipe_id=handler.recipe.id, job_name='Job 1')
recipe_job_2 = RecipeJob.objects.select_related('job').get(
recipe_id=handler.recipe.id, job_name='Job 2')
superseded_jobs = {
'Job 1': recipe_job_1.job,
'Job 2': recipe_job_2.job
}
# Create a new recipe of the same type where we want to reprocess Job 2
graph = self.recipe_type.get_recipe_definition().get_graph()
delta = RecipeGraphDelta(graph, graph)
delta.reprocess_identical_node('Job 2') # We want to reprocess Job 2
new_handler = Recipe.objects.create_recipe(
recipe_type=self.recipe_type,
event=event,
data=None,
superseded_recipe=recipe,
delta=delta,
superseded_jobs=superseded_jobs)
# Check that old recipe and job 2 are superseded, job 1 should be copied (not superseded)
recipe = Recipe.objects.get(id=recipe.id)
job_1 = Job.objects.get(id=recipe_job_1.job_id)
job_2 = Job.objects.get(id=recipe_job_2.job_id)
self.assertTrue(recipe.is_superseded)
self.assertFalse(job_1.is_superseded)
self.assertTrue(job_2.is_superseded)
# Check that new recipe supersedes the old one, job 1 is copied from old recipe, and job 2 supersedes old job 2
new_recipe = Recipe.objects.get(id=new_handler.recipe.id)
new_recipe_job_1 = RecipeJob.objects.select_related('job').get(
recipe_id=new_handler.recipe.id, job_name='Job 1')
new_recipe_job_2 = RecipeJob.objects.select_related('job').get(
recipe_id=new_handler.recipe.id, job_name='Job 2')
self.assertEqual(new_recipe.superseded_recipe_id, recipe.id)
self.assertEqual(new_recipe.root_superseded_recipe_id, recipe.id)
self.assertDictEqual(new_recipe.data, recipe.data)
self.assertEqual(new_recipe_job_1.job.id, job_1.id)
self.assertFalse(new_recipe_job_1.is_original)
self.assertIsNone(new_recipe_job_1.job.superseded_job)
self.assertIsNone(new_recipe_job_1.job.root_superseded_job)
self.assertNotEqual(new_recipe_job_2.job.id, job_2.id)
self.assertTrue(new_recipe_job_2.is_original)
self.assertEqual(new_recipe_job_2.job.superseded_job_id, job_2.id)
self.assertEqual(new_recipe_job_2.job.root_superseded_job_id, job_2.id)
def create_job(job_type=None, event=None, status='PENDING', error=None, input=None, num_exes=1, max_tries=None,
queued=None, started=None, ended=None, last_status_change=None, priority=100, output=None,
superseded_job=None, delete_superseded=True, is_superseded=False, superseded=None, input_file_size=10.0,
save=True):
"""Creates a job model for unit testing
:returns: The job model
:rtype: :class:`job.models.Job`
"""
if not job_type:
job_type = create_job_type()
if not event:
event = trigger_test_utils.create_trigger_event()
if not last_status_change:
last_status_change = timezone.now()
if num_exes == 0:
input_file_size = None
if not input:
if num_exes == 0:
input = {}
else:
input = {
'version': '1.0',
'input_data': [],
'output_data': [],
}
if not output:
output = dict()
if superseded_job and not superseded_job.is_superseded:
Job.objects.supersede_jobs_old([superseded_job], timezone.now())
if is_superseded and not superseded:
superseded = timezone.now()
job = Job.objects.create_job(job_type, event.id, superseded_job=superseded_job, delete_superseded=delete_superseded)
job.priority = priority
job.input = input
job.status = status
job.num_exes = num_exes
job.max_tries = max_tries if max_tries is not None else job_type.max_tries
job.queued = queued
job.started = started
job.ended = ended
job.last_status_change = last_status_change
job.error = error
job.output = output
job.is_superseded = is_superseded
job.superseded = superseded
job.input_file_size = input_file_size
if save:
job.save()
return job
def test_json(self):
"""Tests coverting a ReprocessRecipes message to and from JSON"""
batch = batch_test_utils.create_batch()
event = trigger_test_utils.create_trigger_event()
# Create message
message = create_reprocess_recipes_messages(
self.old_recipe_ids,
self.recipe_1.recipe_type_rev_id,
event.id,
all_jobs=True,
job_names=['Job 1', 'Job 2'],
batch_id=batch.id)[0]
# Convert message to JSON and back, and then execute
message_json_dict = message.to_json()
new_message = ReprocessRecipes.from_json(message_json_dict)
result = new_message.execute()
self.assertTrue(result)
# Make sure new recipes supersede the old ones
for recipe in Recipe.objects.filter(id__in=self.old_recipe_ids):
self.assertTrue(recipe.is_superseded)
new_recipe_1 = Recipe.objects.get(
superseded_recipe_id=self.recipe_1.id)
self.assertEqual(new_recipe_1.batch_id, batch.id)
self.assertEqual(new_recipe_1.event_id, event.id)
self.assertDictEqual(new_recipe_1.input, self.recipe_1.input)
new_recipe_2 = Recipe.objects.get(
superseded_recipe_id=self.recipe_2.id)
self.assertEqual(new_recipe_2.batch_id, batch.id)
self.assertEqual(new_recipe_2.event_id, event.id)
self.assertDictEqual(new_recipe_2.input, self.recipe_2.input)
# Make sure old jobs are superseded
for job in Job.objects.filter(id__in=self.old_job_ids):
self.assertTrue(job.is_superseded)
# Should be three messages, two for processing new recipe input and one for canceling superseded jobs
self.assertEqual(len(new_message.new_messages), 3)
found_process_recipe_input_1 = False
found_process_recipe_input_2 = False
found_cancel_jobs = False
for msg in new_message.new_messages:
if msg.type == 'process_recipe_input':
if found_process_recipe_input_1:
found_process_recipe_input_2 = True
found_process_recipe_input_1 = True
elif msg.type == 'cancel_jobs':
found_cancel_jobs = True
self.assertTrue(found_process_recipe_input_1)
self.assertTrue(found_process_recipe_input_2)
self.assertTrue(found_cancel_jobs)
def test_successful(self):
"""Tests calling RecipeManager.create_recipe() successfully."""
event = trigger_test_utils.create_trigger_event()
handler = Recipe.objects.create_recipe(recipe_type=self.recipe_type, event=event, data=RecipeData(self.data))
# Make sure the recipe jobs get created with the correct job types
recipe_job_1 = RecipeJob.objects.get(recipe_id=handler.recipe.id, job_name='Job 1')
recipe_job_2 = RecipeJob.objects.get(recipe_id=handler.recipe.id, job_name='Job 2')
self.assertEqual(recipe_job_1.job.job_type.id, self.job_type_1.id)
self.assertEqual(recipe_job_2.job.job_type.id, self.job_type_2.id)
# Make sure the recipe jobs get created in the correct order
self.assertLess(recipe_job_1.job_id, recipe_job_2.job_id)
def setUp(self):
django.setup()
self.count = 1
self.job_type = job_test_utils.create_seed_job_type()
self.job = job_test_utils.create_job(job_type=self.job_type)
self.job_exe = job_test_utils.create_job_exe(status='COMPLETED', job=self.job)
self.wp1 = storage_test_utils.create_workspace()
self.wp2 = storage_test_utils.create_workspace()
self.prod1 = storage_test_utils.create_file(file_type='PRODUCT', workspace=self.wp1, job_exe=self.job_exe)
self.prod2 = storage_test_utils.create_file(file_type='PRODUCT', workspace=self.wp1, job_exe=self.job_exe)
self.prod3 = storage_test_utils.create_file(file_type='PRODUCT', workspace=self.wp2, job_exe=self.job_exe)
self.file_1 = storage_test_utils.create_file(file_type='SOURCE')
self.event = trigger_test_utils.create_trigger_event()
PurgeResults.objects.create(source_file_id=self.file_1.id, trigger_event=self.event)
def test_execute_invalid_data(self):
"""Tests calling CreateJobs.execute() when the input data is invalid"""
manifest = {
'seedVersion': '1.0.0',
'job': {
'name': 'name',
'jobVersion': '1.0.0',
'packageVersion': '1.0.0',
'title': 'Title',
'description': 'This is a description',
'maintainer': {
'name': 'John Doe',
'email': '*****@*****.**'
},
'timeout': 10,
'interface': {
'command': 'the command',
'inputs': {
'files': [{
'name': 'input_a'
}]
}
}
}
}
job_type = job_test_utils.create_seed_job_type(manifest=manifest)
event = trigger_test_utils.create_trigger_event()
# Data does not provide required input_a so it is invalid
data_dict = convert_data_to_v6_json(Data()).get_dict()
# Create and execute message
message = create_jobs_message(job_type.name,
job_type.version,
job_type.revision_num,
event.id,
count=10,
input_data_dict=data_dict)
result = message.execute()
self.assertTrue(result)
self.assertEqual(
Job.objects.filter(job_type_id=job_type.id,
event_id=event.id).count(), 0)
# Should be no new messages
self.assertEqual(len(message.new_messages), 0)
def test_successful(self):
"""Tests calling RecipeManager.create_recipe() successfully."""
event = trigger_test_utils.create_trigger_event()
handler = Recipe.objects.create_recipe(recipe_type=self.recipe_type,
event=event,
data=self.data)
# Make sure the recipe jobs get created with the correct job types
recipe_job_1 = RecipeJob.objects.get(recipe_id=handler.recipe_id,
job_name='Job 1')
recipe_job_2 = RecipeJob.objects.get(recipe_id=handler.recipe_id,
job_name='Job 2')
self.assertEqual(recipe_job_1.job.job_type.id, self.job_type_1.id)
self.assertEqual(recipe_job_2.job.job_type.id, self.job_type_2.id)
# Make sure the recipe jobs get created in the correct order
self.assertLess(recipe_job_1.job_id, recipe_job_2.job_id)
def test_json_no_recipe(self):
"""Tests converting a CreateJobs message (without a recipe) to and from JSON"""
manifest = {
'seedVersion': '1.0.0',
'job': {
'name': 'name',
'jobVersion': '1.0.0',
'packageVersion': '1.0.0',
'title': 'Title',
'description': 'This is a description',
'maintainer': {
'name': 'John Doe',
'email': '*****@*****.**'
},
'timeout': 10
}
}
job_type = job_test_utils.create_seed_job_type(manifest=manifest)
event = trigger_test_utils.create_trigger_event()
data_dict = convert_data_to_v6_json(Data()).get_dict()
count = 10
# Create message
message = create_jobs_message(job_type.name,
job_type.version,
job_type.revision_num,
event.id,
count=count,
input_data_dict=data_dict)
# Convert message to JSON and back, and then execute
message_json_dict = message.to_json()
new_message = CreateJobs.from_json(message_json_dict)
result = new_message.execute()
self.assertTrue(result)
self.assertEqual(
Job.objects.filter(job_type_id=job_type.id,
event_id=event.id).count(), count)
# Check for process_job_input messages
self.assertEqual(len(new_message.new_messages), count)
for msg in new_message.new_messages:
self.assertEqual(msg.type, 'process_job_input')
def create_clock_event(rule=None, occurred=None):
'''Creates a scale clock trigger event model for unit testing
:returns: The trigger event model
:rtype: :class:`trigger.models.TriggerEvent`
'''
if not rule:
rule = create_clock_rule()
if not occurred:
occurred = timezone.now()
event_type = None
if rule.configuration and 'event_type' in rule.configuration:
event_type = rule.configuration['event_type']
return trigger_test_utils.create_trigger_event(trigger_type=event_type, rule=rule, occurred=occurred)
def create_clock_event(rule=None, occurred=None):
"""Creates a scale clock trigger event model for unit testing
:returns: The trigger event model
:rtype: :class:`trigger.models.TriggerEvent`
"""
if not rule:
rule = create_clock_rule()
if not occurred:
occurred = timezone.now()
event_type = None
if rule.configuration and 'event_type' in rule.configuration:
event_type = rule.configuration['event_type']
return trigger_test_utils.create_trigger_event(trigger_type=event_type, rule=rule, occurred=occurred)
def create_job(job_type=None,
event=None,
status='PENDING',
error=None,
data=None,
num_exes=0,
queued=None,
started=None,
ended=None,
last_status_change=None,
priority=100,
results=None):
"""Creates a job model for unit testing
:returns: The job model
:rtype: :class:`job.models.Job`
"""
if not job_type:
job_type = create_job_type()
if not event:
event = trigger_test_utils.create_trigger_event()
if not last_status_change:
last_status_change = timezone.now()
if not data:
data = {
'version': '1.0',
'input_data': [],
'output_data': [],
}
job = Job.objects.create_job(job_type, event)
job.priority = priority
job.data = data
job.status = status
job.num_exes = num_exes
job.queued = queued
job.started = started
job.ended = ended
job.last_status_change = last_status_change
job.error = error
job.results = results
job.save()
return job
def test_json(self):
"""Tests coverting a PurgeSourceFile message to and from JSON"""
# Create a file
source_file = storage_test_utils.create_file(file_type='SOURCE')
trigger = trigger_test_utils.create_trigger_event()
PurgeResults.objects.create(source_file_id=source_file.id,
trigger_event=trigger)
# Create message
message = create_purge_source_file_message(
source_file_id=source_file.id, trigger_id=trigger.id)
# Convert message to JSON and back, and then execute
message_json_dict = message.to_json()
new_message = PurgeSourceFile.from_json(message_json_dict)
result = new_message.execute()
self.assertTrue(result)
def test_execute_force_stop(self):
"""Tests calling SpawnDeleteFilesJob.execute with the force stop flag set"""
file_2 = storage_test_utils.create_file(file_type='SOURCE')
trigger = trigger_test_utils.create_trigger_event()
PurgeResults.objects.create(source_file_id=file_2.id, trigger_event=trigger, force_stop_purge=True)
job_type_id = JobType.objects.values_list('id', flat=True).get(name='scale-delete-files')
job_id = 1234574223462
# Make the message
message = create_spawn_delete_files_job(job_id=job_id, trigger_id=trigger.id,
source_file_id=self.file_1.id, purge=True)
# Capture message that creates job
result = message.execute()
self.assertTrue(result)
# Check that no message was created
self.assertEqual(len(message.new_messages), 0)
def test_successful_supersede_same_recipe_type(self):
"""Tests calling RecipeManager.create_recipe() to supersede a recipe with the same recipe type."""
event = trigger_test_utils.create_trigger_event()
handler = Recipe.objects.create_recipe(recipe_type=self.recipe_type, event=event, data=RecipeData(self.data))
recipe = Recipe.objects.get(id=handler.recipe.id)
recipe_job_1 = RecipeJob.objects.select_related('job').get(recipe_id=handler.recipe.id, job_name='Job 1')
recipe_job_2 = RecipeJob.objects.select_related('job').get(recipe_id=handler.recipe.id, job_name='Job 2')
superseded_jobs = {'Job 1': recipe_job_1.job, 'Job 2': recipe_job_2.job}
# Create a new recipe of the same type where we want to reprocess Job 2
graph = self.recipe_type.get_recipe_definition().get_graph()
delta = RecipeGraphDelta(graph, graph)
delta.reprocess_identical_node('Job 2') # We want to reprocess Job 2
new_handler = Recipe.objects.create_recipe(recipe_type=self.recipe_type, event=event, data=None,
superseded_recipe=recipe, delta=delta,
superseded_jobs=superseded_jobs)
# Check that old recipe and job 2 are superseded, job 1 should be copied (not superseded)
recipe = Recipe.objects.get(id=recipe.id)
job_1 = Job.objects.get(id=recipe_job_1.job_id)
job_2 = Job.objects.get(id=recipe_job_2.job_id)
self.assertTrue(recipe.is_superseded)
self.assertFalse(job_1.is_superseded)
self.assertTrue(job_2.is_superseded)
# Check that new recipe supersedes the old one, job 1 is copied from old recipe, and job 2 supersedes old job 2
new_recipe = Recipe.objects.get(id=new_handler.recipe.id)
new_recipe_job_1 = RecipeJob.objects.select_related('job').get(recipe_id=new_handler.recipe.id,
job_name='Job 1')
new_recipe_job_2 = RecipeJob.objects.select_related('job').get(recipe_id=new_handler.recipe.id,
job_name='Job 2')
self.assertEqual(new_recipe.superseded_recipe_id, recipe.id)
self.assertEqual(new_recipe.root_superseded_recipe_id, recipe.id)
self.assertDictEqual(new_recipe.data, recipe.data)
self.assertEqual(new_recipe_job_1.job.id, job_1.id)
self.assertFalse(new_recipe_job_1.is_original)
self.assertIsNone(new_recipe_job_1.job.superseded_job)
self.assertIsNone(new_recipe_job_1.job.root_superseded_job)
self.assertNotEqual(new_recipe_job_2.job.id, job_2.id)
self.assertTrue(new_recipe_job_2.is_original)
self.assertEqual(new_recipe_job_2.job.superseded_job_id, job_2.id)
self.assertEqual(new_recipe_job_2.job.root_superseded_job_id, job_2.id)
def create_recipe(recipe_type=None,
input=None,
event=None,
is_superseded=False,
superseded=None,
superseded_recipe=None,
batch=None,
save=True):
"""Creates a recipe for unit testing
:returns: The recipe model
:rtype: :class:`recipe.models.Recipe`
"""
if not recipe_type:
recipe_type = create_recipe_type()
if not input:
input = {}
if not event:
event = trigger_test_utils.create_trigger_event()
if is_superseded and not superseded:
superseded = timezone.now()
recipe = Recipe()
recipe.recipe_type = recipe_type
recipe.recipe_type_rev = RecipeTypeRevision.objects.get_revision_old(
recipe_type.id, recipe_type.revision_num)
recipe.event = event
recipe.input = input
recipe.is_superseded = is_superseded
recipe.superseded = superseded
recipe.batch = batch
if superseded_recipe:
root_id = superseded_recipe.root_superseded_recipe_id
if root_id is None:
root_id = superseded_recipe.id
recipe.root_superseded_recipe_id = root_id
recipe.superseded_recipe = superseded_recipe
if save:
recipe.save()
return recipe
def create_job(job_type=None, event=None, status='PENDING', error=None, data=None, num_exes=0, queued=None,
started=None, ended=None, last_status_change=None, priority=100, results=None, superseded_job=None,
delete_superseded=True, is_superseded=False, superseded=None):
"""Creates a job model for unit testing
:returns: The job model
:rtype: :class:`job.models.Job`
"""
if not job_type:
job_type = create_job_type()
if not event:
event = trigger_test_utils.create_trigger_event()
if not last_status_change:
last_status_change = timezone.now()
if not data:
data = {
'version': '1.0',
'input_data': [],
'output_data': [],
}
if superseded_job and not superseded_job.is_superseded:
Job.objects.supersede_jobs([superseded_job], timezone.now())
if is_superseded and not superseded:
superseded = timezone.now()
job = Job.objects.create_job(job_type, event, superseded_job=superseded_job, delete_superseded=delete_superseded)
job.priority = priority
job.data = data
job.status = status
job.num_exes = num_exes
job.queued = queued
job.started = started
job.ended = ended
job.last_status_change = last_status_change
job.error = error
job.results = results
job.is_superseded = is_superseded
job.superseded = superseded
job.save()
return job
def test_superseded_job(self):
"""Tests creating a job that supersedes another job"""
old_job = job_test_utils.create_job()
event = trigger_test_utils.create_trigger_event()
new_job = Job.objects.create_job(old_job.job_type, event, old_job, False)
new_job.save()
when = timezone.now()
Job.objects.supersede_jobs([old_job], when)
new_job = Job.objects.get(pk=new_job.id)
self.assertEqual(new_job.status, 'PENDING')
self.assertFalse(new_job.is_superseded)
self.assertEqual(new_job.root_superseded_job_id, old_job.id)
self.assertEqual(new_job.superseded_job_id, old_job.id)
self.assertFalse(new_job.delete_superseded)
self.assertIsNone(new_job.superseded)
old_job = Job.objects.get(pk=old_job.id)
self.assertTrue(old_job.is_superseded)
self.assertEqual(old_job.superseded, when)
def create_recipe_handler(recipe_type=None, data=None, event=None, superseded_recipe=None, delta=None,
superseded_jobs=None):
"""Creates a recipe along with its declared jobs for unit testing
:returns: The recipe handler with created recipe and jobs
:rtype: :class:`recipe.handlers.handler.RecipeHandler`
"""
if not recipe_type:
recipe_type = create_recipe_type()
if not data:
data = {}
if not isinstance(data, RecipeData):
data = RecipeData(data)
if not event:
event = trigger_test_utils.create_trigger_event()
if superseded_recipe and not delta:
delta = RecipeGraphDelta(RecipeGraph(), RecipeGraph())
return Recipe.objects.create_recipe(recipe_type, data, event, superseded_recipe=superseded_recipe,
delta=delta, superseded_jobs=superseded_jobs)
def setUp(self):
django.setup()
self.workspace = storage_test_utils.create_workspace()
self.file = storage_test_utils.create_file()
configuration = {
'version': '1.0',
'condition': {
'media_type': 'text/plain',
},
'data': {
'input_data_name': 'Recipe Input',
'workspace_name': self.workspace.name,
},
}
self.rule = trigger_test_utils.create_trigger_rule(configuration=configuration)
self.event = trigger_test_utils.create_trigger_event(rule=self.rule)
interface_1 = {
'version': '1.0',
'command': 'my_command',
'command_arguments': 'args',
'input_data': [{
'name': 'Test Input 1',
'type': 'file',
'media_types': ['text/plain'],
}],
'output_data': [{
'name': 'Test Output 1',
'type': 'files',
'media_type': 'image/png',
}],
}
self.job_type_1 = job_test_utils.create_job_type(interface=interface_1)
self.definition_1 = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input',
'type': 'file',
'media_types': ['text/plain'],
}],
'jobs': [{
'name': 'Job 1',
'job_type': {
'name': self.job_type_1.name,
'version': self.job_type_1.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input',
'job_input': 'Test Input 1',
}],
}],
}
RecipeDefinition(self.definition_1).validate_job_interfaces()
self.recipe_type = recipe_test_utils.create_recipe_type(definition=self.definition_1, trigger_rule=self.rule)
self.interface_2 = {
'version': '1.0',
'command': 'my_command',
'command_arguments': 'args',
'input_data': [{
'name': 'Test Input 2',
'type': 'files',
'media_types': ['image/tiff'],
}],
'output_data': [{
'name': 'Test Output 2',
'type': 'file',
}],
}
self.job_type_2 = job_test_utils.create_job_type(interface=self.interface_2)
self.definition_2 = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input',
'type': 'file',
'media_types': ['text/plain'],
}],
'jobs': [{
'name': 'Job 1',
'job_type': {
'name': self.job_type_1.name,
'version': self.job_type_1.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input',
'job_input': 'Test Input 1',
}]
}, {
'name': 'Job 2',
'job_type': {
'name': self.job_type_2.name,
'version': self.job_type_2.version,
},
'dependencies': [{
'name': 'Job 1',
'connections': [{
'output': 'Test Output 1',
'input': 'Test Input 2',
}],
}],
}],
}
self.data = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input',
'file_id': self.file.id,
}],
'workspace_id': self.workspace.id,
}
def test_successful_supersede_different_recipe_type(self):
"""Tests calling RecipeManager.create_recipe() to supersede a recipe with a different recipe type version that
has one identical node, and deletes another node to replace it with a new one.
"""
interface_3 = {
'version': '1.0',
'command': 'my_command',
'command_arguments': 'args',
'input_data': [{
'name': 'Test Input 3',
'type': 'files',
'media_types': ['image/tiff'],
}],
'output_data': [{
'name': 'Test Output 3',
'type': 'file',
}]}
job_type_3 = job_test_utils.create_job_type(interface=interface_3)
new_definition = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input',
'type': 'file',
'media_types': ['text/plain'],
}],
'jobs': [{
'name': 'Job 1',
'job_type': {
'name': self.job_type_1.name,
'version': self.job_type_1.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input',
'job_input': 'Test Input 1',
}]
}, {
'name': 'Job 3',
'job_type': {
'name': job_type_3.name,
'version': job_type_3.version,
},
'dependencies': [{
'name': 'Job 1',
'connections': [{
'output': 'Test Output 1',
'input': 'Test Input 3',
}]
}]
}]
}
new_recipe_type = recipe_test_utils.create_recipe_type(name=self.recipe_type.name, definition=new_definition)
event = trigger_test_utils.create_trigger_event()
handler = Recipe.objects.create_recipe(recipe_type=self.recipe_type, event=event, data=RecipeData(self.data))
recipe = Recipe.objects.get(id=handler.recipe.id)
recipe_job_1 = RecipeJob.objects.select_related('job').get(recipe_id=handler.recipe.id, job_name='Job 1')
recipe_job_2 = RecipeJob.objects.select_related('job').get(recipe_id=handler.recipe.id, job_name='Job 2')
job_exe_2 = job_test_utils.create_job_exe(job=recipe_job_2.job)
try:
from product.models import ProductFile
from product.test import utils as product_test_utils
product = product_test_utils.create_product(job_exe=job_exe_2, has_been_published=True, is_published=True)
except ImportError:
product = None
superseded_jobs = {'Job 1': recipe_job_1.job, 'Job 2': recipe_job_2.job}
# Create a new recipe with a different version
graph_a = self.recipe_type.get_recipe_definition().get_graph()
graph_b = new_recipe_type.get_recipe_definition().get_graph()
delta = RecipeGraphDelta(graph_a, graph_b)
new_handler = Recipe.objects.create_recipe(recipe_type=new_recipe_type, event=event, data=None,
superseded_recipe=recipe, delta=delta,
superseded_jobs=superseded_jobs)
# Check that old recipe and job 2 are superseded, job 1 should be copied (not superseded)
recipe = Recipe.objects.get(id=recipe.id)
job_1 = Job.objects.get(id=recipe_job_1.job_id)
job_2 = Job.objects.get(id=recipe_job_2.job_id)
self.assertTrue(recipe.is_superseded)
self.assertFalse(job_1.is_superseded)
self.assertTrue(job_2.is_superseded)
# Check that product of job 2 (which was superseded with no new job) was unpublished
if product:
product = ProductFile.objects.get(id=product.id)
self.assertFalse(product.is_published)
self.assertIsNotNone(product.unpublished)
# Check that new recipe supersedes the old one, job 1 is copied from old recipe, and job 2 is new and does not
# supersede anything
new_recipe = Recipe.objects.get(id=new_handler.recipe.id)
new_recipe_job_1 = RecipeJob.objects.select_related('job').get(recipe_id=new_handler.recipe.id,
job_name='Job 1')
new_recipe_job_2 = RecipeJob.objects.select_related('job').get(recipe_id=new_handler.recipe.id,
job_name='Job 3')
self.assertEqual(new_recipe.superseded_recipe_id, recipe.id)
self.assertEqual(new_recipe.root_superseded_recipe_id, recipe.id)
self.assertDictEqual(new_recipe.data, recipe.data)
self.assertEqual(new_recipe_job_1.job.id, job_1.id)
self.assertFalse(new_recipe_job_1.is_original)
self.assertIsNone(new_recipe_job_1.job.superseded_job)
self.assertIsNone(new_recipe_job_1.job.root_superseded_job)
self.assertNotEqual(new_recipe_job_2.job.id, job_2.id)
self.assertTrue(new_recipe_job_2.is_original)
self.assertIsNone(new_recipe_job_2.job.superseded_job_id)
self.assertIsNone(new_recipe_job_2.job.root_superseded_job_id)
def setUp(self):
django.setup()
workspace = storage_test_utils.create_workspace()
source_file = source_test_utils.create_source(workspace=workspace)
self.event = trigger_test_utils.create_trigger_event()
interface_1 = {
'version': '1.0',
'command': 'test_command',
'command_arguments': 'test_arg',
'input_data': [{
'name': 'Test Input 1',
'type': 'file',
'media_types': ['text/plain'],
}],
'output_data': [{
'name': 'Test Output 1',
'type': 'files',
'media_type': 'image/png',
}]
}
self.job_type_1 = job_test_utils.create_job_type(interface=interface_1)
interface_2 = {
'version': '1.0',
'command': 'test_command',
'command_arguments': 'test_arg',
'input_data': [{
'name': 'Test Input 2',
'type': 'files',
'media_types': ['image/png', 'image/tiff'],
}],
'output_data': [{
'name': 'Test Output 2',
'type': 'file',
}]
}
self.job_type_2 = job_test_utils.create_job_type(interface=interface_2)
definition = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input',
'type': 'file',
'media_types': ['text/plain'],
}],
'jobs': [{
'name': 'Job 1',
'job_type': {
'name': self.job_type_1.name,
'version': self.job_type_1.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input',
'job_input': 'Test Input 1',
}]
}, {
'name': 'Job 2',
'job_type': {
'name': self.job_type_2.name,
'version': self.job_type_2.version,
},
'dependencies': [{
'name': 'Job 1',
'connections': [{
'output': 'Test Output 1',
'input': 'Test Input 2',
}]
}]
}]
}
recipe_definition = RecipeDefinition(definition)
recipe_definition.validate_job_interfaces()
self.recipe_type = recipe_test_utils.create_recipe_type(definition=definition)
data = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input',
'file_id': source_file.id,
}],
'workspace_id': workspace.id,
}
self.data = RecipeData(data)
# Register a fake processor
self.mock_processor = MagicMock(QueueEventProcessor)
Queue.objects.register_processor(lambda: self.mock_processor)
def test_successful_supersede(self):
"""Tests calling QueueManager.queue_new_recipe() successfully when superseding a recipe."""
# Queue initial recipe and complete its first job
node = node_test_utils.create_node()
recipe_id = Queue.objects.queue_new_recipe(self.recipe_type, self.data, self.event)
recipe = Recipe.objects.get(id=recipe_id)
recipe_job_1 = RecipeJob.objects.select_related('job__job_exe').get(recipe_id=recipe_id, job_name='Job 1')
job_exe_1 = JobExecution.objects.get(job_id=recipe_job_1.job_id)
queued_job_exe = QueuedJobExecution(Queue.objects.get(job_exe_id=job_exe_1.id))
queued_job_exe.accepted(node, JobResources(cpus=10, mem=1000, disk_in=1000, disk_out=1000, disk_total=2000))
Queue.objects.schedule_job_executions('123', [queued_job_exe], {})
results = JobResults()
results.add_file_list_parameter('Test Output 1', [product_test_utils.create_product().file_id])
JobExecution.objects.filter(id=job_exe_1.id).update(results=results.get_dict())
Queue.objects.handle_job_completion(job_exe_1.id, now())
# Create a new recipe type that has a new version of job 2 (job 1 is identical)
new_job_type_2 = job_test_utils.create_job_type(name=self.job_type_2.name, version='New Version',
interface=self.job_type_2.interface)
new_definition = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input',
'type': 'file',
'media_types': ['text/plain'],
}],
'jobs': [{
'name': 'New Job 1',
'job_type': {
'name': self.job_type_1.name,
'version': self.job_type_1.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input',
'job_input': 'Test Input 1',
}]
}, {
'name': 'New Job 2',
'job_type': {
'name': new_job_type_2.name,
'version': new_job_type_2.version,
},
'dependencies': [{
'name': 'New Job 1',
'connections': [{
'output': 'Test Output 1',
'input': 'Test Input 2',
}]
}]
}]
}
new_recipe_type = recipe_test_utils.create_recipe_type(name=self.recipe_type.name, definition=new_definition)
event = trigger_test_utils.create_trigger_event()
recipe_job_1 = RecipeJob.objects.select_related('job').get(recipe_id=recipe_id, job_name='Job 1')
recipe_job_2 = RecipeJob.objects.select_related('job').get(recipe_id=recipe_id, job_name='Job 2')
superseded_jobs = {'Job 1': recipe_job_1.job, 'Job 2': recipe_job_2.job}
graph_a = self.recipe_type.get_recipe_definition().get_graph()
graph_b = new_recipe_type.get_recipe_definition().get_graph()
delta = RecipeGraphDelta(graph_a, graph_b)
# Queue new recipe that supersedes the old recipe
new_recipe_id = Queue.objects.queue_new_recipe(new_recipe_type, None, event, recipe, delta, superseded_jobs)
# Ensure old recipe is superseded
recipe = Recipe.objects.get(id=recipe_id)
self.assertTrue(recipe.is_superseded)
# Ensure new recipe supersedes old recipe
new_recipe = Recipe.objects.get(id=new_recipe_id)
self.assertEqual(new_recipe.superseded_recipe_id, recipe_id)
# Ensure that job 1 is already completed (it was copied from original recipe) and that job 2 is queued
new_recipe_job_1 = RecipeJob.objects.select_related('job').get(recipe_id=new_recipe_id, job_name='New Job 1')
new_recipe_job_2 = RecipeJob.objects.select_related('job').get(recipe_id=new_recipe_id, job_name='New Job 2')
self.assertEqual(new_recipe_job_1.job.status, 'COMPLETED')
self.assertFalse(new_recipe_job_1.is_original)
self.assertEqual(new_recipe_job_2.job.status, 'QUEUED')
self.assertTrue(new_recipe_job_2.is_original)
# Complete both the old and new job 2 and check that only the new recipe completes
job_exe_2 = JobExecution.objects.get(job_id=recipe_job_2.job_id)
queued_job_exe_2 = QueuedJobExecution(Queue.objects.get(job_exe_id=job_exe_2.id))
queued_job_exe_2.accepted(node, JobResources(cpus=10, mem=1000, disk_in=1000, disk_out=1000, disk_total=2000))
Queue.objects.schedule_job_executions('123', [queued_job_exe_2], {})
Queue.objects.handle_job_completion(job_exe_2.id, now())
new_job_exe_2 = JobExecution.objects.get(job_id=new_recipe_job_2.job_id)
new_queued_job_exe_2 = QueuedJobExecution(Queue.objects.get(job_exe_id=new_job_exe_2.id))
new_queued_job_exe_2.accepted(node, JobResources(cpus=10, mem=1000, disk_in=1000, disk_out=1000,
disk_total=2000))
Queue.objects.schedule_job_executions('123', [new_queued_job_exe_2], {})
Queue.objects.handle_job_completion(new_job_exe_2.id, now())
recipe = Recipe.objects.get(id=recipe.id)
new_recipe = Recipe.objects.get(id=new_recipe.id)
self.assertIsNone(recipe.completed)
self.assertIsNotNone(new_recipe.completed)
