def handle(self, *args, **options):
if options["job_id"] is None:
logger.error("You must specify a job ID.",
job_id=options["job_id"])
sys.exit(1)
try:
job_type = ProcessorPipeline[options["job_name"]]
except KeyError:
logger.error("You must specify a valid job name.",
job_name=options["job_name"],
job_id=options["job_id"])
sys.exit(1)
if job_type is ProcessorPipeline.AFFY_TO_PCL:
from data_refinery_workers.processors.array_express import affy_to_pcl
affy_to_pcl(options["job_id"])
elif job_type is ProcessorPipeline.TRANSCRIPTOME_INDEX_SHORT:
from data_refinery_workers.processors.transcriptome_index import build_transcriptome_index
build_transcriptome_index(options["job_id"], length="short")
elif job_type is ProcessorPipeline.TRANSCRIPTOME_INDEX_LONG:
from data_refinery_workers.processors.transcriptome_index import build_transcriptome_index
build_transcriptome_index(options["job_id"], length="long")
elif job_type is ProcessorPipeline.AGILENT_TWOCOLOR_TO_PCL:
from data_refinery_workers.processors.agilent_twocolor import agilent_twocolor_to_pcl
agilent_twocolor_to_pcl(options["job_id"])
elif job_type is ProcessorPipeline.ILLUMINA_TO_PCL:
from data_refinery_workers.processors.illumina import illumina_to_pcl
illumina_to_pcl(options["job_id"])
elif job_type is ProcessorPipeline.SALMON:
from data_refinery_workers.processors.salmon import salmon
salmon(options["job_id"])
elif job_type is ProcessorPipeline.SMASHER:
from data_refinery_workers.processors.smasher import smash
smash(options["job_id"])
elif job_type is ProcessorPipeline.NO_OP:
from data_refinery_workers.processors.no_op import no_op_processor
no_op_processor(options["job_id"])
elif job_type is ProcessorPipeline.JANITOR:
from data_refinery_workers.processors.janitor import run_janitor
run_janitor(options["job_id"])
elif job_type is ProcessorPipeline.QN_REFERENCE:
from data_refinery_workers.processors import qn_reference
qn_reference.create_qn_reference(options["job_id"])
else:
logger.error(
("A valid job name was specified for job %s with id %d but "
"no processor function is known to run it."),
options["job_name"], options["job_id"])
sys.exit(1)
sys.exit(0)
def test_janitor(self, mock_nomad):
""" Main tester. """
def mock_get_job(job_id: str):
if job_id == "running_job":
return {"Status": "running"}
else:
return {"Status": "dead"}
def mock_init_nomad(host, port=0, timeout=0):
ret_value = MagicMock()
ret_value.job = MagicMock()
ret_value.job.get_job = MagicMock()
ret_value.job.get_job.side_effect = mock_get_job
return ret_value
mock_nomad.side_effect = mock_init_nomad
job = prepare_job()
final_context = janitor.run_janitor(job.pk)
for i in range(0, JOBS):
# The job with id 1 should appear running.
if i == 1:
self.assertTrue(os.path.exists(LOCAL_ROOT_DIR + '/processor_job_' + str(i)))
else:
self.assertFalse(os.path.exists(LOCAL_ROOT_DIR + '/processor_job_' + str(i)))
self.assertFalse(os.path.exists(LOCAL_ROOT_DIR + '/SRP' + str(i) + '/SRR' + str(i)))
self.assertTrue(os.path.exists(LOCAL_ROOT_DIR + '/processor_job_11_index'))
self.assertTrue(os.path.exists(LOCAL_ROOT_DIR + '/SRP' + str(JOBS+1) + '/SRR' + str(JOBS+1)))
# Deleted all the working directories except for the one that's still running.
self.assertEqual(len(final_context['deleted_items']), (JOBS*2)-1)
def handle(self, *args, **options):
pj = ProcessorJob()
pj.pipeline_applied = "JANITOR"
pj.save()
final_context = run_janitor(pj.pk)
print("Removed: ")
for item in final_context['deleted_items']:
print('\t - ' + item)
sys.exit(0)
def test_janitor(self, mock_describe_jobs):
""" Main tester. """
job = prepare_job()
mock_describe_jobs.return_value = {
"jobs": [{
"jobId": "running_job",
"status": "RUNNING"
}]
}
final_context = janitor.run_janitor(job.pk)
for i in range(JOBS):
# The job with id 1 should appear running.
if i == 1:
self.assertTrue(
os.path.exists(LOCAL_ROOT_DIR + "/processor_job_" +
str(i)))
else:
self.assertFalse(
os.path.exists(LOCAL_ROOT_DIR + "/processor_job_" +
str(i)))
self.assertFalse(
os.path.exists(LOCAL_ROOT_DIR + "/SRP" + str(i) + "/SRR" +
str(i)))
self.assertTrue(
os.path.exists(LOCAL_ROOT_DIR + "/processor_job_11_index"))
self.assertTrue(
os.path.exists(LOCAL_ROOT_DIR + "/SRP" + str(JOBS + 1) + "/SRR" +
str(JOBS + 1)))
# Deleted all the working directories except for the one that's still running.
self.assertEqual(len(final_context["deleted_items"]), (JOBS * 2) - 1)