def handle(self, *args, **options): if options["job_id"] is None: logger.error("You must specify a job ID.", job_id=options["job_id"]) sys.exit(1) try: job_type = ProcessorPipeline[options["job_name"]] except KeyError: logger.error("You must specify a valid job name.", job_name=options["job_name"], job_id=options["job_id"]) sys.exit(1) if job_type is ProcessorPipeline.AFFY_TO_PCL: from data_refinery_workers.processors.array_express import affy_to_pcl affy_to_pcl(options["job_id"]) elif job_type is ProcessorPipeline.TRANSCRIPTOME_INDEX_SHORT: from data_refinery_workers.processors.transcriptome_index import build_transcriptome_index build_transcriptome_index(options["job_id"], length="short") elif job_type is ProcessorPipeline.TRANSCRIPTOME_INDEX_LONG: from data_refinery_workers.processors.transcriptome_index import build_transcriptome_index build_transcriptome_index(options["job_id"], length="long") elif job_type is ProcessorPipeline.AGILENT_TWOCOLOR_TO_PCL: from data_refinery_workers.processors.agilent_twocolor import agilent_twocolor_to_pcl agilent_twocolor_to_pcl(options["job_id"]) elif job_type is ProcessorPipeline.ILLUMINA_TO_PCL: from data_refinery_workers.processors.illumina import illumina_to_pcl illumina_to_pcl(options["job_id"]) elif job_type is ProcessorPipeline.SALMON: from data_refinery_workers.processors.salmon import salmon salmon(options["job_id"]) elif job_type is ProcessorPipeline.SMASHER: from data_refinery_workers.processors.smasher import smash smash(options["job_id"]) elif job_type is ProcessorPipeline.NO_OP: from data_refinery_workers.processors.no_op import no_op_processor no_op_processor(options["job_id"]) elif job_type is ProcessorPipeline.JANITOR: from data_refinery_workers.processors.janitor import run_janitor run_janitor(options["job_id"]) elif job_type is ProcessorPipeline.QN_REFERENCE: from data_refinery_workers.processors import qn_reference qn_reference.create_qn_reference(options["job_id"]) else: logger.error( ("A valid job name was specified for job %s with id %d but " "no processor function is known to run it."), options["job_name"], options["job_id"]) sys.exit(1) sys.exit(0)
def test_janitor(self, mock_nomad): """ Main tester. """ def mock_get_job(job_id: str): if job_id == "running_job": return {"Status": "running"} else: return {"Status": "dead"} def mock_init_nomad(host, port=0, timeout=0): ret_value = MagicMock() ret_value.job = MagicMock() ret_value.job.get_job = MagicMock() ret_value.job.get_job.side_effect = mock_get_job return ret_value mock_nomad.side_effect = mock_init_nomad job = prepare_job() final_context = janitor.run_janitor(job.pk) for i in range(0, JOBS): # The job with id 1 should appear running. if i == 1: self.assertTrue(os.path.exists(LOCAL_ROOT_DIR + '/processor_job_' + str(i))) else: self.assertFalse(os.path.exists(LOCAL_ROOT_DIR + '/processor_job_' + str(i))) self.assertFalse(os.path.exists(LOCAL_ROOT_DIR + '/SRP' + str(i) + '/SRR' + str(i))) self.assertTrue(os.path.exists(LOCAL_ROOT_DIR + '/processor_job_11_index')) self.assertTrue(os.path.exists(LOCAL_ROOT_DIR + '/SRP' + str(JOBS+1) + '/SRR' + str(JOBS+1))) # Deleted all the working directories except for the one that's still running. self.assertEqual(len(final_context['deleted_items']), (JOBS*2)-1)
def handle(self, *args, **options): pj = ProcessorJob() pj.pipeline_applied = "JANITOR" pj.save() final_context = run_janitor(pj.pk) print("Removed: ") for item in final_context['deleted_items']: print('\t - ' + item) sys.exit(0)
def test_janitor(self, mock_describe_jobs): """ Main tester. """ job = prepare_job() mock_describe_jobs.return_value = { "jobs": [{ "jobId": "running_job", "status": "RUNNING" }] } final_context = janitor.run_janitor(job.pk) for i in range(JOBS): # The job with id 1 should appear running. if i == 1: self.assertTrue( os.path.exists(LOCAL_ROOT_DIR + "/processor_job_" + str(i))) else: self.assertFalse( os.path.exists(LOCAL_ROOT_DIR + "/processor_job_" + str(i))) self.assertFalse( os.path.exists(LOCAL_ROOT_DIR + "/SRP" + str(i) + "/SRR" + str(i))) self.assertTrue( os.path.exists(LOCAL_ROOT_DIR + "/processor_job_11_index")) self.assertTrue( os.path.exists(LOCAL_ROOT_DIR + "/SRP" + str(JOBS + 1) + "/SRR" + str(JOBS + 1))) # Deleted all the working directories except for the one that's still running. self.assertEqual(len(final_context["deleted_items"]), (JOBS * 2) - 1)