def test_run_fails_cause_genchallenge_is_not_found(self): temp_dir = tempfile.mkdtemp() try: params = D3RParameters() params.genchallenge = '/bin/doesnotexist' params.pdbdb = '/foo' params.version = '1' blastnfilter = BlastNFilterTask(temp_dir, params) blastnfilter.create_dir() open(os.path.join(blastnfilter.get_dir(), D3RTask.COMPLETE_FILE), 'a').close() chall = ChallengeDataTask(temp_dir, params) chall.run() self.assertEqual(chall.get_error(), 'Caught Exception trying to run ' + '/bin/doesnotexist --candidatedir ' + blastnfilter.get_dir() + ' --pdbdb ' + '/foo --outdir ' + chall.get_dir() + '/' + chall.get_celpp_challenge_data_dir_name() + ' : [Errno 2] No such file or directory') # test files get created errfile = os.path.join(chall.get_dir(), D3RTask.ERROR_FILE) self.assertEqual(os.path.isfile(errfile), True) finally: shutil.rmtree(temp_dir)
def test_run_fails_cause_chimeraprep_is_not_found_rdkitpython_set(self): temp_dir = tempfile.mkdtemp() try: params = D3RParameters() params.chimeraprep = '/bin/doesnotexist' params.pdbdb = '/foo' params.rdkitpython = '/data/miniconda2/bin' chall = ChallengeDataTask(temp_dir, params) chall.create_dir() challdir = os.path.join(chall.get_dir(), chall.get_celpp_challenge_data_dir_name()) open(os.path.join(chall.get_dir(), D3RTask.COMPLETE_FILE), 'a').close() task = ChimeraProteinLigPrepTask(temp_dir, params) task.run() self.assertEqual( task.get_error(), 'Caught Exception trying to run ' + '/bin/doesnotexist --candidatedir ' + challdir + ' --rdkitpython \'/data/miniconda2/bin\' ' + '--pdbdb ' + '/foo --outdir ' + task.get_dir() + ' : [Errno 2] No such file or directory') # test files get created errfile = os.path.join(task.get_dir(), D3RTask.ERROR_FILE) self.assertEqual(os.path.isfile(errfile), True) finally: shutil.rmtree(temp_dir)
def run(self): """Runs ChimeraProteinLigPrepTask after verifying BlastNFilterTask was good Method requires can_run() to be called before hand with successful outcome Otherwise method invokes D3RTask.start then this method creates a directory and invokes blastnfilter script. Upon completion results are analyzed and success or error status is set appropriately and D3RTask.end is invoked """ super(ChimeraProteinLigPrepTask, self).run() if self._can_run is False: logger.debug(self.get_dir_name() + ' cannot run cause _can_run flag ' 'is False') return try: logger.debug('chimeraprep set to ' + self.get_args().chimeraprep) except AttributeError: self.set_error('chimeraprep not set') self.end() return try: logger.debug('pdbdb set to ' + self.get_args().pdbdb) except AttributeError: self.set_error('pdbdb not set') self.end() return rdkitpython = '' try: logger.debug('rdkitpython set to ' + self.get_args().rdkitpython) rdkitpython = self.get_args().rdkitpython except AttributeError: logger.debug chall = ChallengeDataTask(self._path, self._args) # # chimeraprep.py --candidatedir <path to stage.3.challengedata> \ # --outdir <path to stage.4.chimeraligprep> challdir = os.path.join(chall.get_dir(), chall.get_celpp_challenge_data_dir_name()) cmd_to_run = (self.get_args().chimeraprep + ' --candidatedir ' + challdir + ' --rdkitpython \'' + rdkitpython + '\'' + ' --pdbdb ' + self.get_args().pdbdb + ' --outdir ' + self.get_dir()) chimeraprep_name = os.path.basename(self.get_args().chimeraprep) self.run_external_command(chimeraprep_name, cmd_to_run, True) # assess the result self.end()
def test_get_celpp_challenge_data_dir_name_path_not_found(self): temp_dir = tempfile.mkdtemp() try: params = D3RParameters() task = ChallengeDataTask(temp_dir, params) self.assertEqual(task.get_celpp_challenge_data_dir_name(), 'celpp_week0_0') finally: shutil.rmtree(temp_dir)
def __init__(self, path, args): """Constructor """ try: logger.debug('ftpconfig set to ' + args.ftpconfig) self._file_transfer = FtpFileTransfer(args.ftpconfig) except Exception: logger.exception('Caught exception') self._file_transfer = None self._path = path self._args = args ctask = ChallengeDataTask(path, args) self._chall_dir_name = ctask.get_celpp_challenge_data_dir_name()
def test_get_celpp_challenge_data_dir_name(self): temp_dir = tempfile.mkdtemp() try: params = D3RParameters() cyear = os.path.join(temp_dir, '2016') cweek = os.path.join(cyear, 'dataset.week.5') os.mkdir(cyear) os.mkdir(cweek) task = ChallengeDataTask(cweek, params) task.create_dir() self.assertEqual(task.get_celpp_challenge_data_dir_name(), 'celpp_week5_2016') finally: shutil.rmtree(temp_dir)
def test_tar_challenge_dir_unable_to_write_tarfile(self): temp_dir = tempfile.mkdtemp() try: params = D3RParameters() task = ChallengeDataTask(temp_dir, params) chall_dir = task.get_celpp_challenge_data_dir_name() try: task._tar_challenge_dir(chall_dir) self.fail('Expected IOError') except IOError: pass finally: shutil.rmtree(temp_dir)
def test_tar_challenge_dir(self): temp_dir = tempfile.mkdtemp() try: params = D3RParameters() dimport = DataImportTask(temp_dir, params) dimport.create_dir() open(dimport.get_sequence_tsv(), 'a').close() open(dimport.get_nonpolymer_tsv(), 'a').close() open(dimport.get_crystalph_tsv(), 'a').close() task = ChallengeDataTask(temp_dir, params) task.create_dir() chall_dir = task._create_challenge_dir() final_log = os.path.join(chall_dir, 'final.log') open(final_log, 'a').close() task._create_readme(chall_dir) task._copy_over_tsv_files(chall_dir) # make a fake candidate file_list = self.make_fake_candidate_dir(chall_dir, '5hib', '2eb2', 'CSX') self.assertEqual(os.path.isdir(chall_dir), True) name = task.get_celpp_challenge_data_dir_name() tfile = task._tar_challenge_dir(name) self.assertEqual(os.path.isfile(tfile), True) foodir = os.path.join(temp_dir, 'foo') tar = tarfile.open(tfile, 'r:*') tar.extractall(path=foodir) tar.close() cdir = os.path.join(foodir, name) readme = os.path.join(cdir, ChallengeDataTask.README_TXT_FILE) self.assertEqual(os.path.isfile(readme), True) final_log = os.path.join(foodir, 'final.log') self.assertEqual(os.path.isfile(final_log), False) for fname in file_list: chk = os.path.join(cdir, fname) self.assertEqual(os.path.isfile(chk), True, chk) finally: shutil.rmtree(temp_dir)
def test_run_succeeds_one_evaluation_and_no_emailer(self): temp_dir = tempfile.mkdtemp() try: params = D3RParameters() params.postevaluation = 'echo' etask = EvaluationTask( temp_dir, 'foo_dock.' + EvaluationTaskFactory.SCORING_SUFFIX, None, params) etask.create_dir() task = PostEvaluationTask(temp_dir, params) ctask = ChallengeDataTask(temp_dir, params) task.run() self.assertEqual(task.get_error(), None) self.assertEqual(task.get_status(), D3RTask.COMPLETE_STATUS) cfile = os.path.join(task.get_dir(), D3RTask.COMPLETE_FILE) self.assertTrue(os.path.isfile(cfile)) stdout_file = os.path.join(task.get_dir(), 'echo' + D3RTask.STDOUT_SUFFIX) self.assertTrue(os.path.isfile(stdout_file)) f = open(stdout_file, 'r') data = f.read() f.close() ctaskdir = os.path.join(ctask.get_dir(), ctask.get_celpp_challenge_data_dir_name()) self.assertTrue(' --evaluationdir ' + etask.get_dir() in data) self.assertTrue(' --challengedir ' + ctaskdir in data) self.assertTrue('--stageprefix stage.7. --evaluationsuffix ' + '.extsubmission.evaluation$|.evaluation$' in data) stderr_file = os.path.join(task.get_dir(), 'echo' + D3RTask.STDERR_SUFFIX) self.assertTrue(os.path.isfile(stderr_file)) finally: shutil.rmtree(temp_dir)
def test_create_challenge_dir_unable_to_make_dir(self): temp_dir = tempfile.mkdtemp() try: params = D3RParameters() cyear = os.path.join(temp_dir, '2016') cweek = os.path.join(cyear, 'dataset.week.5') os.mkdir(cyear) os.mkdir(cweek) task = ChallengeDataTask(cweek, params) task.create_dir() cdir = os.path.join(task.get_dir(), task.get_celpp_challenge_data_dir_name()) open(cdir, 'a').close() try: task._create_challenge_dir() self.fail('Expected OSError') except OSError: pass finally: shutil.rmtree(temp_dir)
def test_tar_challenge_dir_nothing_but_readme(self): temp_dir = tempfile.mkdtemp() try: params = D3RParameters() task = ChallengeDataTask(temp_dir, params) task.create_dir() chall_dir = task._create_challenge_dir() task._create_readme(chall_dir) self.assertEqual(os.path.isdir(chall_dir), True) name = task.get_celpp_challenge_data_dir_name() tfile = task._tar_challenge_dir(name) self.assertEqual(os.path.isfile(tfile), True) foodir = os.path.join(temp_dir, 'foo') tar = tarfile.open(tfile, 'r:*') tar.extractall(path=foodir) tar.close() cdir = os.path.join(foodir, name) readme = os.path.join(cdir, ChallengeDataTask.README_TXT_FILE) self.assertEqual(os.path.isfile(readme), True) finally: shutil.rmtree(temp_dir)
def run(self): """Runs EvaluationTask after verifying dock was good Method requires can_run() to be called before hand with successful outcome Otherwise method invokes D3RTask.start then this method creates a directory and invokes evaluation script. Upon completion results are analyzed and success or error status is set appropriately and D3RTask.end is invoked """ super(EvaluationTask, self).run() if self._can_run is False: logger.debug(self.get_dir_name() + ' cannot run cause _can_run flag ' 'is False') return try: logger.debug('evaluation set to ' + self.get_args().evaluation) except AttributeError: self.set_error('evaluation not set') self.end() return try: logger.debug('pdbdb set to ' + self.get_args().pdbdb) except AttributeError: self.set_error('pdbdb not set') self.end() return try: evaltimeout = self.get_args().evaluationtimeout logger.debug('Setting evaluation timeout to ' + str(evaltimeout)) except AttributeError: evaltimeout = None try: killdelay = self.get_args().evaluationtimeoutkilldelay logger.debug('Setting evaluation kill delay to ' + str(killdelay)) except AttributeError: killdelay = 60 blastnfilter = BlastNFilterTask(self._path, self._args) challenge = ChallengeDataTask(self._path, self._args) challdir = os.path.join(challenge.get_dir(), challenge.get_celpp_challenge_data_dir_name()) # # --pdbdb <path to pdb.extracted> --dockdir <stage.4.glide> \ # --outdir <path to stage.5.glide.evaluation> # cmd_to_run = (self.get_args().evaluation + ' --pdbdb ' + self.get_args().pdbdb + ' --dockdir ' + self._docktask.get_dir() + ' --blastnfilterdir ' + blastnfilter.get_dir() + ' --challengedir ' + challdir + ' --outdir ' + self.get_dir()) eval_name = os.path.basename(self.get_args().evaluation) ecode = self.run_external_command(eval_name, cmd_to_run, False, timeout=evaltimeout, kill_delay=killdelay) # write out evaluate exit code file self._write_evaluate_exitcode_file(ecode) # attempt to send evaluation email try: self._emailer.send_evaluation_email(self) self.append_to_email_log(self._emailer.get_message_log()) except Exception as e: logger.exception('Caught exception trying to send evaluation ' 'email') self.append_to_email_log('Caught exception trying to send ' 'evaluation email ' + str(e) + '\n') # assess the result self.end()
def test_run_succeeds_one_evaluation_with_emailer(self): temp_dir = tempfile.mkdtemp() try: params = D3RParameters() foo_script = os.path.join(temp_dir, 'foo.py') f = open(foo_script, 'w') f.write('#! /usr/bin/env python\n') f.write('import sys\nimport os\n') f.write('outdir = sys.argv[1]\n') f.write('sys.stdout.write(" ".join(sys.argv))\n') f.write('f = open(os.path.join(outdir, "summary.txt"),"w")\n') f.write('f.write("summary")\n') f.write('f.flush()\nf.close()\n') f.write('f = open(os.path.join(outdir, "yo.csv"), "a")\n') f.write('f.write("hi")\nf.flush()\nf.close()\n') f.write('sys.exit(0)\n') f.flush() f.close() os.chmod(foo_script, stat.S_IRWXU) params.postevaluation = foo_script etask = EvaluationTask( temp_dir, 'foo_dock.' + EvaluationTaskFactory.SCORING_SUFFIX, None, params) etask.create_dir() task = PostEvaluationTask(temp_dir, params) # setup emailer smtpemailer = SmtpEmailer() mockserver = D3RParameters() mockserver.sendmail = Mock() mockserver.quit = Mock() smtpemailer.set_alternate_smtp_server(mockserver) pee = PostEvaluationEmailer(['*****@*****.**'], smtpemailer) task.set_evaluation_emailer(pee) ctask = ChallengeDataTask(temp_dir, params) task.run() self.assertEqual(task.get_error(), None) self.assertEqual(task.get_status(), D3RTask.COMPLETE_STATUS) cfile = os.path.join(task.get_dir(), D3RTask.COMPLETE_FILE) self.assertTrue(os.path.isfile(cfile)) stdout_file = os.path.join(task.get_dir(), 'foo.py' + D3RTask.STDOUT_SUFFIX) self.assertTrue(os.path.isfile(stdout_file)) f = open(stdout_file, 'r') data = f.read() f.close() ctaskdir = os.path.join(ctask.get_dir(), ctask.get_celpp_challenge_data_dir_name()) self.assertTrue(' --evaluationdir ' + etask.get_dir() in data) self.assertTrue(' --challengedir ' + ctaskdir in data) self.assertTrue('--stageprefix stage.7. --evaluationsuffix ' + '.extsubmission.evaluation$|.evaluation$' in data) stderr_file = os.path.join(task.get_dir(), 'foo.py' + D3RTask.STDERR_SUFFIX) f = open(stdout_file, 'r') data = f.read() f.close() self.assertTrue(' --evaluationdir ' + etask.get_dir() in data) self.assertTrue(os.path.isfile(stderr_file)) self.assertEqual(task.get_error(), None) self.assertTrue('Sent post evaluation email to: ' '*****@*****.**' in task.get_email_log()) finally: shutil.rmtree(temp_dir)
def run(self): """Runs PostEvaluationTask Method requires can_run() to be called before hand with successful outcome Otherwise method invokes D3RTask.start then this method creates a directory and invokes evaluation script. Upon completion results are analyzed and success or error status is set appropriately and D3RTask.end is invoked """ super(PostEvaluationTask, self).run() if self._can_run is False: logger.debug(self.get_dir_name() + ' cannot run cause _can_run flag ' 'is False') return try: logger.debug('postevaluation set to ' + self.get_args().postevaluation) except AttributeError: self.set_error('postevaluation not set') self.end() return # # --evaluationdir <path to stage.X.evaluation> # --evaluationdir <path to another etc...> # --challengedir <path to challenge dir ie # stage.4.challengedata/celpp_week10_2017> # chall = ChallengeDataTask(self.get_path(), self.get_args()) chall_dname = chall.get_celpp_challenge_data_dir_name() evaldir_args = self._get_evaluationdir_args() if evaldir_args is not '': cmd_to_run = (self.get_args().postevaluation + ' ' + self.get_dir() + ' ' + evaldir_args + ' --challengedir ' + os.path.join(chall.get_dir(), chall_dname) + ' --stageprefix ' + self._eval_task_prefix_str + ' --evaluationsuffix ' + self._eval_task_suffix_str) peval_name = os.path.basename(self.get_args().postevaluation) self.run_external_command(peval_name, cmd_to_run, True) else: self.set_error('Did not find any Evaluation tasks to summarize') # attempt to send postevaluation email try: self._emailer.send_postevaluation_email(self) self.append_to_email_log(self._emailer.get_message_log()) except Exception as e: logger.exception('Caught exception trying to send ' 'postevaluation email') self.append_to_email_log('Caught exception trying to send ' 'evaluation email ' + str(e) + '\n') # assess the result self.end()
def test_run_succeeds_no_ftp(self): temp_dir = tempfile.mkdtemp() try: script = self.create_gen_challenge_script(temp_dir) params = D3RParameters() params.genchallenge = script params.pdbdb = '/foo' params.version = '1' blastnfilter = BlastNFilterTask(temp_dir, params) blastnfilter.create_dir() open(os.path.join(blastnfilter.get_dir(), D3RTask.COMPLETE_FILE), 'a').close() chall = ChallengeDataTask(temp_dir, params) dimport = DataImportTask(temp_dir, params) dimport.create_dir() ctsv = dimport.get_crystalph_tsv() f = open(ctsv, 'w') f.write('crystal') f.flush() f.close() nonpoly = dimport.get_nonpolymer_tsv() f = open(nonpoly, 'w') f.write('nonpoly') f.flush() f.close() seq = dimport.get_sequence_tsv() f = open(seq, 'w') f.write('seq') f.flush() f.close() chall.run() self.assertEqual(chall.get_error(), None) # verify test files get created errfile = os.path.join(chall.get_dir(), D3RTask.ERROR_FILE) self.assertEqual(os.path.isfile(errfile), False) compfile = os.path.join(chall.get_dir(), D3RTask.COMPLETE_FILE) self.assertEqual(os.path.isfile(compfile), True) stderr = os.path.join(chall.get_dir(), 'genchallenge.py.stderr') self.assertEqual(os.path.isfile(stderr), True) stdout = os.path.join(chall.get_dir(), 'genchallenge.py.stdout') self.assertEqual(os.path.isfile(stdout), True) # verify challenge directory is created and # filled with valid files chall_dir = os.path.join(chall.get_dir(), chall.get_celpp_challenge_data_dir_name()) self.assertEqual(os.path.isdir(chall_dir), True) readme = os.path.join(chall_dir, ChallengeDataTask.README_TXT_FILE) self.assertEqual(os.path.isfile(readme), True) crystal = os.path.join(chall_dir, DataImportTask.CRYSTALPH_TSV) self.assertEqual(os.path.isfile(crystal), True) seq = os.path.join(chall_dir, DataImportTask.SEQUENCE_TSV) self.assertEqual(os.path.isfile(seq), True) nonpoly = os.path.join(chall_dir, DataImportTask.NONPOLYMER_TSV) self.assertEqual(os.path.isfile(nonpoly), True) fivehibdir = os.path.join(chall_dir, '5hib') fivehibtxt = os.path.join(fivehibdir, '5hib.txt') self.assertEqual(os.path.isfile(fivehibtxt), True) fivehibfas = os.path.join(fivehibdir, '5hib.fasta') self.assertEqual(os.path.isfile(fivehibfas), True) fivehicdir = os.path.join(chall_dir, '5hic') fivehictxt = os.path.join(fivehicdir, '5hic.txt') self.assertEqual(os.path.isfile(fivehictxt), True) fivehicfas = os.path.join(fivehicdir, '5hic.fasta') self.assertEqual(os.path.isfile(fivehicfas), True) # verify tarball is created tfile = chall.get_celpp_challenge_data_tar_file() self.assertEqual(os.path.isfile(tfile), True) foodir = os.path.join(temp_dir, 'foo') os.mkdir(foodir) tar = tarfile.open(tfile, 'r:*') tar.extractall(path=foodir) tar.close() name = chall.get_celpp_challenge_data_dir_name() cdir = os.path.join(foodir, name) readme = os.path.join(cdir, ChallengeDataTask.README_TXT_FILE) self.assertEqual(os.path.isfile(readme), True) final = os.path.join(foodir, name, 'final.log') self.assertEqual(os.path.isfile(final), False) e_con = os.path.join(foodir, name, 'error_container') self.assertEqual(os.path.isdir(e_con), False) finally: shutil.rmtree(temp_dir)