def test_generate_post_evaluation_email_body(self): temp_dir = tempfile.mkdtemp() try: params = D3RParameters() params.version = '1.2.3' weekyear = os.path.join(temp_dir, '2017', 'dataset.week.12') os.makedirs(weekyear, mode=0o755) task = PostEvaluationTask(weekyear, params) task.create_dir() f = open(task.get_summary_txt(), 'w') f.write('hello') f.flush() f.close() pee = PostEvaluationEmailer(None, None) subject, msg = pee._generate_post_evaluation_email_body(task) self.assertEqual( subject, D3RTask.SUBJECT_LINE_PREFIX + '2017 week 12 post evaluation summary report') self.assertEqual( msg, 'Dear CELPP Admin,\n\nHere is the post ' 'evaluation summary reports for CELPP ' 'week 12\n\n\n\nhello\n\n\nSincerely,\n\n' 'CELPP Automation 1.2.3') finally: shutil.rmtree(temp_dir)
def test_get_summary_txt(self): params = D3RParameters() task = PostEvaluationTask('/foo', params) self.assertEqual( task.get_summary_txt(), os.path.join(task.get_dir(), PostEvaluationTask.SUMMARY_TXT))
def test_run_fails_one_evaluation_and_no_emailer(self): temp_dir = tempfile.mkdtemp() try: params = D3RParameters() params.postevaluation = 'false' etask = EvaluationTask( temp_dir, 'foo_dock.' + EvaluationTaskFactory.SCORING_SUFFIX, None, params) etask.create_dir() task = PostEvaluationTask(temp_dir, params) task.run() self.assertEqual( task.get_error(), 'Non zero exit code: 1 received. ' 'Standard out: Standard error: ') self.assertEqual(task.get_status(), D3RTask.ERROR_STATUS) self.assertTrue( os.path.isfile(os.path.join(task.get_dir(), D3RTask.ERROR_FILE))) self.assertTrue('Caught exception trying to ' 'send evaluation email' in task.get_email_log()) stdout_file = os.path.join(task.get_dir(), 'false' + D3RTask.STDOUT_SUFFIX) self.assertTrue(os.path.isfile(stdout_file)) stderr_file = os.path.join(task.get_dir(), 'false' + D3RTask.STDERR_SUFFIX) self.assertTrue(os.path.isfile(stderr_file)) finally: shutil.rmtree(temp_dir)
def test_get_all_evaluation_tasks(self): temp_dir = tempfile.mkdtemp() try: params = D3RParameters() # no evaluation tasks task = PostEvaluationTask(temp_dir, params) res = task.get_all_evaluation_tasks() self.assertEqual(len(res), 0) # directory in path that does NOT match suffix task.create_dir() os.makedirs(os.path.join(temp_dir, 'stage.x.1'), mode=0o755) res = task.get_all_evaluation_tasks() self.assertEqual(len(res), 0) # one task, but is file (weird case) weirdtaskfile = os.path.join( temp_dir, 'stage.7.hi.' + EvaluationTaskFactory.SCORING_SUFFIX) open(weirdtaskfile, 'a').close() res = task.get_all_evaluation_tasks() self.assertEqual(len(res), 0) self.assertEqual( task.get_email_log(), 'Just a note, found a task with valid name, but ' 'it is not a directory ' + weirdtaskfile) # one task etask = EvaluationTask(temp_dir, 'foo.evaluation', None, params) etask.create_dir() res = task.get_all_evaluation_tasks() self.assertEqual(len(res), 1) self.assertTrue(res[0].get_dir_name(), etask.get_dir_name()) # three tasks two with non complete status etask2 = EvaluationTask(temp_dir, 'bla_dock.extsubmission.evaluation', None, params) etask2.create_dir() open(os.path.join(etask2.get_dir(), D3RTask.ERROR_FILE), 'a').close() etask3 = EvaluationTask(temp_dir, '12_x.extsubmission.evaluation', None, params) etask3.create_dir() open(os.path.join(etask3.get_dir(), D3RTask.COMPLETE_FILE), 'a').close() res = task.get_all_evaluation_tasks() self.assertEqual(len(res), 3) d_name_list = [] for entry in res: d_name_list.append(entry.get_dir_name()) self.assertTrue(etask.get_dir_name() in d_name_list) self.assertTrue(etask2.get_dir_name() in d_name_list) self.assertTrue(etask3.get_dir_name() in d_name_list) finally: shutil.rmtree(temp_dir)
def test_send_postevaluation_email_exception(self): temp_dir = tempfile.mkdtemp() try: params = D3RParameters() weekyear = os.path.join(temp_dir, '2017', 'dataset.week.12') os.makedirs(weekyear, mode=0o755) task = PostEvaluationTask(weekyear, params) task.create_dir() csvfile = os.path.join(task.get_dir(), 'foo.csv') open(csvfile, 'a').close() smtpemailer = SmtpEmailer() mockserver = D3RParameters() mockserver.sendmail = Mock(side_effect=IOError('ha')) mockserver.quit = Mock() smtpemailer.set_alternate_smtp_server(mockserver) pee = PostEvaluationEmailer(['*****@*****.**'], smtpemailer) task.set_evaluation_emailer(pee) f = open(task.get_summary_txt(), 'w') f.write('hello') f.flush() f.close() pee.send_postevaluation_email(task) mockserver.quit.assert_any_call() self.assertEqual( pee.get_message_log(), '\nCaught exception trying to email :' ' [email protected] : Caught exception ha\n') self.assertEqual(mockserver.sendmail.call_count, 1) finally: shutil.rmtree(temp_dir)
def test_send_postevaluation_email_success(self): temp_dir = tempfile.mkdtemp() try: params = D3RParameters() weekyear = os.path.join(temp_dir, '2017', 'dataset.week.12') os.makedirs(weekyear, mode=0o755) task = PostEvaluationTask(weekyear, params) task.create_dir() smtpemailer = SmtpEmailer() mockserver = D3RParameters() mockserver.sendmail = Mock() mockserver.quit = Mock() smtpemailer.set_alternate_smtp_server(mockserver) pee = PostEvaluationEmailer(['*****@*****.**'], smtpemailer) task.set_evaluation_emailer(pee) f = open(task.get_summary_txt(), 'w') f.write('hello') f.flush() f.close() pee.send_postevaluation_email(task) mockserver.quit.assert_any_call() self.assertEqual(pee.get_message_log(), '\nSent post evaluation email to: [email protected]\n') self.assertEqual(mockserver.sendmail.call_count, 1) finally: shutil.rmtree(temp_dir)
def test_get_evaluationdir_args(self): temp_dir = tempfile.mkdtemp() try: params = D3RParameters() # no evaluation tasks task = PostEvaluationTask(temp_dir, params) res = task._get_evaluationdir_args() self.assertEqual(res, '') # one task etask = EvaluationTask(temp_dir, 'foo.evaluation', None, params) etask.create_dir() res = task._get_evaluationdir_args() self.assertEqual( res, ' ' + PostEvaluationTask.EVALUATIONDIR_ARG + ' ' + etask.get_dir()) # three tasks etask2 = EvaluationTask(temp_dir, 'bla_dock.extsubmission.evaluation', None, params) etask2.create_dir() open(os.path.join(etask2.get_dir(), D3RTask.ERROR_FILE), 'a').close() etask3 = EvaluationTask(temp_dir, '12_x.extsubmission.evaluation', None, params) etask3.create_dir() cfile = os.path.join(etask3.get_dir(), D3RTask.COMPLETE_FILE) open(cfile, 'a').close() res = task._get_evaluationdir_args() self.assertTrue(' ' + PostEvaluationTask.EVALUATIONDIR_ARG + ' ' + etask.get_dir() in res) self.assertTrue(' ' + PostEvaluationTask.EVALUATIONDIR_ARG + ' ' + etask2.get_dir() in res) self.assertTrue(' ' + PostEvaluationTask.EVALUATIONDIR_ARG + ' ' + etask3.get_dir() in res) finally: shutil.rmtree(temp_dir)
def test_run_fails_cause_can_run_is_false(self): temp_dir = tempfile.mkdtemp() try: params = D3RParameters() task = PostEvaluationTask(temp_dir, params) task.create_dir() task.run() self.assertEqual( task.get_error(), task.get_dir_name() + ' already exists and status is ' + D3RTask.UNKNOWN_STATUS) finally: shutil.rmtree(temp_dir)
def test_run_fails_cause_postevaluation_not_set(self): temp_dir = tempfile.mkdtemp() try: params = D3RParameters() task = PostEvaluationTask(temp_dir, params) task.run() self.assertEqual(task.get_error(), 'postevaluation not set') # test files get created self.assertEqual(os.path.isdir(task.get_dir()), True) errfile = os.path.join(task.get_dir(), D3RTask.ERROR_FILE) self.assertEqual(os.path.isfile(errfile), True) finally: shutil.rmtree(temp_dir)
def test_run_succeeds_one_evaluation_and_no_emailer(self): temp_dir = tempfile.mkdtemp() try: params = D3RParameters() params.postevaluation = 'echo' etask = EvaluationTask( temp_dir, 'foo_dock.' + EvaluationTaskFactory.SCORING_SUFFIX, None, params) etask.create_dir() task = PostEvaluationTask(temp_dir, params) ctask = ChallengeDataTask(temp_dir, params) task.run() self.assertEqual(task.get_error(), None) self.assertEqual(task.get_status(), D3RTask.COMPLETE_STATUS) cfile = os.path.join(task.get_dir(), D3RTask.COMPLETE_FILE) self.assertTrue(os.path.isfile(cfile)) stdout_file = os.path.join(task.get_dir(), 'echo' + D3RTask.STDOUT_SUFFIX) self.assertTrue(os.path.isfile(stdout_file)) f = open(stdout_file, 'r') data = f.read() f.close() ctaskdir = os.path.join(ctask.get_dir(), ctask.get_celpp_challenge_data_dir_name()) self.assertTrue(' --evaluationdir ' + etask.get_dir() in data) self.assertTrue(' --challengedir ' + ctaskdir in data) self.assertTrue('--stageprefix stage.7. --evaluationsuffix ' + '.extsubmission.evaluation$|.evaluation$' in data) stderr_file = os.path.join(task.get_dir(), 'echo' + D3RTask.STDERR_SUFFIX) self.assertTrue(os.path.isfile(stderr_file)) finally: shutil.rmtree(temp_dir)
def test_run_fail_no_evaluations_and_no_emailer(self): temp_dir = tempfile.mkdtemp() try: params = D3RParameters() params.postevaluation = 'true' task = PostEvaluationTask(temp_dir, params) task.run() self.assertEqual(task.get_error(), 'Did not find any Evaluation tasks to summarize') self.assertEqual(task.get_status(), D3RTask.ERROR_STATUS) self.assertTrue( os.path.isfile(os.path.join(task.get_dir(), D3RTask.ERROR_FILE))) finally: shutil.rmtree(temp_dir)
def main(args): """Main entry into celpprunner :param args: should be set to sys.argv which is a list of arguments starting with script name as the first argument """ blasttask = BlastNFilterTask('', p) dataimport = DataImportTask('', p) challenge = ChallengeDataTask('', p) glide = GlideTask('', p) makedb = MakeBlastDBTask('', p) prot = ProteinLigPrepTask('', p) vina = AutoDockVinaTask('', p) chimeraprep = ChimeraProteinLigPrepTask('', p) postevalstage = PostEvaluationTask('', p) desc = """ Version {version} Runs the 11 stages (makedb, import, blast, challengedata, proteinligprep, {chimeraprep}, extsubmission, glide, vina, evaluation, & {postevaluation}) of CELPP processing pipeline (http://www.drugdesigndata.org) CELPP processing pipeline relies on a set of directories with specific structure. The pipeline runs a set of stages Each stage has a numerical value and a name. The numerical value denotes order and the stage name identifies separate tasks to run in the stage. The filesystem structure of the stage is: stage.<stage number>.<task name> The stage(s) run are defined via the required {stageflag} flag. To run multiple stages serially just pass a comma delimited list to the {stageflag} flag. Example: {stageflag} import,blast NOTE: When running multiple stages serially the program will not run subsequent stages if a task in a stage fails. Also note order matters, ie putting blast,import will cause celpprunner.py to run blast stage first. This program drops a pid lockfile (celpprunner.<stage>.lockpid) in celppdir to prevent duplicate invocation. When run, this program will examine the stage and see if work can be done. If stage is complete or previous steps have not completed, the program will exit silently. If previous steps have failed or current stage already exists in an error or uncomplete state then program will report the error via email using addresses set in --email and in --summaryemail flag. Errors will also be reported via stderr/stdout. The program will also exit with nonzero exit code. This program utilizes simple token files to denote stage completion. If within the stage directory there is a: '{complete}' file - then stage is done and no other checking is done. 'error' file - then stage failed. 'start' file - then stage is running. Notification of stage start and end will be sent to addresses set via --email flag and only errors will be sent to addresses in --summaryemail. Emails are sent via smtp which can be configured via {smtpconfig} parameter which takes a configuration file in the following format: [{smtp_section}] {smtp_host} = smtp.foo.com {smtp_port} = 1234 {smtp_user} = bob {smtp_pass} = somepass {smtp_from_address} = [email protected] {smtp_replyto_address} = [email protected] Unless --customweekdir is set, this program will examine the 'celppdir' (last argument passed on commandline) to find the latest directory with this path: <year>/dataset.week.# The program will find the latest <year> and within that year the dataset.week.# with highest #. The output directories created will be put within this directory. Setting --customweekdir will cause program to use 'celppdir' path. Setting the --createweekdir flag will instruct this program to create a new directory for the current celpp week/year before running any stage processing. NOTE: CELPP weeks start on Friday and end on Thursday and week # follows ISO8601 rules so week numbers at the end and start of the year are a bit wonky. Breakdown of behavior of program is defined by value passed with {stageflag} flag: If {stageflag} '{createchallenge}' This is NOT a stage, but has the same effect as calling {stageflag} makedb,import,blast,challengedata The four stages that need to run to generate the challenge data package. If {stageflag} 'makedb' In this stage the file {pdb_seqres} is downloaded from an ftp site set by --pdbsequrl. This file is then gunzipped and NCBI makeblastdb (set by --makeblastdb) is run on it to create a blast database. The files are stored in {makeblastdb_dirname} If {stageflag} 'import' In this stage 4 files are downloaded from urls specified by --compinchi and --pdbfileurl flags on the commandline into {dataimport_dirname} directory. The tsv files are (--pdbfileurl flag sets url to download these files from): {nonpolymer_tsv} {sequence_tsv} {crystal_tsv} The Components ich file is (--compinchi flag sets base url to download this file from): {compinchi_ich} This stage will just wait and retry if any of the tsv files have NOT been updated since the start of the current celpp week as determined by a HEAD request. To bypass this delay add --skipimportwait flag. --importsleep denotes the time to wait before re-examining the update time of the tsv files and --importretry sets number of times to retry before giving up. If {stageflag} 'blast' Verifies {dataimport_dirname} exists and has '{complete}' file. Also verifies {makeblastdb_dirname} exists and has '{complete}' file. If both conditions are met then the 'blast' stage is run which invokes script set by --blastnfilter flag and output stored in {blast_dirname}. Requires --pdbdb to be set to a directory with valid PDB database files. Note: --blastnfilter script is killed after time set with --blastnfiltertimeout flag. If {stageflag} 'challengedata' Verifies {blast_dirname} exists and has '{complete}' file. If complete, this stage runs which invokes program set in --genchallenge flag to create a challenge dataset file. The --pdbdb flag must also be set when calling this stage. If --ftpconfig is set with {challengepath} field then this stage will also upload the challenge dataset tarfile to the ftp server with path set by {challengepath}. The code will also upload a {latest_txt} file containing name of the tarfile to the same destination overwriting any {latest_txt} file that already exists. Example file for --ftpconfig: {host} some.ftp.com {user} bob {passn} mypass {path} /celpp {challengepath} /challenge {submissionpath} /submissions If {stageflag} '{chimeraprep}' Verifies {challenge_dirname} exists and has '{complete}' file. If complete, this stage runs which invokes program set in --chimeraprep flag to prepare pdb and inchi files storing output in {chimeraprep_dirname}. --pdbdb flag must also be set when calling this stage. If {stageflag} 'proteinligprep' Verifies {challenge_dirname} exists and has '{complete}' file. If complete, this stage runs which invokes program set in --proteinligprep flag to prepare pdb and inchi files storing output in {proteinligprep_dirname}. --pdbdb flag must also be set when calling this stage. If {stageflag} 'extsubmission' Connects to server specified by --ftpconfig and downloads external docking submissions from {submissionpath} on remote server. Submissions should be named: celpp_weekXX_YYYY_dockedresults_ZZZZ.tar.gz as documented here: https://github.com/drugdata/d3r/wiki/Proposed-challenge-docked\ -results-file-structure For each submission a directory named stage.X.ZZZZ.extsubmission will be created and uncompressed contents of package will be stored in that directory. If data does not conform properly 'error' file will be placed in directory denoting failure If {stageflag} 'glide' Verifies {proteinligprep_dirname} exists and has a '{complete}' file within it. If complete, this stage runs which invokes program set in --glide flag to perform docking via glide storing output in {glide_dirname} If {stageflag} 'vina' Verifies {proteinligprep_dirname} exists and has a '{complete}' file within it. If complete, this stage runs which invokes program set in --vina flag to perform docking via AutoDock Vina storing output in {vina_dirname} If {stageflag} 'evaluation' Finds all stage.{dockstage}.<algo> directories with '{complete}' files in them which do not end in name '{webdata}' and runs script set via --evaluation parameter storing the result of the script into stage.{evalstage}.<algo>.evaluation. --pdbdb flag must also be set when calling this stage. If {stageflag} '{postevaluation}' Finds all stage.{evalstage}.<algo>.evaluation directories and runs script set via --postevaluation parameter storing a summary of found docking evaluations into {postevalstage} into a file named summary.txt and a set of files with name Overall_RMSD_<candidate_type>.csv. In addition, results will be emailed to people in --summaryemail and --email lists. """.format(makeblastdb_dirname=makedb.get_dir_name(), dataimport_dirname=dataimport.get_dir_name(), blast_dirname=blasttask.get_dir_name(), challenge_dirname=challenge.get_dir_name(), createchallenge=CREATE_CHALLENGE, proteinligprep_dirname=prot.get_dir_name(), glide_dirname=glide.get_dir_name(), vina_dirname=vina.get_dir_name(), dockstage=str(glide.get_stage()), evalstage=str(glide.get_stage() + 1), complete=blasttask.COMPLETE_FILE, chimeraprep_dirname=chimeraprep.get_dir_name(), chimeraprep=CHIMERA_PREP, compinchi_ich=DataImportTask.COMPINCHI_ICH, pdb_seqres=MakeBlastDBTask.PDB_SEQRES_TXT_GZ, nonpolymer_tsv=DataImportTask.NONPOLYMER_TSV, sequence_tsv=DataImportTask.SEQUENCE_TSV, crystal_tsv=DataImportTask.CRYSTALPH_TSV, webdata=EvaluationTaskFactory.WEB_DATA_SUFFIX, latest_txt=ChallengeDataTask.LATEST_TXT, host=FtpFileTransfer.HOST, user=FtpFileTransfer.USER, passn=FtpFileTransfer.PASS, path=FtpFileTransfer.PATH, challengepath=FtpFileTransfer.CHALLENGEPATH, submissionpath=FtpFileTransfer.SUBMISSIONPATH, version=d3r.__version__, stageflag=STAGE_FLAG, postevaluation=POST_EVAL, postevalstage=postevalstage.get_dir_name(), smtpconfig=SMTPCONFIG_FLAG, smtp_section=SmtpConfig.DEFAULT, smtp_host=SmtpConfig.SMTP_HOST, smtp_port=SmtpConfig.SMTP_PORT, smtp_user=SmtpConfig.SMTP_USER, smtp_pass=SmtpConfig.SMTP_PASS, smtp_from_address=SmtpConfig.SMTP_FROM_ADDRESS, smtp_replyto_address=SmtpConfig.SMTP_REPLYTO_ADDRESS) theargs = _parse_arguments(desc, args[1:]) theargs.program = args[0] theargs.version = d3r.__version__ util.setup_logging(theargs) try: return run_stages(theargs) except Exception: logger.exception("Error caught exception") return 2
def test_postevaluationtask_constructor(self): params = D3RParameters() task = PostEvaluationTask('/blah', params) self.assertEquals(task.get_name(), PostEvaluationTask.TASK_NAME) self.assertEquals(task.get_stage(), 8) self.assertEqual(task.get_status(), D3RTask.UNKNOWN_STATUS)
def test_get_all_csv_files_in_dir(self): temp_dir = tempfile.mkdtemp() try: params = D3RParameters() task = PostEvaluationTask(temp_dir, params) task.create_dir() # test empty dir res = task.get_all_csv_files_in_dir() self.assertEqual(len(res), 0) # add directory with csv ending csvdir = os.path.join(task.get_dir(), 'hi ' + PostEvaluationTask.CSV_SUFFIX) os.makedirs(csvdir, mode=0o755) res = task.get_all_csv_files_in_dir() self.assertEqual(len(res), 0) # 1 csv onecsv = os.path.join( task.get_dir(), 'Overall_RMSD_hi' + PostEvaluationTask.CSV_SUFFIX) open(onecsv, 'a').close() res = task.get_all_csv_files_in_dir() self.assertEqual(len(res), 1) self.assertTrue(onecsv in res) # 3 csv twocsv = os.path.join( task.get_dir(), 'Overall_RMSD_bye' + PostEvaluationTask.CSV_SUFFIX) open(twocsv, 'a').close() threecsv = os.path.join(task.get_dir(), 'some' + PostEvaluationTask.CSV_SUFFIX) open(threecsv, 'a').close() res = task.get_all_csv_files_in_dir() self.assertEqual(len(res), 3) self.assertTrue(onecsv in res) self.assertTrue(twocsv in res) self.assertTrue(threecsv in res) finally: shutil.rmtree(temp_dir)
def test_get_postevaluation_summary(self): temp_dir = tempfile.mkdtemp() try: params = D3RParameters() task = PostEvaluationTask(temp_dir, params) task.create_dir() # no file res = task.get_postevaluation_summary() self.assertEqual( res, '\n\nNo ' + task.get_summary_txt() + ' file found\n') # empty file s_file = os.path.join(task.get_dir(), PostEvaluationTask.SUMMARY_TXT) open(s_file, 'a').close() res = task.get_postevaluation_summary() self.assertEqual(res, '\n\n\n') os.chmod(task.get_summary_txt(), 0) res = task.get_postevaluation_summary() self.assertTrue('\n\nUnable to generate postevaluation' in res) os.chmod(task.get_summary_txt(), stat.S_IRWXU) # some data in the file f = open(s_file, 'w') f.write('hello\nhi\n') f.flush() f.close() res = task.get_postevaluation_summary() self.assertEqual(res, '\n\nhello\nhi\n\n') finally: shutil.rmtree(temp_dir)
def test_run_succeeds_one_evaluation_with_emailer(self): temp_dir = tempfile.mkdtemp() try: params = D3RParameters() foo_script = os.path.join(temp_dir, 'foo.py') f = open(foo_script, 'w') f.write('#! /usr/bin/env python\n') f.write('import sys\nimport os\n') f.write('outdir = sys.argv[1]\n') f.write('sys.stdout.write(" ".join(sys.argv))\n') f.write('f = open(os.path.join(outdir, "summary.txt"),"w")\n') f.write('f.write("summary")\n') f.write('f.flush()\nf.close()\n') f.write('f = open(os.path.join(outdir, "yo.csv"), "a")\n') f.write('f.write("hi")\nf.flush()\nf.close()\n') f.write('sys.exit(0)\n') f.flush() f.close() os.chmod(foo_script, stat.S_IRWXU) params.postevaluation = foo_script etask = EvaluationTask( temp_dir, 'foo_dock.' + EvaluationTaskFactory.SCORING_SUFFIX, None, params) etask.create_dir() task = PostEvaluationTask(temp_dir, params) # setup emailer smtpemailer = SmtpEmailer() mockserver = D3RParameters() mockserver.sendmail = Mock() mockserver.quit = Mock() smtpemailer.set_alternate_smtp_server(mockserver) pee = PostEvaluationEmailer(['*****@*****.**'], smtpemailer) task.set_evaluation_emailer(pee) ctask = ChallengeDataTask(temp_dir, params) task.run() self.assertEqual(task.get_error(), None) self.assertEqual(task.get_status(), D3RTask.COMPLETE_STATUS) cfile = os.path.join(task.get_dir(), D3RTask.COMPLETE_FILE) self.assertTrue(os.path.isfile(cfile)) stdout_file = os.path.join(task.get_dir(), 'foo.py' + D3RTask.STDOUT_SUFFIX) self.assertTrue(os.path.isfile(stdout_file)) f = open(stdout_file, 'r') data = f.read() f.close() ctaskdir = os.path.join(ctask.get_dir(), ctask.get_celpp_challenge_data_dir_name()) self.assertTrue(' --evaluationdir ' + etask.get_dir() in data) self.assertTrue(' --challengedir ' + ctaskdir in data) self.assertTrue('--stageprefix stage.7. --evaluationsuffix ' + '.extsubmission.evaluation$|.evaluation$' in data) stderr_file = os.path.join(task.get_dir(), 'foo.py' + D3RTask.STDERR_SUFFIX) f = open(stdout_file, 'r') data = f.read() f.close() self.assertTrue(' --evaluationdir ' + etask.get_dir() in data) self.assertTrue(os.path.isfile(stderr_file)) self.assertEqual(task.get_error(), None) self.assertTrue('Sent post evaluation email to: ' '*****@*****.**' in task.get_email_log()) finally: shutil.rmtree(temp_dir)
def test_get_uploadable_files(self): temp_dir = tempfile.mkdtemp() try: params = D3RParameters() task = PostEvaluationTask(temp_dir, params) # try with no dir self.assertEqual(task.get_uploadable_files(), []) # try with empty dir task.create_dir() self.assertEqual(task.get_uploadable_files(), []) # try with final log final_log = os.path.join(task.get_dir(), EvaluationTask.FINAL_LOG) open(final_log, 'a').close() flist = task.get_uploadable_files() self.assertEqual(len(flist), 1) self.assertTrue(final_log in flist) # try with csv file csvfile = os.path.join( task.get_dir(), 'Overall_RMSD_foo' + PostEvaluationTask.CSV_SUFFIX) open(csvfile, 'a').close() flist = task.get_uploadable_files() self.assertEqual(len(flist), 2) self.assertTrue(csvfile in flist) self.assertTrue(final_log in flist) # try with summary.txt file open(task.get_summary_txt(), 'a').close() flist = task.get_uploadable_files() self.assertEqual(len(flist), 3) self.assertTrue(task.get_summary_txt() in flist) self.assertTrue(final_log in flist) # try with 2 csv file csvfile2 = os.path.join( task.get_dir(), 'Overall_RMSD_foo2' + PostEvaluationTask.CSV_SUFFIX) open(csvfile2, 'a').close() flist = task.get_uploadable_files() self.assertEqual(len(flist), 4) self.assertTrue(csvfile in flist) self.assertTrue(csvfile2 in flist) self.assertTrue(task.get_summary_txt() in flist) self.assertTrue(final_log in flist) finally: shutil.rmtree(temp_dir)
def test_can_run(self): temp_dir = tempfile.mkdtemp() try: params = D3RParameters() # post evaluation task exists already task = PostEvaluationTask(temp_dir, params) task.create_dir() self.assertEqual(task.can_run(), False) self.assertEqual( task.get_error(), task.get_dir_name() + ' already exists and status is unknown') # post evaluation task already complete task = PostEvaluationTask(temp_dir, params) open(os.path.join(task.get_dir(), D3RTask.COMPLETE_FILE), 'a').close() self.assertEqual(task.can_run(), False) self.assertEqual(task.get_error(), None) finally: shutil.rmtree(temp_dir)
def get_task_list_for_stage(theargs, stage_name): """Factory method that generates a list of tasks for given stage Using stage_name get the list of tasks that need to be run. :param theargs: parameters set via commandline along with ``theargs.latest_weekly`` which should be set to to base directory where stages will be run :param stage_name: Name of stage to run """ if stage_name is None: raise NotImplementedError('stage_name is None') task_list = [] logger.debug('Getting task list for ' + stage_name) if stage_name == CREATE_CHALLENGE: task_list.append(MakeBlastDBTask(theargs.latest_weekly, theargs)) task_list.append(DataImportTask(theargs.latest_weekly, theargs)) task_list.append(BlastNFilterTask(theargs.latest_weekly, theargs)) task_list.append(ChallengeDataTask(theargs.latest_weekly, theargs)) if stage_name == 'makedb': task_list.append(MakeBlastDBTask(theargs.latest_weekly, theargs)) if stage_name == 'import': task_list.append(DataImportTask(theargs.latest_weekly, theargs)) if stage_name == 'blast': task_list.append(BlastNFilterTask(theargs.latest_weekly, theargs)) if stage_name == 'challengedata': task_list.append(ChallengeDataTask(theargs.latest_weekly, theargs)) if stage_name == 'proteinligprep': task_list.append(ProteinLigPrepTask(theargs.latest_weekly, theargs)) if stage_name == 'glide': task_list.append(GlideTask(theargs.latest_weekly, theargs)) if stage_name == 'vina': task_list.append(AutoDockVinaTask(theargs.latest_weekly, theargs)) if stage_name == CHIMERA_PREP: task_list.append( ChimeraProteinLigPrepTask(theargs.latest_weekly, theargs)) if stage_name == 'extsubmission': extfac = ExternalDataSubmissionFactory(theargs.latest_weekly, theargs) task_list.extend(extfac.get_external_data_submissions()) if stage_name == 'evaluation': # use util function call to get all evaluation tasks # append them to the task_list eval_task_factory = EvaluationTaskFactory(theargs.latest_weekly, theargs) task_list.extend(eval_task_factory.get_evaluation_tasks()) if stage_name == POST_EVAL: # create PostEvaluationEmailer object ptask = PostEvaluationTask(theargs.latest_weekly, theargs) a_list = _get_set_of_email_address_from_email_flags(theargs) efac = SmtpEmailerFactory(theargs) pmailer = PostEvaluationEmailer(a_list, efac.get_smtp_emailer()) ptask.set_evaluation_emailer(pmailer) task_list.append(ptask) if len(task_list) is 0: raise NotImplementedError('uh oh no tasks for ' + stage_name + ' stage') return task_list