def test_run_fails_cause_genchallenge_is_not_found(self):
temp_dir = tempfile.mkdtemp()
try:
params = D3RParameters()
params.genchallenge = '/bin/doesnotexist'
params.pdbdb = '/foo'
params.version = '1'
blastnfilter = BlastNFilterTask(temp_dir, params)
blastnfilter.create_dir()
open(os.path.join(blastnfilter.get_dir(), D3RTask.COMPLETE_FILE),
'a').close()
chall = ChallengeDataTask(temp_dir, params)
chall.run()
self.assertEqual(chall.get_error(),
'Caught Exception trying to run ' +
'/bin/doesnotexist --candidatedir ' +
blastnfilter.get_dir() + ' --pdbdb ' +
'/foo --outdir ' +
chall.get_dir() +
'/' + chall.get_celpp_challenge_data_dir_name() +
' : [Errno 2] No such file or directory')
# test files get created
errfile = os.path.join(chall.get_dir(),
D3RTask.ERROR_FILE)
self.assertEqual(os.path.isfile(errfile), True)
finally:
shutil.rmtree(temp_dir)
def test_run_fails_cause_chimeraprep_is_not_found_rdkitpython_set(self):
temp_dir = tempfile.mkdtemp()
try:
params = D3RParameters()
params.chimeraprep = '/bin/doesnotexist'
params.pdbdb = '/foo'
params.rdkitpython = '/data/miniconda2/bin'
chall = ChallengeDataTask(temp_dir, params)
chall.create_dir()
challdir = os.path.join(chall.get_dir(),
chall.get_celpp_challenge_data_dir_name())
open(os.path.join(chall.get_dir(), D3RTask.COMPLETE_FILE),
'a').close()
task = ChimeraProteinLigPrepTask(temp_dir, params)
task.run()
self.assertEqual(
task.get_error(), 'Caught Exception trying to run ' +
'/bin/doesnotexist --candidatedir ' + challdir +
' --rdkitpython \'/data/miniconda2/bin\' ' + '--pdbdb ' +
'/foo --outdir ' + task.get_dir() +
' : [Errno 2] No such file or directory')
# test files get created
errfile = os.path.join(task.get_dir(), D3RTask.ERROR_FILE)
self.assertEqual(os.path.isfile(errfile), True)
finally:
shutil.rmtree(temp_dir)
def run(self):
"""Runs ChimeraProteinLigPrepTask after verifying BlastNFilterTask
was good
Method requires can_run() to be called before hand with
successful outcome
Otherwise method invokes D3RTask.start then this method
creates a directory and invokes blastnfilter script. Upon
completion results are analyzed and success or error status
is set appropriately and D3RTask.end is invoked
"""
super(ChimeraProteinLigPrepTask, self).run()
if self._can_run is False:
logger.debug(self.get_dir_name() +
' cannot run cause _can_run flag '
'is False')
return
try:
logger.debug('chimeraprep set to ' + self.get_args().chimeraprep)
except AttributeError:
self.set_error('chimeraprep not set')
self.end()
return
try:
logger.debug('pdbdb set to ' + self.get_args().pdbdb)
except AttributeError:
self.set_error('pdbdb not set')
self.end()
return
rdkitpython = ''
try:
logger.debug('rdkitpython set to ' + self.get_args().rdkitpython)
rdkitpython = self.get_args().rdkitpython
except AttributeError:
logger.debug
chall = ChallengeDataTask(self._path, self._args)
#
# chimeraprep.py --candidatedir <path to stage.3.challengedata> \
# --outdir <path to stage.4.chimeraligprep>
challdir = os.path.join(chall.get_dir(),
chall.get_celpp_challenge_data_dir_name())
cmd_to_run = (self.get_args().chimeraprep + ' --candidatedir ' +
challdir + ' --rdkitpython \'' + rdkitpython + '\'' +
' --pdbdb ' + self.get_args().pdbdb + ' --outdir ' +
self.get_dir())
chimeraprep_name = os.path.basename(self.get_args().chimeraprep)
self.run_external_command(chimeraprep_name, cmd_to_run, True)
# assess the result
self.end()
def test_get_celpp_challenge_data_dir_name_path_not_found(self):
temp_dir = tempfile.mkdtemp()
try:
params = D3RParameters()
task = ChallengeDataTask(temp_dir, params)
self.assertEqual(task.get_celpp_challenge_data_dir_name(),
'celpp_week0_0')
finally:
shutil.rmtree(temp_dir)
def __init__(self, path, args):
"""Constructor
"""
try:
logger.debug('ftpconfig set to ' + args.ftpconfig)
self._file_transfer = FtpFileTransfer(args.ftpconfig)
except Exception:
logger.exception('Caught exception')
self._file_transfer = None
self._path = path
self._args = args
ctask = ChallengeDataTask(path, args)
self._chall_dir_name = ctask.get_celpp_challenge_data_dir_name()
def test_get_celpp_challenge_data_dir_name(self):
temp_dir = tempfile.mkdtemp()
try:
params = D3RParameters()
cyear = os.path.join(temp_dir, '2016')
cweek = os.path.join(cyear, 'dataset.week.5')
os.mkdir(cyear)
os.mkdir(cweek)
task = ChallengeDataTask(cweek, params)
task.create_dir()
self.assertEqual(task.get_celpp_challenge_data_dir_name(),
'celpp_week5_2016')
finally:
shutil.rmtree(temp_dir)
def test_tar_challenge_dir_unable_to_write_tarfile(self):
temp_dir = tempfile.mkdtemp()
try:
params = D3RParameters()
task = ChallengeDataTask(temp_dir, params)
chall_dir = task.get_celpp_challenge_data_dir_name()
try:
task._tar_challenge_dir(chall_dir)
self.fail('Expected IOError')
except IOError:
pass
finally:
shutil.rmtree(temp_dir)
def test_tar_challenge_dir(self):
temp_dir = tempfile.mkdtemp()
try:
params = D3RParameters()
dimport = DataImportTask(temp_dir, params)
dimport.create_dir()
open(dimport.get_sequence_tsv(), 'a').close()
open(dimport.get_nonpolymer_tsv(), 'a').close()
open(dimport.get_crystalph_tsv(), 'a').close()
task = ChallengeDataTask(temp_dir, params)
task.create_dir()
chall_dir = task._create_challenge_dir()
final_log = os.path.join(chall_dir, 'final.log')
open(final_log, 'a').close()
task._create_readme(chall_dir)
task._copy_over_tsv_files(chall_dir)
# make a fake candidate
file_list = self.make_fake_candidate_dir(chall_dir, '5hib',
'2eb2', 'CSX')
self.assertEqual(os.path.isdir(chall_dir), True)
name = task.get_celpp_challenge_data_dir_name()
tfile = task._tar_challenge_dir(name)
self.assertEqual(os.path.isfile(tfile), True)
foodir = os.path.join(temp_dir, 'foo')
tar = tarfile.open(tfile, 'r:*')
tar.extractall(path=foodir)
tar.close()
cdir = os.path.join(foodir, name)
readme = os.path.join(cdir, ChallengeDataTask.README_TXT_FILE)
self.assertEqual(os.path.isfile(readme), True)
final_log = os.path.join(foodir, 'final.log')
self.assertEqual(os.path.isfile(final_log), False)
for fname in file_list:
chk = os.path.join(cdir, fname)
self.assertEqual(os.path.isfile(chk), True, chk)
finally:
shutil.rmtree(temp_dir)
def test_run_succeeds_one_evaluation_and_no_emailer(self):
temp_dir = tempfile.mkdtemp()
try:
params = D3RParameters()
params.postevaluation = 'echo'
etask = EvaluationTask(
temp_dir, 'foo_dock.' + EvaluationTaskFactory.SCORING_SUFFIX,
None, params)
etask.create_dir()
task = PostEvaluationTask(temp_dir, params)
ctask = ChallengeDataTask(temp_dir, params)
task.run()
self.assertEqual(task.get_error(), None)
self.assertEqual(task.get_status(), D3RTask.COMPLETE_STATUS)
cfile = os.path.join(task.get_dir(), D3RTask.COMPLETE_FILE)
self.assertTrue(os.path.isfile(cfile))
stdout_file = os.path.join(task.get_dir(),
'echo' + D3RTask.STDOUT_SUFFIX)
self.assertTrue(os.path.isfile(stdout_file))
f = open(stdout_file, 'r')
data = f.read()
f.close()
ctaskdir = os.path.join(ctask.get_dir(),
ctask.get_celpp_challenge_data_dir_name())
self.assertTrue(' --evaluationdir ' + etask.get_dir() in data)
self.assertTrue(' --challengedir ' + ctaskdir in data)
self.assertTrue('--stageprefix stage.7. --evaluationsuffix ' +
'.extsubmission.evaluation$|.evaluation$' in data)
stderr_file = os.path.join(task.get_dir(),
'echo' + D3RTask.STDERR_SUFFIX)
self.assertTrue(os.path.isfile(stderr_file))
finally:
shutil.rmtree(temp_dir)
def test_create_challenge_dir_unable_to_make_dir(self):
temp_dir = tempfile.mkdtemp()
try:
params = D3RParameters()
cyear = os.path.join(temp_dir, '2016')
cweek = os.path.join(cyear, 'dataset.week.5')
os.mkdir(cyear)
os.mkdir(cweek)
task = ChallengeDataTask(cweek, params)
task.create_dir()
cdir = os.path.join(task.get_dir(),
task.get_celpp_challenge_data_dir_name())
open(cdir, 'a').close()
try:
task._create_challenge_dir()
self.fail('Expected OSError')
except OSError:
pass
finally:
shutil.rmtree(temp_dir)
def test_tar_challenge_dir_nothing_but_readme(self):
temp_dir = tempfile.mkdtemp()
try:
params = D3RParameters()
task = ChallengeDataTask(temp_dir, params)
task.create_dir()
chall_dir = task._create_challenge_dir()
task._create_readme(chall_dir)
self.assertEqual(os.path.isdir(chall_dir), True)
name = task.get_celpp_challenge_data_dir_name()
tfile = task._tar_challenge_dir(name)
self.assertEqual(os.path.isfile(tfile), True)
foodir = os.path.join(temp_dir, 'foo')
tar = tarfile.open(tfile, 'r:*')
tar.extractall(path=foodir)
tar.close()
cdir = os.path.join(foodir, name)
readme = os.path.join(cdir, ChallengeDataTask.README_TXT_FILE)
self.assertEqual(os.path.isfile(readme), True)
finally:
shutil.rmtree(temp_dir)
def run(self):
"""Runs EvaluationTask after verifying dock was good
Method requires can_run() to be called before hand with
successful outcome
Otherwise method invokes D3RTask.start then this method
creates a directory and invokes evaluation script. Upon
completion results are analyzed and success or error status
is set appropriately and D3RTask.end is invoked
"""
super(EvaluationTask, self).run()
if self._can_run is False:
logger.debug(self.get_dir_name() +
' cannot run cause _can_run flag '
'is False')
return
try:
logger.debug('evaluation set to ' + self.get_args().evaluation)
except AttributeError:
self.set_error('evaluation not set')
self.end()
return
try:
logger.debug('pdbdb set to ' + self.get_args().pdbdb)
except AttributeError:
self.set_error('pdbdb not set')
self.end()
return
try:
evaltimeout = self.get_args().evaluationtimeout
logger.debug('Setting evaluation timeout to ' + str(evaltimeout))
except AttributeError:
evaltimeout = None
try:
killdelay = self.get_args().evaluationtimeoutkilldelay
logger.debug('Setting evaluation kill delay to ' + str(killdelay))
except AttributeError:
killdelay = 60
blastnfilter = BlastNFilterTask(self._path, self._args)
challenge = ChallengeDataTask(self._path, self._args)
challdir = os.path.join(challenge.get_dir(),
challenge.get_celpp_challenge_data_dir_name())
#
# --pdbdb <path to pdb.extracted> --dockdir <stage.4.glide> \
# --outdir <path to stage.5.glide.evaluation>
#
cmd_to_run = (self.get_args().evaluation + ' --pdbdb ' +
self.get_args().pdbdb + ' --dockdir ' +
self._docktask.get_dir() + ' --blastnfilterdir ' +
blastnfilter.get_dir() + ' --challengedir ' + challdir +
' --outdir ' + self.get_dir())
eval_name = os.path.basename(self.get_args().evaluation)
ecode = self.run_external_command(eval_name,
cmd_to_run,
False,
timeout=evaltimeout,
kill_delay=killdelay)
# write out evaluate exit code file
self._write_evaluate_exitcode_file(ecode)
# attempt to send evaluation email
try:
self._emailer.send_evaluation_email(self)
self.append_to_email_log(self._emailer.get_message_log())
except Exception as e:
logger.exception('Caught exception trying to send evaluation '
'email')
self.append_to_email_log('Caught exception trying to send '
'evaluation email ' + str(e) + '\n')
# assess the result
self.end()
def test_run_succeeds_one_evaluation_with_emailer(self):
temp_dir = tempfile.mkdtemp()
try:
params = D3RParameters()
foo_script = os.path.join(temp_dir, 'foo.py')
f = open(foo_script, 'w')
f.write('#! /usr/bin/env python\n')
f.write('import sys\nimport os\n')
f.write('outdir = sys.argv[1]\n')
f.write('sys.stdout.write(" ".join(sys.argv))\n')
f.write('f = open(os.path.join(outdir, "summary.txt"),"w")\n')
f.write('f.write("summary")\n')
f.write('f.flush()\nf.close()\n')
f.write('f = open(os.path.join(outdir, "yo.csv"), "a")\n')
f.write('f.write("hi")\nf.flush()\nf.close()\n')
f.write('sys.exit(0)\n')
f.flush()
f.close()
os.chmod(foo_script, stat.S_IRWXU)
params.postevaluation = foo_script
etask = EvaluationTask(
temp_dir, 'foo_dock.' + EvaluationTaskFactory.SCORING_SUFFIX,
None, params)
etask.create_dir()
task = PostEvaluationTask(temp_dir, params)
# setup emailer
smtpemailer = SmtpEmailer()
mockserver = D3RParameters()
mockserver.sendmail = Mock()
mockserver.quit = Mock()
smtpemailer.set_alternate_smtp_server(mockserver)
pee = PostEvaluationEmailer(['*****@*****.**'], smtpemailer)
task.set_evaluation_emailer(pee)
ctask = ChallengeDataTask(temp_dir, params)
task.run()
self.assertEqual(task.get_error(), None)
self.assertEqual(task.get_status(), D3RTask.COMPLETE_STATUS)
cfile = os.path.join(task.get_dir(), D3RTask.COMPLETE_FILE)
self.assertTrue(os.path.isfile(cfile))
stdout_file = os.path.join(task.get_dir(),
'foo.py' + D3RTask.STDOUT_SUFFIX)
self.assertTrue(os.path.isfile(stdout_file))
f = open(stdout_file, 'r')
data = f.read()
f.close()
ctaskdir = os.path.join(ctask.get_dir(),
ctask.get_celpp_challenge_data_dir_name())
self.assertTrue(' --evaluationdir ' + etask.get_dir() in data)
self.assertTrue(' --challengedir ' + ctaskdir in data)
self.assertTrue('--stageprefix stage.7. --evaluationsuffix ' +
'.extsubmission.evaluation$|.evaluation$' in data)
stderr_file = os.path.join(task.get_dir(),
'foo.py' + D3RTask.STDERR_SUFFIX)
f = open(stdout_file, 'r')
data = f.read()
f.close()
self.assertTrue(' --evaluationdir ' + etask.get_dir() in data)
self.assertTrue(os.path.isfile(stderr_file))
self.assertEqual(task.get_error(), None)
self.assertTrue('Sent post evaluation email to: '
'*****@*****.**' in task.get_email_log())
finally:
shutil.rmtree(temp_dir)
def run(self):
"""Runs PostEvaluationTask
Method requires can_run() to be called before hand with
successful outcome
Otherwise method invokes D3RTask.start then this method
creates a directory and invokes evaluation script. Upon
completion results are analyzed and success or error status
is set appropriately and D3RTask.end is invoked
"""
super(PostEvaluationTask, self).run()
if self._can_run is False:
logger.debug(self.get_dir_name() +
' cannot run cause _can_run flag '
'is False')
return
try:
logger.debug('postevaluation set to ' +
self.get_args().postevaluation)
except AttributeError:
self.set_error('postevaluation not set')
self.end()
return
#
# --evaluationdir <path to stage.X.evaluation>
# --evaluationdir <path to another etc...>
# --challengedir <path to challenge dir ie
# stage.4.challengedata/celpp_week10_2017>
#
chall = ChallengeDataTask(self.get_path(), self.get_args())
chall_dname = chall.get_celpp_challenge_data_dir_name()
evaldir_args = self._get_evaluationdir_args()
if evaldir_args is not '':
cmd_to_run = (self.get_args().postevaluation + ' ' +
self.get_dir() + ' ' + evaldir_args +
' --challengedir ' +
os.path.join(chall.get_dir(), chall_dname) +
' --stageprefix ' + self._eval_task_prefix_str +
' --evaluationsuffix ' + self._eval_task_suffix_str)
peval_name = os.path.basename(self.get_args().postevaluation)
self.run_external_command(peval_name, cmd_to_run, True)
else:
self.set_error('Did not find any Evaluation tasks to summarize')
# attempt to send postevaluation email
try:
self._emailer.send_postevaluation_email(self)
self.append_to_email_log(self._emailer.get_message_log())
except Exception as e:
logger.exception('Caught exception trying to send '
'postevaluation email')
self.append_to_email_log('Caught exception trying to send '
'evaluation email ' + str(e) + '\n')
# assess the result
self.end()
def test_run_succeeds_no_ftp(self):
temp_dir = tempfile.mkdtemp()
try:
script = self.create_gen_challenge_script(temp_dir)
params = D3RParameters()
params.genchallenge = script
params.pdbdb = '/foo'
params.version = '1'
blastnfilter = BlastNFilterTask(temp_dir, params)
blastnfilter.create_dir()
open(os.path.join(blastnfilter.get_dir(), D3RTask.COMPLETE_FILE),
'a').close()
chall = ChallengeDataTask(temp_dir, params)
dimport = DataImportTask(temp_dir, params)
dimport.create_dir()
ctsv = dimport.get_crystalph_tsv()
f = open(ctsv, 'w')
f.write('crystal')
f.flush()
f.close()
nonpoly = dimport.get_nonpolymer_tsv()
f = open(nonpoly, 'w')
f.write('nonpoly')
f.flush()
f.close()
seq = dimport.get_sequence_tsv()
f = open(seq, 'w')
f.write('seq')
f.flush()
f.close()
chall.run()
self.assertEqual(chall.get_error(), None)
# verify test files get created
errfile = os.path.join(chall.get_dir(),
D3RTask.ERROR_FILE)
self.assertEqual(os.path.isfile(errfile), False)
compfile = os.path.join(chall.get_dir(),
D3RTask.COMPLETE_FILE)
self.assertEqual(os.path.isfile(compfile), True)
stderr = os.path.join(chall.get_dir(),
'genchallenge.py.stderr')
self.assertEqual(os.path.isfile(stderr), True)
stdout = os.path.join(chall.get_dir(),
'genchallenge.py.stdout')
self.assertEqual(os.path.isfile(stdout), True)
# verify challenge directory is created and
# filled with valid files
chall_dir = os.path.join(chall.get_dir(),
chall.get_celpp_challenge_data_dir_name())
self.assertEqual(os.path.isdir(chall_dir), True)
readme = os.path.join(chall_dir, ChallengeDataTask.README_TXT_FILE)
self.assertEqual(os.path.isfile(readme), True)
crystal = os.path.join(chall_dir, DataImportTask.CRYSTALPH_TSV)
self.assertEqual(os.path.isfile(crystal), True)
seq = os.path.join(chall_dir, DataImportTask.SEQUENCE_TSV)
self.assertEqual(os.path.isfile(seq), True)
nonpoly = os.path.join(chall_dir, DataImportTask.NONPOLYMER_TSV)
self.assertEqual(os.path.isfile(nonpoly), True)
fivehibdir = os.path.join(chall_dir, '5hib')
fivehibtxt = os.path.join(fivehibdir, '5hib.txt')
self.assertEqual(os.path.isfile(fivehibtxt), True)
fivehibfas = os.path.join(fivehibdir, '5hib.fasta')
self.assertEqual(os.path.isfile(fivehibfas), True)
fivehicdir = os.path.join(chall_dir, '5hic')
fivehictxt = os.path.join(fivehicdir, '5hic.txt')
self.assertEqual(os.path.isfile(fivehictxt), True)
fivehicfas = os.path.join(fivehicdir, '5hic.fasta')
self.assertEqual(os.path.isfile(fivehicfas), True)
# verify tarball is created
tfile = chall.get_celpp_challenge_data_tar_file()
self.assertEqual(os.path.isfile(tfile), True)
foodir = os.path.join(temp_dir, 'foo')
os.mkdir(foodir)
tar = tarfile.open(tfile, 'r:*')
tar.extractall(path=foodir)
tar.close()
name = chall.get_celpp_challenge_data_dir_name()
cdir = os.path.join(foodir, name)
readme = os.path.join(cdir, ChallengeDataTask.README_TXT_FILE)
self.assertEqual(os.path.isfile(readme), True)
final = os.path.join(foodir, name, 'final.log')
self.assertEqual(os.path.isfile(final), False)
e_con = os.path.join(foodir, name, 'error_container')
self.assertEqual(os.path.isdir(e_con), False)
finally:
shutil.rmtree(temp_dir)