def test_upload_task(self):
task = D3RTask(None, D3RParameters())
self.assertEqual(task._file_uploader, None)
# try calling upload_task where _file_uploader is None
task._upload_task()
# try calling upload_task where _file_uploader is NOT None
params = D3RParameters()
params.ftpconfig = 'foo'
task = D3RTask(None, params)
self.assertEqual(task._file_uploader, None)
# try calling upload_task where _file_uploader is None
task._upload_task()
# try calling upload_task where _file_uploader.upload_files
# throws exception
uploader = MockFileTransfer()
uploader.set_upload_files_exception(MockException('hi'))
task = D3RTask(None, D3RParameters())
task.set_file_transfer(uploader)
task._upload_task()
# try calling where upload_summary throws exception
uploader.set_upload_files_exception(None)
uploader.set_upload_summary_exception(MockException('boo'))
task._upload_task()
# try calling where upload_summary returns None
uploader.set_upload_summary_exception(None)
uploader.set_upload_summary(None)
task._upload_task()
uploader.set_upload_summary('hello')
task._upload_task()
self.assertEqual(task._email_log, '\nhello\n')
def test_constructor(self):
params = D3RParameters()
task = D3RTask('/path', params)
self.assertEqual(task.get_name(), None)
self.assertEqual(task.get_path(), '/path')
self.assertEqual(task.get_stage(), None)
self.assertEqual(task.get_status(), D3RTask.UNKNOWN_STATUS)
self.assertEqual(task.get_error(), None)
task.set_name('foo')
task.set_path('blah')
task.set_stage(4)
task.set_status(D3RTask.START_STATUS)
task.set_error('error')
task.set_file_transfer('yah')
self.assertEqual(task.get_name(), 'foo')
self.assertEqual(task.get_path(), 'blah')
self.assertEqual(task.get_stage(), 4)
self.assertEqual(task.get_status(), D3RTask.START_STATUS)
self.assertEqual(task.get_error(), 'error')
self.assertEqual(task.get_file_transfer(), 'yah')
params.ftpconfig = '/somefile'
task = D3RTask('/path', params)
self.assertEqual(task.get_file_transfer(), None)
def test_get_program_version(self):
params = D3RParameters()
task = D3RTask(None, params)
self.assertEqual(task.get_program_version(), '')
params.version = '0.1.1'
task = D3RTask(None, params)
self.assertEqual(task.get_program_version(), '0.1.1')
def test_get_program_name(self):
params = D3RParameters()
task = D3RTask(None, params)
self.assertTrue(task._get_program_name().find('task.py') > 0,
task._get_program_name())
params.program = 'proggy'
params.version = 'versy'
task = D3RTask(None, params)
self.assertEqual(task._get_program_name(), 'proggy versy')
def test_get_mail_truncated_string_truncated(self):
params = D3RParameters()
task = D3RTask('/foo', params)
val = '123456789 abcdefghijklmnop'
res = task._get_email_truncated_string(val, 15)
self.assertEqual(res, D3RTask.TEXT_TRUNCATED_STR +
'bcdefghijklmnop')
def test_get_dir(self):
params = D3RParameters()
task = D3RTask(None, params)
try:
task.get_dir()
self.fail('Expected UnsetPathError')
except UnsetPathError:
pass
task.set_path('/blah')
try:
task.get_dir()
self.fail('Expected UnsetStageError')
except UnsetStageError:
pass
task.set_stage(1)
try:
task.get_dir()
self.fail('Expected UnsetNameError')
except UnsetNameError:
pass
task.set_name('foo')
self.assertEqual(task.get_dir(), '/blah/stage.1.foo')
def test_send_external_submission_email_sendmail_exception(self):
temp_dir = tempfile.mkdtemp()
try:
params = D3RParameters()
params.program = 'foo'
params.version = '1'
dtask = D3RTask('/foo', params)
dtask.set_name('12345' + EvaluationTask.EXT_SUBMISSION_SUFFIX)
task = EvaluationTask(temp_dir, dtask.get_name(), dtask, params)
plist = [
Participant('1name', '1d3rusername', '12345', '*****@*****.**')
]
# try single email address
smtpemailer = SmtpEmailer()
mockserver = D3RParameters()
mockserver.sendmail = Mock(side_effect=IOError('ha'))
mockserver.quit = Mock()
smtpemailer.set_alternate_smtp_server(mockserver)
emailer = EvaluationEmailer(ParticipantDatabase(plist), None)
emailer.set_alternate_smtp_emailer(smtpemailer)
emailer.send_evaluation_email(task)
mockserver.quit.assert_any_call()
self.assertEqual(
emailer.get_message_log(),
'\nCaught exception trying to email '
'participant : Caught exception ha\n')
finally:
shutil.rmtree(temp_dir)
def test_send_external_submission_email_success(self):
temp_dir = tempfile.mkdtemp()
try:
params = D3RParameters()
params.program = 'foo'
params.version = '1'
dtask = D3RTask('/foo', params)
dtask.set_name('12345' + EvaluationTask.EXT_SUBMISSION_SUFFIX)
task = EvaluationTask(temp_dir, dtask.get_name(), dtask, params)
task.create_dir()
f = open(task.get_rmsd_txt(), 'w')
f.write('hi\n')
f.flush()
f.close()
plist = [
Participant('1name', '1d3rusername', '12345', '*****@*****.**')
]
# try single email address
smtpemailer = SmtpEmailer()
mockserver = D3RParameters()
mockserver.sendmail = Mock()
mockserver.quit = Mock()
smtpemailer.set_alternate_smtp_server(mockserver)
emailer = EvaluationEmailer(ParticipantDatabase(plist), None)
emailer.set_alternate_smtp_emailer(smtpemailer)
emailer.send_evaluation_email(task)
mockserver.quit.assert_any_call()
self.assertEqual(emailer.get_message_log(),
'\nSent evaluation email to: [email protected]\n')
self.assertEqual(mockserver.sendmail.call_count, 1)
finally:
shutil.rmtree(temp_dir)
def test_end(self):
temp_dir = tempfile.mkdtemp()
try:
params = D3RParameters()
task = D3RTask(temp_dir, params)
task.set_stage(1)
task.set_name('foo')
task.start()
task.end()
self.assertEqual(
os.path.isfile(
os.path.join(task.get_dir(), D3RTask.COMPLETE_FILE)), True)
self.assertEqual(task.get_error(), None)
self.assertEqual(task.get_status(), D3RTask.COMPLETE_STATUS)
task.set_error('some error')
task.end()
self.assertEqual(task.get_error(), 'some error')
self.assertEqual(task.get_status(), D3RTask.ERROR_STATUS)
err_file = os.path.join(task.get_dir(), D3RTask.ERROR_FILE)
self.assertEqual(os.path.isfile(err_file), True)
f = open(err_file, 'r')
err_msg = f.readline()
self.assertEqual(err_msg, 'some error\n')
task.set_status(D3RTask.ERROR_STATUS)
os.remove(err_file)
task.set_error(None)
task.end()
self.assertEqual(task.get_status(), D3RTask.ERROR_STATUS)
self.assertEqual(os.path.isfile(err_file), True)
self.assertEqual(os.path.getsize(err_file), 0)
finally:
shutil.rmtree(temp_dir)
def test_get_external_submitter_email_valid(self):
params = D3RParameters()
dtask = D3RTask('/foo', params)
dtask.set_name('12345' + EvaluationTask.EXT_SUBMISSION_SUFFIX)
task = EvaluationTask('/foo', dtask.get_name(), dtask, params)
plist = [Participant('1name', '1d3rusername', '12345', '*****@*****.**')]
# try single email address
emailer = EvaluationEmailer(ParticipantDatabase(plist), None)
emails = emailer._get_external_submitter_email(task)
self.assertEqual(emails[0], '*****@*****.**')
self.assertEqual(len(emails), 1)
self.assertEqual(emailer.get_message_log(), None)
# try multiple email address
plist = [
Participant('1name', '1d3rusername', '12345',
'[email protected],[email protected]')
]
# try single email address
emailer = EvaluationEmailer(ParticipantDatabase(plist), None)
emails = emailer._get_external_submitter_email(task)
self.assertEqual(emails[0], '*****@*****.**')
self.assertEqual(emails[1], '*****@*****.**')
self.assertEqual(len(emails), 2)
self.assertEqual(emailer.get_message_log(), None)
def test_run_external_command_withtimeout_cmd_succeeds(self):
temp_dir = tempfile.mkdtemp()
try:
params = D3RParameters()
task = D3RTask(None, params)
task.set_name('foo')
task.set_stage(1)
task.set_path(temp_dir)
task.create_dir()
task.set_path(temp_dir)
self.assertEquals(
0, task.run_external_command('hi',
'echo hi',
False,
timeout=30))
self.assertEquals(task.get_error(), None)
self.assertEquals(task.get_status(), D3RTask.UNKNOWN_STATUS)
self.assertEquals(
os.path.exists(os.path.join(task.get_dir(), 'hi.stdout')),
True)
self.assertEquals(
os.path.exists(os.path.join(task.get_dir(), 'hi.stderr')),
True)
finally:
shutil.rmtree(temp_dir)
def test_generate_external_submission_email_body(self):
temp_dir = tempfile.mkdtemp()
try:
params = D3RParameters()
dtask = D3RTask('/foo', params)
dtask.set_name('foo')
task = EvaluationTask(temp_dir, dtask.get_name(), dtask, params)
task.create_dir()
emailer = EvaluationEmailer(None, None)
task.set_evaluation_emailer(emailer)
rmsd = task.get_rmsd_txt()
f = open(rmsd, 'w')
f.write(' LMCSS XXX\n1fcz 0.2 0.4\n')
f.flush()
f.close()
subject, body = emailer\
._generate_external_submission_email_body(task)
self.assertEqual(subject,
'[d3rcelpp] Week 0 evaluation results for foo')
self.assertEqual(
body, 'Dear CELPP Participant,\n\nHere are your '
'docking evaluation results '
'(RMSD, Angstroms) '
'for CELPP week '
'0\n\n\nEvaluation of docking'
'\n=====================\n LMCSS XXX\n'
'1fcz 0.2 0.4\n\n\n\nSincerely,\n\n'
'CELPP Automation')
finally:
shutil.rmtree(temp_dir)
def test_run_succeeds_no_emailer(self):
temp_dir = tempfile.mkdtemp()
try:
params = D3RParameters()
params.evaluation = 'true'
params.pdbdb = '/data/pdb'
docktask = D3RTask(temp_dir, params)
docktask.set_name('foo')
docktask.set_stage(EvaluationTaskFactory.DOCKSTAGE)
docktask.create_dir()
open(os.path.join(docktask.get_dir(), D3RTask.COMPLETE_FILE),
'a').close()
evaluation = EvaluationTask(temp_dir, 'foo.evaluation', docktask,
params)
evaluation.run()
self.assertEqual(evaluation.get_error(), None)
# test files get created
errfile = os.path.join(evaluation.get_dir(), D3RTask.ERROR_FILE)
self.assertEqual(os.path.isfile(errfile), False)
compfile = os.path.join(evaluation.get_dir(),
D3RTask.COMPLETE_FILE)
self.assertEqual(os.path.isfile(compfile), True)
stderr = os.path.join(evaluation.get_dir(), 'true.stderr')
self.assertEqual(os.path.isfile(stderr), True)
stdout = os.path.join(evaluation.get_dir(), 'true.stdout')
self.assertEqual(os.path.isfile(stdout), True)
finally:
shutil.rmtree(temp_dir)
def test_run_fails_cause_evaluation_is_not_found(self):
temp_dir = tempfile.mkdtemp()
try:
params = D3RParameters()
params.evaluation = '/bin/doesnotexist'
params.pdbdb = '/data/pdb'
docktask = D3RTask(temp_dir, params)
docktask.set_name('foo')
docktask.set_stage(EvaluationTaskFactory.DOCKSTAGE)
docktask.create_dir()
open(os.path.join(docktask.get_dir(), D3RTask.COMPLETE_FILE),
'a').close()
evaluation = EvaluationTask(temp_dir, 'foo.evaluation', docktask,
params)
evaluation.run()
self.assertEqual(
evaluation.get_error(), 'Caught Exception trying to run ' +
'/bin/doesnotexist --pdbdb /data/pdb ' + '--dockdir ' +
docktask.get_dir() + ' --outdir ' + evaluation.get_dir() +
' : [Errno 2] No such file or directory')
# test files get created
errfile = os.path.join(evaluation.get_dir(), D3RTask.ERROR_FILE)
self.assertEqual(os.path.isfile(errfile), True)
finally:
shutil.rmtree(temp_dir)
def test_run_fails_cause_evaluation_fails(self):
temp_dir = tempfile.mkdtemp()
try:
params = D3RParameters()
params.evaluation = 'false'
params.pdbdb = '/data/pdb'
docktask = D3RTask(temp_dir, params)
docktask.set_name('foo')
docktask.set_stage(EvaluationTaskFactory.DOCKSTAGE)
docktask.create_dir()
open(os.path.join(docktask.get_dir(), D3RTask.COMPLETE_FILE),
'a').close()
evaluation = EvaluationTask(temp_dir, 'foo.evaluation', docktask,
params)
evaluation.run()
self.assertEqual(
evaluation.get_error(),
'Non zero exit code: 1 received. Standard out: ' +
' Standard error: ')
# test file gets created
errfile = os.path.join(evaluation.get_dir(), D3RTask.ERROR_FILE)
self.assertEqual(os.path.isfile(errfile), True)
stderr = os.path.join(evaluation.get_dir(), 'false.stderr')
self.assertEqual(os.path.isfile(stderr), True)
stdout = os.path.join(evaluation.get_dir(), 'false.stdout')
self.assertEqual(os.path.isfile(stdout), True)
finally:
shutil.rmtree(temp_dir)
def test_send_email_where_email_is_none(self):
params = D3RParameters()
params.email = None
task = D3RTask('/foo', params)
task.set_stage(1)
task.set_name('foo')
task._send_end_email()
def test_start_with_version(self):
temp_dir = tempfile.mkdtemp()
try:
params = D3RParameters()
params.version = '1.6.0'
task = D3RTask(temp_dir, params)
task.set_stage(1)
task.set_name('foo')
task.start()
sfile = os.path.join(task.get_dir(), D3RTask.START_FILE)
self.assertEqual(os.path.isfile(sfile), True)
f = open(sfile, 'r')
self.assertEqual(f.read(), '1.6.0')
self.assertEqual(task.get_error(), None)
self.assertEqual(task.get_status(), D3RTask.START_STATUS)
task.start()
self.assertNotEqual(task.get_error(), None)
self.assertEqual(task.get_status(), D3RTask.ERROR_STATUS)
self.assertEqual(
os.path.isfile(os.path.join(task.get_dir(),
D3RTask.ERROR_FILE)), True)
finally:
shutil.rmtree(temp_dir)
def test_run_external_command_cmd_raises_exception(self):
temp_dir = tempfile.mkdtemp()
try:
params = D3RParameters()
task = D3RTask(None, params)
task.set_name('foo')
task.set_stage(1)
task.set_path(temp_dir)
task.create_dir()
task.set_path(temp_dir)
self.assertEquals(
1, task.run_external_command('hi', 'asdfasdf', None))
self.assertEquals(
task.get_error(), 'Caught Exception trying ' +
'to run asdfasdf : [Errno 2] No such file ' + 'or directory')
self.assertEquals(
os.path.exists(os.path.join(task.get_dir(), 'hi.stdout')),
False)
self.assertEquals(
os.path.exists(os.path.join(task.get_dir(), 'hi.stderr')),
False)
finally:
shutil.rmtree(temp_dir)
def test_write_to_file(self):
tempDir = tempfile.mkdtemp()
try:
params = D3RParameters()
task = D3RTask(None, params)
try:
task.write_to_file('hello', None)
self.fail('Expected UnsetFileNameError')
except UnsetFileNameError:
pass
try:
task.write_to_file('hello', 'foo')
self.fail('Expected UnsetPathError')
except UnsetPathError:
pass
task.set_name('foo')
task.set_stage(1)
task.set_path(tempDir)
try:
task.write_to_file('hello', 'foo')
self.fail('Expected IOError')
except IOError:
pass
task.create_dir()
task.write_to_file('hello', 'foo')
self.assertEqual(
os.path.isfile(os.path.join(task.get_dir(), 'foo')), True)
finally:
shutil.rmtree(tempDir)
def test_update_status_from_filesystem(self):
params = D3RParameters()
task = D3RTask(None, params)
try:
task.update_status_from_filesystem()
self.fail("Expected UnsetPathError")
except UnsetPathError:
pass
tempDir = tempfile.mkdtemp()
try:
task.set_path(tempDir)
# Test unset stage
try:
task.update_status_from_filesystem()
self.fail("Expected UnsetStageError")
except UnsetStageError:
pass
finally:
shutil.rmtree(tempDir)
task.set_stage(1)
task.set_name('foo')
try_update_status_from_filesystem(self, task)
def test_is_external_submission(self):
params = D3RParameters()
dtask = D3RTask('/ha', params)
task = EvaluationTask('/ha', 'foo', dtask, params)
dtask.set_name('blah')
self.assertEqual(task.is_external_submission(), False)
dtask.set_name('blah' + EvaluationTask.EXT_SUBMISSION_SUFFIX)
self.assertEqual(task.is_external_submission(), True)
def test_get_mail_truncated_string_maxchars_none_or_negative(self):
params = D3RParameters()
task = D3RTask('/foo', params)
self.assertEqual(task._get_email_truncated_string('foo',
None), 'foo')
self.assertEqual(task._get_email_truncated_string('foo',
-1), 'foo')
def test_run_external_command_name_cmd_to_run_None(self):
params = D3RParameters()
task = D3RTask(None, params)
try:
task.run_external_command('hello', None, None)
self.fail('expected UnsetCommandError')
except UnsetCommandError:
pass
def test_run_external_command_name_None(self):
params = D3RParameters()
task = D3RTask(None, params)
try:
task.run_external_command(None, 'true', None)
self.fail('expected UnsetNameError')
except UnsetNameError:
pass
def get_evaluation_tasks(self):
"""Generate EvaluationTasks
This method examines the path directory
set via the constructor or set_path() method
for all stage 4 tasks excluding tasks
that end with 'webdata' A EvaluationTask
object is created for each of these tasks
and returned in a list.
:return: list of EvaluationTask objects or empty list if none found
"""
path = self.get_path()
logger.debug('Examining ' + path + ' for docking tasks')
if not os.path.isdir(path):
raise PathNotDirectoryError(path + ' is not a directory')
scoring_tasks = []
path_list = os.listdir(path)
participant_db = self._get_participant_database()
efac = SmtpEmailerFactory(self.get_args())
emailer = EvaluationEmailer(participant_db, efac.get_smtp_emailer())
for entry in path_list:
logger.debug('Checking if ' + entry + ' is a docking task')
full_path = os.path.join(path, entry)
if os.path.isdir(full_path):
if entry.startswith(EvaluationTaskFactory.DOCKSTAGE_PREFIX):
if entry.endswith(EvaluationTaskFactory.WEB_DATA_SUFFIX):
logger.debug('Skipping ' + entry + ' due to suffix')
continue
# we have a valid docking path
docktask = D3RTask(path, self.get_args())
docktask.set_stage(EvaluationTaskFactory.DOCKSTAGE)
docktask.set_name(
entry[len(EvaluationTaskFactory.DOCKSTAGE_PREFIX):])
stask = EvaluationTask(
path,
docktask.get_name() + '.' +
EvaluationTaskFactory.SCORING_SUFFIX, docktask,
self.get_args())
if stask.can_run():
logger.debug('Adding task ' + stask.get_name())
stask.set_evaluation_emailer(emailer)
scoring_tasks.append(stask)
else:
if stask.get_error() is None:
logger.debug(stask.get_name() + ' cannot be' +
' added, no error though')
else:
logger.debug(stask.get_name() + ' cannot be' +
' added : ' + stask.get_error())
return self._sort_tasks_by_participant_priority(
scoring_tasks, participant_db)
def test_evaluationtask_constructor(self):
params = D3RParameters()
# no dock task found so it cannot run
docktask = D3RTask('/blah', params)
docktask.set_name('foo')
docktask.set_stage(EvaluationTaskFactory.DOCKSTAGE)
evaluation = EvaluationTask('/blah', 'foo.evaluation', docktask,
params)
self.assertEquals(evaluation.get_name(), 'foo.evaluation')
self.assertEquals(evaluation.get_stage(), 7)
def test_get_external_submitter_email_no_participant_email(self):
params = D3RParameters()
dtask = D3RTask('/foo', params)
dtask.set_name('444' + EvaluationTask.EXT_SUBMISSION_SUFFIX)
task = EvaluationTask('/foo', dtask.get_name(), dtask, params)
plist = [Participant('1name', '1d3rusername', '444', None)]
emailer = EvaluationEmailer(ParticipantDatabase(plist), None)
self.assertEqual(emailer._get_external_submitter_email(task), None)
self.assertEqual(emailer.get_message_log(),
'\nEmail not set for participant\n')
def test_send_evaluation_email_no_database(self):
params = D3RParameters()
dtask = D3RTask('/foo', params)
dtask.set_name('444' + EvaluationTask.EXT_SUBMISSION_SUFFIX)
task = EvaluationTask('/foo', dtask.get_name(), dtask, params)
emailer = EvaluationEmailer(None, None)
emailer.send_evaluation_email(task)
self.assertEqual(
emailer.get_message_log(),
'\nParticipant database is None cannot send docking '
'evaluation email!!!\n')
def test_send_end_email_throws_exception(self):
temp_dir = tempfile.mkdtemp()
try:
params = D3RParameters()
params.smtp = 'doesnotexistxxx.x.x'
params.smtpport = 12345
params.email = '*****@*****.**'
task = D3RTask(temp_dir, params)
task.set_stage(1)
task.set_name('foo')
task._send_end_email()
finally:
shutil.rmtree(temp_dir)
def test_get_uploadable_files(self):
task = D3RTask(None, D3RParameters())
self.assertEqual(task.get_uploadable_files(), [])
temp_dir = tempfile.mkdtemp()
try:
task = D3RTask(temp_dir, D3RParameters())
task.set_stage(1)
task.set_name('foo')
task.create_dir()
self.assertEqual(task.get_uploadable_files(), [])
# add error file
err_file = os.path.join(task.get_dir(), D3RTask.ERROR_FILE)
open(err_file, 'a').close()
flist = task.get_uploadable_files()
self.assertEqual(len(flist), 1)
self.assertEqual(flist[0], err_file)
# add stderr file
foostderr = os.path.join(task.get_dir(), 'foo.stderr')
open(foostderr, 'a').close()
flist = task.get_uploadable_files()
flist.index(err_file)
flist.index(foostderr)
self.assertEqual(len(flist), 2)
# add stdout file
foostdout = os.path.join(task.get_dir(), 'foo.stdout')
open(foostdout, 'a').close()
flist = task.get_uploadable_files()
self.assertEqual(len(flist), 3)
flist.index(err_file)
flist.index(foostderr)
flist.index(foostdout)
finally:
shutil.rmtree(temp_dir)
