def test_get_external_data_submissions_no_chall_packages(self):
temp_dir = tempfile.mkdtemp()
try:
year = os.path.join(temp_dir, '2016')
week = os.path.join(year, 'dataset.week.13')
os.makedirs(week)
dr = ExternalDataSubmissionFactory.DOCKEDRESULTS
params = D3RParameters()
fac = ExternalDataSubmissionFactory(week, params)
mockft = D3RParameters()
mockft.connect = Mock()
mockft.disconnect = Mock()
mockft.get_remote_submission_dir = Mock(return_value='/remote')
mockft.list_dirs = Mock(return_value=['yo'])
mockft.list_files = Mock(return_value=['hi',
'celpp_week13_2015' + dr +
'dname.tar.gz',
'celpp_week13_2016' + dr +
'yuck.tar.gz'])
fac.set_file_transfer(mockft)
tlist = fac.get_external_data_submissions()
self.assertEqual(len(tlist), 0)
mockft.list_dirs.assert_called_with('/remote')
mockft.list_files.assert_called_with('/remote/yo')
finally:
shutil.rmtree(temp_dir)
def test_get_external_data_submissions_two_extdatasubmission(self):
temp_dir = tempfile.mkdtemp()
try:
year = os.path.join(temp_dir, '2017')
week = os.path.join(year, 'dataset.week.13')
os.makedirs(week)
dr = ExternalDataSubmissionFactory.DOCKEDRESULTS
params = D3RParameters()
fac = ExternalDataSubmissionFactory(week, params)
mockft = D3RParameters()
mockft.connect = Mock()
mockft.delete_file = Mock(return_value=True)
mockft.get_remote_challenge_dir = Mock(return_value='/chall')
mockft.get_remote_submission_dir = Mock(return_value='/remote')
mockft.list_dirs = Mock(return_value=['yo', 'yuck'])
mockft.list_files = Mock(return_value=['hi',
'celpp_week13_2017' + dr +
'yo.tar.gz',
'celpp_week13_2017' + dr +
'yuck.tar.gz'])
fac.set_file_transfer(mockft)
tlist = fac.get_external_data_submissions()
self.assertEqual(len(tlist), 2)
self.assertEqual(tlist[0].get_name(), 'yo.extsubmission')
self.assertEqual(tlist[0].get_remote_challenge_data_package(),
'/remote/yo/celpp_week13_2017' + dr + 'yo.tar.gz')
self.assertEqual(tlist[1].get_name(), 'yuck.extsubmission')
self.assertEqual(tlist[1].get_remote_challenge_data_package(),
'/remote/yuck/celpp_week13_2017' + dr +
'yuck.tar.gz')
mockft.list_dirs.assert_called_with('/remote')
mockft.list_files.assert_called_with('/remote/yuck')
mockft.delete_file.assert_called_with('/chall/latest.txt')
finally:
shutil.rmtree(temp_dir)
def test_get_submission_dirs_list_is_none(self):
params = D3RParameters()
mockft = D3RParameters()
mockft.list_dirs = Mock(return_value=None)
fac = ExternalDataSubmissionFactory('/foo', params)
fac.set_file_transfer(mockft)
self.assertEqual(len(fac._get_submission_dirs('ha')), 0)
mockft.list_dirs.assert_called_with('ha')
def test_get_submission_dirs_filetransfer_is_none(self):
params = D3RParameters()
try:
fac = ExternalDataSubmissionFactory('/foo', params)
fac._get_submission_dirs('ha')
self.fail('Expected AttributeError')
except AttributeError:
pass
def test_get_challenge_data_package_file_raise_exception(self):
params = D3RParameters()
fac = ExternalDataSubmissionFactory('/foo', params)
try:
fac._get_challenge_data_package_file('/remote', 'dname')
self.fail('Expected AttributeError')
except AttributeError:
pass
def test_get_external_data_submissions_no_dirs(self):
params = D3RParameters()
fac = ExternalDataSubmissionFactory('/foo', params)
mockft = D3RParameters()
mockft.list_dirs = Mock(return_value=[])
fac.set_file_transfer(mockft)
tlist = fac.get_external_data_submissions()
self.assertEqual(len(tlist), 0)
def test_get_challenge_data_package_file_no_files(self):
params = D3RParameters()
mockft = D3RParameters()
mockft.list_files = Mock(return_value=[])
fac = ExternalDataSubmissionFactory('/foo', params)
fac.set_file_transfer(mockft)
dlist = fac._get_challenge_data_package_file('/remote', 'dname')
self.assertEqual(dlist, None)
mockft.list_files.assert_called_with('/remote/dname')
def test_get_submission_dirs_one_dir(self):
params = D3RParameters()
mockft = D3RParameters()
mockft.list_dirs = Mock(return_value=['somedir'])
fac = ExternalDataSubmissionFactory('/foo', params)
fac.set_file_transfer(mockft)
dlist = fac._get_submission_dirs('ha')
self.assertEqual(dlist[0], 'somedir')
mockft.list_dirs.assert_called_with('ha')
def test_externaldatasubmissionfactory_getter_setter(self):
params = D3RParameters()
params.ha = 'hi'
fac = ExternalDataSubmissionFactory('/foo', params)
self.assertEqual(fac.get_file_transfer(), None)
self.assertEqual(fac.get_path(), '/foo')
self.assertEqual(fac.get_args().ha, 'hi')
fac.set_file_transfer('yo')
self.assertEqual(fac.get_file_transfer(), 'yo')
def test_externaldatasubmissionfactory_ftpconfig_set(self):
temp_dir = tempfile.mkdtemp()
try:
figfile = os.path.join(temp_dir, 'foo')
f = open(figfile, 'w')
f.write('host blah.blah.com\n')
f.write('user bob\n')
f.write('pass ha\n')
f.write('path /celppweekly/ha\n')
f.write('challengepath /celppweekly/ha\n')
f.flush()
f.close()
params = D3RParameters()
params.ftpconfig = figfile
fac = ExternalDataSubmissionFactory('/foo', params)
self.assertTrue(fac.get_file_transfer() is not None)
finally:
shutil.rmtree(temp_dir)
def test_get_challenge_data_package_file_no_match(self):
temp_dir = tempfile.mkdtemp()
try:
dr = ExternalDataSubmissionFactory.DOCKEDRESULTS
year = os.path.join(temp_dir, '2016')
week = os.path.join(year, 'dataset.week.13')
os.makedirs(week)
params = D3RParameters()
mockft = D3RParameters()
mockft.list_files = Mock(return_value=['hi', 'celpp_week13_2015_' +
dr + '_dname.tar.gz',
'celpp_week13_2016' + dr +
'yuck.tar.gz'])
fac = ExternalDataSubmissionFactory(week, params)
fac.set_file_transfer(mockft)
dlist = fac._get_challenge_data_package_file('/remote', 'dname')
self.assertEqual(dlist, None)
mockft.list_files.assert_called_with('/remote/dname')
finally:
shutil.rmtree(temp_dir)
def test_get_challenge_package_results_file_name(self):
params = D3RParameters()
# test under non existant dir
fac = ExternalDataSubmissionFactory('/foo', params)
self.assertEqual(fac._get_challenge_package_results_file_name('xxx'),
'celpp_week0_0_dockedresults_xxx.tar.gz')
# test under 2016 week 40
temp_dir = tempfile.mkdtemp()
try:
year = os.path.join(temp_dir, '2016')
week = os.path.join(year, util.DATA_SET_WEEK_PREFIX + '40')
os.makedirs(week)
fac = ExternalDataSubmissionFactory(week, params)
rfname = fac._get_challenge_package_results_file_name('xxx')
self.assertEqual(rfname,
'celpp_week40_2016_dockedresults_xxx.tar.gz')
finally:
shutil.rmtree(temp_dir)
def test_externaldatasubmissionfactory_noftpconfig(self):
params = D3RParameters()
fac = ExternalDataSubmissionFactory('/foo', params)
self.assertEqual(fac.get_file_transfer(), None)
def test_get_external_data_submissions_raise_exception(self):
params = D3RParameters()
fac = ExternalDataSubmissionFactory('/foo', params)
tlist = fac.get_external_data_submissions()
self.assertEqual(len(tlist), 0)
def get_task_list_for_stage(theargs, stage_name):
"""Factory method that generates a list of tasks for given stage
Using stage_name get the list of tasks that need to
be run.
:param theargs: parameters set via commandline along with
``theargs.latest_weekly`` which should be set to
to base directory where stages will be run
:param stage_name: Name of stage to run
"""
if stage_name is None:
raise NotImplementedError('stage_name is None')
task_list = []
logger.debug('Getting task list for ' + stage_name)
if stage_name == CREATE_CHALLENGE:
task_list.append(MakeBlastDBTask(theargs.latest_weekly, theargs))
task_list.append(DataImportTask(theargs.latest_weekly, theargs))
task_list.append(BlastNFilterTask(theargs.latest_weekly, theargs))
task_list.append(ChallengeDataTask(theargs.latest_weekly, theargs))
if stage_name == 'makedb':
task_list.append(MakeBlastDBTask(theargs.latest_weekly, theargs))
if stage_name == 'import':
task_list.append(DataImportTask(theargs.latest_weekly, theargs))
if stage_name == 'blast':
task_list.append(BlastNFilterTask(theargs.latest_weekly, theargs))
if stage_name == 'challengedata':
task_list.append(ChallengeDataTask(theargs.latest_weekly, theargs))
if stage_name == 'proteinligprep':
task_list.append(ProteinLigPrepTask(theargs.latest_weekly, theargs))
if stage_name == 'glide':
task_list.append(GlideTask(theargs.latest_weekly, theargs))
if stage_name == 'vina':
task_list.append(AutoDockVinaTask(theargs.latest_weekly, theargs))
if stage_name == CHIMERA_PREP:
task_list.append(
ChimeraProteinLigPrepTask(theargs.latest_weekly, theargs))
if stage_name == 'extsubmission':
extfac = ExternalDataSubmissionFactory(theargs.latest_weekly, theargs)
task_list.extend(extfac.get_external_data_submissions())
if stage_name == 'evaluation':
# use util function call to get all evaluation tasks
# append them to the task_list
eval_task_factory = EvaluationTaskFactory(theargs.latest_weekly,
theargs)
task_list.extend(eval_task_factory.get_evaluation_tasks())
if len(task_list) is 0:
raise NotImplementedError('uh oh no tasks for ' + stage_name +
' stage')
return task_list