def test_run_where_everything_is_successful(self):
temp_dir = tempfile.mkdtemp()
try:
params = D3RParameters()
fakegz = os.path.join(temp_dir, 'fake.gz')
f = gzip.open(fakegz, 'wb')
f.write('hello\n')
f.flush()
f.close()
params.pdbsequrl = 'file://' + fakegz
params.makeblastdb = 'echo'
task = MakeBlastDBTask(temp_dir, params)
task._retrysleep = 0
task._maxretries = 1
task.run()
self.assertEqual(task.get_error(), None)
# check echo.stdout file for valid arguments
f = open(os.path.join(task.get_dir(), 'echo.stdout'), 'r')
line = f.readline()
self.assertEqual(
line, '-in ' + task.get_pdb_seqres_txt() + ' -out ' +
os.path.join(task.get_dir(), 'pdb_db') + ' -dbtype prot\n')
f.close()
lines = task.get_email_log().split('\n')
self.assertEqual(lines[2], '# sequence(s): 0')
f.close()
finally:
shutil.rmtree(temp_dir)
def test_run_all_compinchi_fail(self):
temp_dir = tempfile.mkdtemp()
try:
params = D3RParameters()
params.pdbfileurl = 'file://' + temp_dir
params.compinchi = 'file://' + temp_dir
make_blast = MakeBlastDBTask(temp_dir, params)
make_blast.create_dir()
open(os.path.join(make_blast.get_dir(), D3RTask.COMPLETE_FILE),
'a').close()
task = DataImportTask(temp_dir, params)
task._retrysleep = 0
open(os.path.join(temp_dir, task.NONPOLYMER_TSV), 'a').close()
open(os.path.join(temp_dir, task.SEQUENCE_TSV), 'a').close()
open(os.path.join(temp_dir, task.OLDSEQUENCE_TSV), 'a').close()
open(os.path.join(temp_dir, task.CRYSTALPH_TSV), 'a').close()
task.run()
self.assertEquals(
task.get_error(), 'Unable to download file ' + 'from ' +
params.compinchi + ' to ' + task.get_components_inchi_file())
finally:
shutil.rmtree(temp_dir)
def test_can_run_does_not_exist_or_error(self):
temp_dir = tempfile.mkdtemp()
try:
params = D3RParameters()
task = DataImportTask(temp_dir, params)
# no make blast db
self.assertEquals(task.can_run(), False)
self.assertEquals(task.get_error(),
'makeblastdb task has notfound status')
self.assertEquals(task._can_run, False)
make_blast = MakeBlastDBTask(temp_dir, params)
make_blast.create_dir()
# make blast db failed
err_file = os.path.join(make_blast.get_dir(), D3RTask.ERROR_FILE)
open(err_file, 'a').close()
self.assertEquals(task.can_run(), False)
self.assertEquals(task.get_error(),
'makeblastdb task has error status')
self.assertEquals(task._can_run, False)
os.remove(err_file)
# make blast db success
open(os.path.join(make_blast.get_dir(), D3RTask.COMPLETE_FILE),
'a').close()
self.assertEquals(task.can_run(), True)
self.assertEquals(task.get_error(), None)
self.assertEquals(task._can_run, True)
task.create_dir()
open(os.path.join(task.get_dir(), D3RTask.ERROR_FILE), 'a').close()
self.assertEquals(task.can_run(), False)
self.assertEquals(task._can_run, False)
self.assertEquals(
task.get_error(),
task.get_dir_name() + ' already exists and ' + 'status is ' +
D3RTask.ERROR_STATUS)
finally:
shutil.rmtree(temp_dir)
def test_can_run_where_task_failed(self):
temp_dir = tempfile.mkdtemp()
try:
params = D3RParameters()
task = MakeBlastDBTask(temp_dir, params)
task.create_dir()
open(os.path.join(task.get_dir(), 'error'), 'a').close()
self.assertEqual(task.can_run(), False)
finally:
shutil.rmtree(temp_dir)
def test_get_uploadable_files(self):
temp_dir = tempfile.mkdtemp()
try:
params = D3RParameters()
task = MakeBlastDBTask(temp_dir, params)
task.create_dir()
# test on empty dir
self.assertEqual(task.get_uploadable_files(), [])
# test with stderr/stdout files
stdout = os.path.join(task.get_dir(), 'makeblastdb.stdout')
open(stdout, 'a').close()
stderr = os.path.join(task.get_dir(), 'makeblastdb.stderr')
open(stderr, 'a').close()
flist = task.get_uploadable_files()
self.assertEqual(len(flist), 2)
flist.index(stdout)
flist.index(stderr)
# test with pdb_seqres.txt.gz file
open(task.get_pdb_seqres_txt_gz(), 'a').close()
flist = task.get_uploadable_files()
self.assertEqual(len(flist), 3)
flist.index(stdout)
flist.index(stderr)
flist.index(task.get_pdb_seqres_txt_gz())
os.unlink(task.get_pdb_seqres_txt_gz())
# test where pdb_seqres.txt is a dir unlikely but why not check
os.makedirs(task.get_pdb_seqres_txt_gz())
flist = task.get_uploadable_files()
self.assertEqual(len(flist), 2)
flist.index(stdout)
flist.index(stderr)
finally:
shutil.rmtree(temp_dir)
def test_run_all_success(self):
temp_dir = tempfile.mkdtemp()
try:
fakeftp = FtpFileTransfer(None)
mftp = D3RParameters()
fakeftp.set_connection(mftp)
fakeftp.set_remote_dir('/foo2')
mftp.get = Mock()
params = D3RParameters()
params.pdbfileurl = 'file://' + temp_dir
params.compinchi = 'file://' + temp_dir
make_blast = MakeBlastDBTask(temp_dir, params)
make_blast.create_dir()
open(os.path.join(make_blast.get_dir(), D3RTask.COMPLETE_FILE),
'a').close()
task = DataImportTask(temp_dir, params)
task.set_file_transfer(fakeftp)
task._retrysleep = 0
open(os.path.join(temp_dir, task.NONPOLYMER_TSV), 'a').close()
open(os.path.join(temp_dir, task.SEQUENCE_TSV), 'a').close()
open(os.path.join(temp_dir, task.OLDSEQUENCE_TSV), 'a').close()
open(os.path.join(temp_dir, task.CRYSTALPH_TSV), 'a').close()
open(os.path.join(temp_dir, task.COMPINCHI_ICH), 'a').close()
task.run()
self.assertEquals(task.get_error(), None)
# check line count is 1 now which indicates
# standard was added
self.assertEqual(
util.get_file_line_count(task.get_nonpolymer_tsv()), 1)
self.assertEqual(util.get_file_line_count(task.get_sequence_tsv()),
1)
self.assertEqual(
util.get_file_line_count(task.get_oldsequence_tsv()), 1)
self.assertEqual(
util.get_file_line_count(task.get_crystalph_tsv()), 1)
mftp.get.assert_called_with('/foo2/' +
DataImportTask.PARTICIPANT_LIST_CSV,
local=task.get_participant_list_csv())
finally:
shutil.rmtree(temp_dir)
def test_run_all_nonpolymer_fail(self):
temp_dir = tempfile.mkdtemp()
try:
params = D3RParameters()
params.pdbfileurl = 'file://' + temp_dir
params.compinchi = 'file://' + temp_dir
make_blast = MakeBlastDBTask(temp_dir, params)
make_blast.create_dir()
open(os.path.join(make_blast.get_dir(), D3RTask.COMPLETE_FILE),
'a').close()
task = DataImportTask(temp_dir, params)
task._retrysleep = 0
task.run()
self.assertEquals(
task.get_error(), 'Unable to download file ' + 'from ' +
params.pdbfileurl + ' to ' + task.get_nonpolymer_tsv())
finally:
shutil.rmtree(temp_dir)
def test_run_all_success_except_participant_download_fails(self):
temp_dir = tempfile.mkdtemp()
try:
params = D3RParameters()
params.pdbfileurl = 'file://' + temp_dir
params.compinchi = 'file://' + temp_dir
make_blast = MakeBlastDBTask(temp_dir, params)
make_blast.create_dir()
open(os.path.join(make_blast.get_dir(), D3RTask.COMPLETE_FILE),
'a').close()
task = DataImportTask(temp_dir, params)
task._retrysleep = 0
open(os.path.join(temp_dir, task.NONPOLYMER_TSV), 'a').close()
open(os.path.join(temp_dir, task.SEQUENCE_TSV), 'a').close()
open(os.path.join(temp_dir, task.OLDSEQUENCE_TSV), 'a').close()
open(os.path.join(temp_dir, task.CRYSTALPH_TSV), 'a').close()
open(os.path.join(temp_dir, task.COMPINCHI_ICH), 'a').close()
task.run()
self.assertEquals(task.get_error(), None)
# check line count is 1 now which indicates
# standard was added
self.assertEqual(
util.get_file_line_count(task.get_nonpolymer_tsv()), 1)
self.assertEqual(util.get_file_line_count(task.get_sequence_tsv()),
1)
self.assertEqual(
util.get_file_line_count(task.get_oldsequence_tsv()), 1)
self.assertEqual(
util.get_file_line_count(task.get_crystalph_tsv()), 1)
self.assertTrue(task.get_email_log().startswith(
'\nWARNING: Unable to download'))
finally:
shutil.rmtree(temp_dir)
def run(self):
"""Runs blastnfilter task after verifying dataimport was good
Method requires can_run() to be called before hand with
successful outcome
Otherwise method invokes D3RTask.start then this method
creates a directory and invokes blastnfilter script and
postanalysis script. Upon completion results are
analyzed and success or error status is set
appropriately and D3RTask.end is invoked
"""
super(BlastNFilterTask, self).run()
if self._can_run is False:
logger.debug(self.get_dir_name() +
' cannot run cause _can_run flag '
'is False')
return
data_import = DataImportTask(self._path, self._args)
make_blastdb = MakeBlastDBTask(self._path, self._args)
try:
loglevel = self.get_args().loglevel
except AttributeError:
logger.debug('No log level set in arguments using WARNING')
loglevel = 'WARNING'
# verify sequence.tsv file exists on filesystem.
# if not fall back to oldsequence.tsv file
sequencetsv = data_import.get_sequence_tsv()
if not os.path.isfile(sequencetsv):
logger.warning(sequencetsv + ' file not found. falling '
'back to old file')
self.append_to_email_log('\n ' + sequencetsv + ' file not found ' +
'falling back to ' +
data_import.get_oldsequence_tsv() + '\n')
sequencetsv = data_import.get_oldsequence_tsv()
cmd_to_run = (self.get_args().blastnfilter + ' --nonpolymertsv ' +
data_import.get_nonpolymer_tsv() + ' --sequencetsv ' +
sequencetsv + ' --pdbblastdb ' + make_blastdb.get_dir() +
' --compinchi ' +
data_import.get_components_inchi_file() +
' --crystalpH ' + data_import.get_crystalph_tsv() +
' --pdbdb ' + self.get_args().pdbdb + ' --log ' +
loglevel + ' --outdir ' + self.get_dir())
blastnfilter_name = os.path.basename(self.get_args().blastnfilter)
self.run_external_command(
blastnfilter_name,
cmd_to_run,
False,
)
self.set_status(D3RTask.COMPLETE_STATUS)
cmd_to_run = (self.get_args().postanalysis + ' --compinchi ' +
data_import.get_components_inchi_file() + ' ' +
self.get_dir())
postanalysis_name = os.path.basename(self.get_args().postanalysis)
self.run_external_command(postanalysis_name, cmd_to_run, False)
try:
# examine output to get candidate hit count DR-12
hit_stats = self._parse_blastnfilter_output_for_hit_stats()
if hit_stats is not None:
self.append_to_email_log(hit_stats)
except Exception:
logger.exception("Error caught exception")
# assess the result
self.end()