def test_run_where_everything_is_successful(self): temp_dir = tempfile.mkdtemp() try: params = D3RParameters() fakegz = os.path.join(temp_dir, 'fake.gz') f = gzip.open(fakegz, 'wb') f.write('hello\n') f.flush() f.close() params.pdbsequrl = 'file://' + fakegz params.makeblastdb = 'echo' task = MakeBlastDBTask(temp_dir, params) task._retrysleep = 0 task._maxretries = 1 task.run() self.assertEqual(task.get_error(), None) # check echo.stdout file for valid arguments f = open(os.path.join(task.get_dir(), 'echo.stdout'), 'r') line = f.readline() self.assertEqual( line, '-in ' + task.get_pdb_seqres_txt() + ' -out ' + os.path.join(task.get_dir(), 'pdb_db') + ' -dbtype prot\n') f.close() lines = task.get_email_log().split('\n') self.assertEqual(lines[2], '# sequence(s): 0') f.close() finally: shutil.rmtree(temp_dir)
def test_run_all_compinchi_fail(self): temp_dir = tempfile.mkdtemp() try: params = D3RParameters() params.pdbfileurl = 'file://' + temp_dir params.compinchi = 'file://' + temp_dir make_blast = MakeBlastDBTask(temp_dir, params) make_blast.create_dir() open(os.path.join(make_blast.get_dir(), D3RTask.COMPLETE_FILE), 'a').close() task = DataImportTask(temp_dir, params) task._retrysleep = 0 open(os.path.join(temp_dir, task.NONPOLYMER_TSV), 'a').close() open(os.path.join(temp_dir, task.SEQUENCE_TSV), 'a').close() open(os.path.join(temp_dir, task.OLDSEQUENCE_TSV), 'a').close() open(os.path.join(temp_dir, task.CRYSTALPH_TSV), 'a').close() task.run() self.assertEquals( task.get_error(), 'Unable to download file ' + 'from ' + params.compinchi + ' to ' + task.get_components_inchi_file()) finally: shutil.rmtree(temp_dir)
def test_can_run_does_not_exist_or_error(self): temp_dir = tempfile.mkdtemp() try: params = D3RParameters() task = DataImportTask(temp_dir, params) # no make blast db self.assertEquals(task.can_run(), False) self.assertEquals(task.get_error(), 'makeblastdb task has notfound status') self.assertEquals(task._can_run, False) make_blast = MakeBlastDBTask(temp_dir, params) make_blast.create_dir() # make blast db failed err_file = os.path.join(make_blast.get_dir(), D3RTask.ERROR_FILE) open(err_file, 'a').close() self.assertEquals(task.can_run(), False) self.assertEquals(task.get_error(), 'makeblastdb task has error status') self.assertEquals(task._can_run, False) os.remove(err_file) # make blast db success open(os.path.join(make_blast.get_dir(), D3RTask.COMPLETE_FILE), 'a').close() self.assertEquals(task.can_run(), True) self.assertEquals(task.get_error(), None) self.assertEquals(task._can_run, True) task.create_dir() open(os.path.join(task.get_dir(), D3RTask.ERROR_FILE), 'a').close() self.assertEquals(task.can_run(), False) self.assertEquals(task._can_run, False) self.assertEquals( task.get_error(), task.get_dir_name() + ' already exists and ' + 'status is ' + D3RTask.ERROR_STATUS) finally: shutil.rmtree(temp_dir)
def test_can_run_where_task_failed(self): temp_dir = tempfile.mkdtemp() try: params = D3RParameters() task = MakeBlastDBTask(temp_dir, params) task.create_dir() open(os.path.join(task.get_dir(), 'error'), 'a').close() self.assertEqual(task.can_run(), False) finally: shutil.rmtree(temp_dir)
def test_get_uploadable_files(self): temp_dir = tempfile.mkdtemp() try: params = D3RParameters() task = MakeBlastDBTask(temp_dir, params) task.create_dir() # test on empty dir self.assertEqual(task.get_uploadable_files(), []) # test with stderr/stdout files stdout = os.path.join(task.get_dir(), 'makeblastdb.stdout') open(stdout, 'a').close() stderr = os.path.join(task.get_dir(), 'makeblastdb.stderr') open(stderr, 'a').close() flist = task.get_uploadable_files() self.assertEqual(len(flist), 2) flist.index(stdout) flist.index(stderr) # test with pdb_seqres.txt.gz file open(task.get_pdb_seqres_txt_gz(), 'a').close() flist = task.get_uploadable_files() self.assertEqual(len(flist), 3) flist.index(stdout) flist.index(stderr) flist.index(task.get_pdb_seqres_txt_gz()) os.unlink(task.get_pdb_seqres_txt_gz()) # test where pdb_seqres.txt is a dir unlikely but why not check os.makedirs(task.get_pdb_seqres_txt_gz()) flist = task.get_uploadable_files() self.assertEqual(len(flist), 2) flist.index(stdout) flist.index(stderr) finally: shutil.rmtree(temp_dir)
def test_run_all_success(self): temp_dir = tempfile.mkdtemp() try: fakeftp = FtpFileTransfer(None) mftp = D3RParameters() fakeftp.set_connection(mftp) fakeftp.set_remote_dir('/foo2') mftp.get = Mock() params = D3RParameters() params.pdbfileurl = 'file://' + temp_dir params.compinchi = 'file://' + temp_dir make_blast = MakeBlastDBTask(temp_dir, params) make_blast.create_dir() open(os.path.join(make_blast.get_dir(), D3RTask.COMPLETE_FILE), 'a').close() task = DataImportTask(temp_dir, params) task.set_file_transfer(fakeftp) task._retrysleep = 0 open(os.path.join(temp_dir, task.NONPOLYMER_TSV), 'a').close() open(os.path.join(temp_dir, task.SEQUENCE_TSV), 'a').close() open(os.path.join(temp_dir, task.OLDSEQUENCE_TSV), 'a').close() open(os.path.join(temp_dir, task.CRYSTALPH_TSV), 'a').close() open(os.path.join(temp_dir, task.COMPINCHI_ICH), 'a').close() task.run() self.assertEquals(task.get_error(), None) # check line count is 1 now which indicates # standard was added self.assertEqual( util.get_file_line_count(task.get_nonpolymer_tsv()), 1) self.assertEqual(util.get_file_line_count(task.get_sequence_tsv()), 1) self.assertEqual( util.get_file_line_count(task.get_oldsequence_tsv()), 1) self.assertEqual( util.get_file_line_count(task.get_crystalph_tsv()), 1) mftp.get.assert_called_with('/foo2/' + DataImportTask.PARTICIPANT_LIST_CSV, local=task.get_participant_list_csv()) finally: shutil.rmtree(temp_dir)
def test_run_all_nonpolymer_fail(self): temp_dir = tempfile.mkdtemp() try: params = D3RParameters() params.pdbfileurl = 'file://' + temp_dir params.compinchi = 'file://' + temp_dir make_blast = MakeBlastDBTask(temp_dir, params) make_blast.create_dir() open(os.path.join(make_blast.get_dir(), D3RTask.COMPLETE_FILE), 'a').close() task = DataImportTask(temp_dir, params) task._retrysleep = 0 task.run() self.assertEquals( task.get_error(), 'Unable to download file ' + 'from ' + params.pdbfileurl + ' to ' + task.get_nonpolymer_tsv()) finally: shutil.rmtree(temp_dir)
def test_run_all_success_except_participant_download_fails(self): temp_dir = tempfile.mkdtemp() try: params = D3RParameters() params.pdbfileurl = 'file://' + temp_dir params.compinchi = 'file://' + temp_dir make_blast = MakeBlastDBTask(temp_dir, params) make_blast.create_dir() open(os.path.join(make_blast.get_dir(), D3RTask.COMPLETE_FILE), 'a').close() task = DataImportTask(temp_dir, params) task._retrysleep = 0 open(os.path.join(temp_dir, task.NONPOLYMER_TSV), 'a').close() open(os.path.join(temp_dir, task.SEQUENCE_TSV), 'a').close() open(os.path.join(temp_dir, task.OLDSEQUENCE_TSV), 'a').close() open(os.path.join(temp_dir, task.CRYSTALPH_TSV), 'a').close() open(os.path.join(temp_dir, task.COMPINCHI_ICH), 'a').close() task.run() self.assertEquals(task.get_error(), None) # check line count is 1 now which indicates # standard was added self.assertEqual( util.get_file_line_count(task.get_nonpolymer_tsv()), 1) self.assertEqual(util.get_file_line_count(task.get_sequence_tsv()), 1) self.assertEqual( util.get_file_line_count(task.get_oldsequence_tsv()), 1) self.assertEqual( util.get_file_line_count(task.get_crystalph_tsv()), 1) self.assertTrue(task.get_email_log().startswith( '\nWARNING: Unable to download')) finally: shutil.rmtree(temp_dir)
def run(self): """Runs blastnfilter task after verifying dataimport was good Method requires can_run() to be called before hand with successful outcome Otherwise method invokes D3RTask.start then this method creates a directory and invokes blastnfilter script and postanalysis script. Upon completion results are analyzed and success or error status is set appropriately and D3RTask.end is invoked """ super(BlastNFilterTask, self).run() if self._can_run is False: logger.debug(self.get_dir_name() + ' cannot run cause _can_run flag ' 'is False') return data_import = DataImportTask(self._path, self._args) make_blastdb = MakeBlastDBTask(self._path, self._args) try: loglevel = self.get_args().loglevel except AttributeError: logger.debug('No log level set in arguments using WARNING') loglevel = 'WARNING' # verify sequence.tsv file exists on filesystem. # if not fall back to oldsequence.tsv file sequencetsv = data_import.get_sequence_tsv() if not os.path.isfile(sequencetsv): logger.warning(sequencetsv + ' file not found. falling ' 'back to old file') self.append_to_email_log('\n ' + sequencetsv + ' file not found ' + 'falling back to ' + data_import.get_oldsequence_tsv() + '\n') sequencetsv = data_import.get_oldsequence_tsv() cmd_to_run = (self.get_args().blastnfilter + ' --nonpolymertsv ' + data_import.get_nonpolymer_tsv() + ' --sequencetsv ' + sequencetsv + ' --pdbblastdb ' + make_blastdb.get_dir() + ' --compinchi ' + data_import.get_components_inchi_file() + ' --crystalpH ' + data_import.get_crystalph_tsv() + ' --pdbdb ' + self.get_args().pdbdb + ' --log ' + loglevel + ' --outdir ' + self.get_dir()) blastnfilter_name = os.path.basename(self.get_args().blastnfilter) self.run_external_command( blastnfilter_name, cmd_to_run, False, ) self.set_status(D3RTask.COMPLETE_STATUS) cmd_to_run = (self.get_args().postanalysis + ' --compinchi ' + data_import.get_components_inchi_file() + ' ' + self.get_dir()) postanalysis_name = os.path.basename(self.get_args().postanalysis) self.run_external_command(postanalysis_name, cmd_to_run, False) try: # examine output to get candidate hit count DR-12 hit_stats = self._parse_blastnfilter_output_for_hit_stats() if hit_stats is not None: self.append_to_email_log(hit_stats) except Exception: logger.exception("Error caught exception") # assess the result self.end()