Exemple #1
0
    def test_download_participant_list_file_not_found(self):
        temp_dir = tempfile.mkdtemp()
        try:
            foo = FtpFileTransfer(None)
            mockftp = D3RParameters()
            mockftp.get = Mock()
            foo.set_remote_dir('/foo')
            foo.set_connection(mockftp)

            params = D3RParameters()
            task = DataImportTask(temp_dir, params)
            task.set_file_transfer(foo)
            task.create_dir()
            task._download_participant_list_csv()
            self.assertEqual(
                task.get_email_log(),
                '\nWARNING: participant_list.csv not downloaded '
                'which means external users will NOT get '
                'evaluation email\n')
            mockftp.get\
                .assert_called_with('/foo/' +
                                    DataImportTask.PARTICIPANT_LIST_CSV,
                                    local=task.get_participant_list_csv())
        finally:
            shutil.rmtree(temp_dir)
Exemple #2
0
 def _get_participant_database(self):
     """Creates `ParticipantDatabase`
     :returns: ParticipantDatabase
     """
     dimport = DataImportTask(self.get_path(), self.get_args())
     csvfile = dimport.get_participant_list_csv()
     pfac = ParticipantDatabaseFromCSVFactory(csvfile)
     return pfac.get_participant_database()
Exemple #3
0
    def __init__(self, path, args):
        super(BlastNFilterTask, self).__init__(path, args)
        self.set_name(BlastNFilterTask.TASK_NAME)

        # set the stage to be 1 higher then Data Import Stage
        dataimport = DataImportTask(path, args)
        self.set_stage(dataimport.get_stage() + 1)
        self.set_status(D3RTask.UNKNOWN_STATUS)
Exemple #4
0
 def test_get_set_of_pdbid_from_crystalph_tsv_nonexistant_file(self):
     temp_dir = tempfile.mkdtemp()
     try:
         params = D3RParameters()
         task = DataImportTask(temp_dir, params)
         task.create_dir()
         pdbid_set = task.get_set_of_pdbid_in_crystalph_tsv_and_pdb_seqres()
         self.assertEqual(len(pdbid_set), 0)
     finally:
         shutil.rmtree(temp_dir)
Exemple #5
0
    def test_get_evaluation_tasks_on_dir_with_lower_stages_dirs(self):
        temp_dir = tempfile.mkdtemp()
        try:
            params = D3RParameters()
            dataimport = DataImportTask(temp_dir, params)
            blast = BlastNFilterTask(temp_dir, params)
            os.mkdir(os.path.join(temp_dir, dataimport.get_dir_name()))
            os.mkdir(os.path.join(temp_dir, blast.get_dir_name()))

            stf = EvaluationTaskFactory(temp_dir, params)
            task_list = stf.get_evaluation_tasks()
            self.assertEquals(len(task_list), 0)
        finally:
            shutil.rmtree(temp_dir)
Exemple #6
0
    def test_run_with_blast_success_postanalysis_success_no_summary_file(self):
        temp_dir = tempfile.mkdtemp()

        try:
            params = D3RParameters()
            params.blastnfilter = 'true'
            params.postanalysis = '/bin/echo'
            params.pdbdb = '/pdbdb'
            blasttask = BlastNFilterTask(temp_dir, params)
            blasttask._can_run = True
            blasttask.run()
            self.assertEqual(blasttask.get_status(), D3RTask.COMPLETE_STATUS)
            self.assertEqual(blasttask.get_error(), None)
            complete_file = os.path.join(blasttask.get_dir(),
                                         D3RTask.COMPLETE_FILE)

            self.assertEqual(os.path.isfile(complete_file), True)

            std_err_file = os.path.join(blasttask.get_dir(), 'echo.stderr')

            self.assertEqual(os.path.isfile(std_err_file), True)

            std_out_file = os.path.join(blasttask.get_dir(), 'echo.stdout')
            dataimport = DataImportTask(temp_dir, params)
            f = open(std_out_file, 'r')
            echo_out = f.read().replace('\n', '')
            echo_out.index('--compinchi ' +
                           os.path.join(temp_dir, dataimport.get_dir_name(),
                                        DataImportTask.COMPINCHI_ICH))

            echo_out.index(' ' +
                           os.path.join(temp_dir, blasttask.get_dir_name()))
            f.close()

            self.assertEqual(os.path.isfile(std_out_file), True)
            self.assertEquals(blasttask.get_status(), D3RTask.COMPLETE_STATUS)
            self.assertEquals(
                os.path.exists(os.path.join(blasttask.get_dir(),
                                            'true.stderr')), True)
            self.assertEquals(
                os.path.exists(os.path.join(blasttask.get_dir(),
                                            'true.stdout')), True)
            res = blasttask.get_email_log().rstrip('\n')
            res.index('/bin/echo')
            res.index('# txt files found: 0')
            res.index('Output from summary.txt')
        finally:
            shutil.rmtree(temp_dir)
Exemple #7
0
 def test_get_set_of_pdbid_from_crystalph_tsv_invalid_entries(self):
     """Header missing and 4rfr has 3 columns
     """
     temp_dir = tempfile.mkdtemp()
     try:
         params = D3RParameters()
         task = DataImportTask(temp_dir, params)
         task.create_dir()
         f = open(task.get_crystalph_tsv(), 'w')
         f.write('4X09\t6.5\n')
         f.write('4rfr 7.5 8\n')
         f.write('4XET\t6.2\n')
         f.write('4XF1\t6.2\n')
         f.write('4XF3\t6.2\n')
         f.flush()
         f.close()
         pdbid_set = task.get_set_of_pdbid_from_crystalph_tsv()
         self.assertEqual(os.path.isfile(task.get_crystalph_tsv()), True)
         self.assertEqual(len(pdbid_set), 3)
         self.assertEqual('4RFR' in pdbid_set, False)
         self.assertEqual('4X09' in pdbid_set, False)
         self.assertEqual('4XET' in pdbid_set, True)
         self.assertEqual('4XF1' in pdbid_set, True)
         self.assertEqual('4XF3' in pdbid_set, True)
     finally:
         shutil.rmtree(temp_dir)
Exemple #8
0
    def test_run_all_compinchi_fail(self):
        temp_dir = tempfile.mkdtemp()
        try:
            params = D3RParameters()
            params.pdbfileurl = 'file://' + temp_dir
            params.compinchi = 'file://' + temp_dir

            make_blast = MakeBlastDBTask(temp_dir, params)
            make_blast.create_dir()
            open(os.path.join(make_blast.get_dir(), D3RTask.COMPLETE_FILE),
                 'a').close()

            task = DataImportTask(temp_dir, params)
            task._retrysleep = 0
            open(os.path.join(temp_dir, task.NONPOLYMER_TSV), 'a').close()
            open(os.path.join(temp_dir, task.SEQUENCE_TSV), 'a').close()
            open(os.path.join(temp_dir, task.OLDSEQUENCE_TSV), 'a').close()
            open(os.path.join(temp_dir, task.CRYSTALPH_TSV), 'a').close()
            task.run()
            self.assertEquals(
                task.get_error(), 'Unable to download file ' + 'from ' +
                params.compinchi + ' to ' + task.get_components_inchi_file())

        finally:
            shutil.rmtree(temp_dir)
Exemple #9
0
    def can_run(self):
        """Determines if task can actually run

           This method first verifies the `MakeBlastDBTask` and
           `DataImportTask` task have
           `D3RTask.COMPLETE_STATUS` for status.  The method then
           verifies a `BlastNFilterTask` does not already exist.
             If above is not true then self.set_error() is set
             with information about the issue
           :return: True if can run otherwise False
        """
        self._can_run = False
        self._error = None
        # check blast
        make_blastdb = MakeBlastDBTask(self._path, self._args)
        make_blastdb.update_status_from_filesystem()
        if make_blastdb.get_status() != D3RTask.COMPLETE_STATUS:
            logger.info('Cannot run ' + self.get_name() + ' task ' +
                        'because ' + make_blastdb.get_name() + ' task' +
                        'has a status of ' + make_blastdb.get_status())
            self.set_error(make_blastdb.get_name() + ' task has ' +
                           make_blastdb.get_status() + ' status')
            return False

        # check data import
        data_import = DataImportTask(self._path, self._args)
        data_import.update_status_from_filesystem()
        if data_import.get_status() != D3RTask.COMPLETE_STATUS:
            logger.info('Cannot run ' + self.get_name() + ' task ' +
                        'because ' + data_import.get_name() + ' task' +
                        'has a status of ' + data_import.get_status())
            self.set_error(data_import.get_name() + ' task has ' +
                           data_import.get_status() + ' status')
            return False

        # check blast is not complete and does not exist

        self.update_status_from_filesystem()
        if self.get_status() == D3RTask.COMPLETE_STATUS:
            logger.debug("No work needed for " + self.get_name() + " task")
            return False

        if self.get_status() != D3RTask.NOTFOUND_STATUS:
            logger.warning(self.get_name() + " task was already " +
                           "attempted, but there was a problem")
            self.set_error(self.get_dir_name() + ' already exists and ' +
                           'status is ' + self.get_status())
            return False
        self._can_run = True
        return True
Exemple #10
0
 def test_get_set_of_pdbid_from_crystalph_tsv_valid_entries(self):
     temp_dir = tempfile.mkdtemp()
     try:
         params = D3RParameters()
         task = DataImportTask(temp_dir, params)
         task.create_dir()
         f = open(task.get_crystalph_tsv(), 'w')
         f.write('PDB_ID  _exptl_crystal_grow.pH\n')
         f.write('4rfr    7.5\n')
         f.write('4X09\t6.5\n')
         f.write('4XET\t6.2\n')
         f.write('4XF1\t6.2\n')
         f.write('4XF3\t6.2\n')
         f.flush()
         f.close()
         pdbid_set = task.get_set_of_pdbid_from_crystalph_tsv()
         self.assertEqual(os.path.isfile(task.get_crystalph_tsv()), True)
         self.assertEqual(len(pdbid_set), 5)
         self.assertEqual('4RFR' in pdbid_set, True)
         self.assertEqual('4X09' in pdbid_set, True)
         self.assertEqual('4XET' in pdbid_set, True)
         self.assertEqual('4XF1' in pdbid_set, True)
         self.assertEqual('4XF3' in pdbid_set, True)
     finally:
         shutil.rmtree(temp_dir)
Exemple #11
0
 def test_get_participant_database_with_valid_csvfile(self):
     temp_dir = tempfile.mkdtemp()
     try:
         params = D3RParameters()
         dimport = DataImportTask(temp_dir, params)
         dimport.create_dir()
         csvfile = dimport.get_participant_list_csv()
         f = open(csvfile, 'w')
         f.write('name,d3rusername,guid,email\n')
         f.write('joe,jj,123,[email protected]\n')
         f.flush()
         f.close()
         etf = EvaluationTaskFactory(temp_dir, params)
         pdb = etf._get_participant_database()
         p = pdb.get_participant_by_guid('123')
         self.assertEqual(p.get_email(), '*****@*****.**')
     finally:
         shutil.rmtree(temp_dir)
Exemple #12
0
    def test_run_success_with_ftp_upload(self):
        temp_dir = tempfile.mkdtemp()
        try:
            script = self.create_gen_challenge_script(temp_dir)
            params = D3RParameters()
            params.genchallenge = script
            params.pdbdb = '/foo'
            params.version = '1'

            blastnfilter = BlastNFilterTask(temp_dir, params)
            blastnfilter.create_dir()
            open(os.path.join(blastnfilter.get_dir(), D3RTask.COMPLETE_FILE),
                 'a').close()

            chall = ChallengeDataTask(temp_dir, params)
            mockftp = D3RParameters()
            mockftp.put = Mock(side_effect=[3, 5])
            ftp = FtpFileTransfer(None)
            ftp.set_remote_challenge_dir('/challenge')
            ftp.set_connection(mockftp)
            chall.set_file_transfer(ftp)

            dimport = DataImportTask(temp_dir, params)
            dimport.create_dir()

            ctsv = dimport.get_crystalph_tsv()
            f = open(ctsv, 'w')
            f.write('crystal')
            f.flush()
            f.close()

            nonpoly = dimport.get_nonpolymer_tsv()
            f = open(nonpoly, 'w')
            f.write('nonpoly')
            f.flush()
            f.close()

            seq = dimport.get_sequence_tsv()
            f = open(seq, 'w')
            f.write('seq')
            f.flush()
            f.close()

            chall.run()
            self.assertEqual(chall.get_error(), None)
            # verify test files get created
            errfile = os.path.join(chall.get_dir(),
                                   D3RTask.ERROR_FILE)
            self.assertEqual(os.path.isfile(errfile), False)

            compfile = os.path.join(chall.get_dir(),
                                    D3RTask.COMPLETE_FILE)
            self.assertEqual(os.path.isfile(compfile), True)
            ftp.disconnect()

        finally:
            shutil.rmtree(temp_dir)
Exemple #13
0
    def test_get_set_of_pdbid_in_crystalph_tsv_and_pdb_seqres_w_hits(self):
        temp_dir = tempfile.mkdtemp()
        try:
            params = D3RParameters()
            task = DataImportTask(temp_dir, params)
            task.create_dir()
            f = open(task.get_crystalph_tsv(), 'w')
            f.write('PDB_ID  _exptl_crystal_grow.pH\n')
            f.write('4X09\t6.5\n')
            f.write('4rfr\t8\n')
            f.write('4XET\t6.2\n')
            f.write('4XF1\t6.2\n')
            f.write('4XF3\t6.2\n')
            f.flush()
            f.close()

            makeblast = MakeBlastDBTask(temp_dir, params)
            makeblast.create_dir()
            f = open(makeblast.get_pdb_seqres_txt(), 'w')
            f.write('>4rfr_A mol:protein length:154  MYOGLOBIN\n')
            f.write('MVLSEGEWQLVLHVWAKVEADVAGHGQDILIRLFKSHPETLEKFDRVK'
                    'HLKTEAEMKASEDLKKHG\n')
            f.write('>102l_A mol:protein length:165  T4 LYSOZYME\n')
            f.write('MNIFEMLRIDEGLRLKIYKDTEGYYTIGIGHLLTKSPSLNAAAKSELDKA'
                    'IGRNTNGVITKDEAEKLFNQDVDAAVRGILRNAKLKPVYDSLDAVRRAAL'
                    'INMVFQMGETGVAGFTNSLRMLQQKRWDEAAVNLAKSRWYNQTPNRAKRV'
                    'ITTFRTGTWDAYKNL\n')
            f.flush()
            f.close()

            pdbset = task.get_set_of_pdbid_in_crystalph_tsv_and_pdb_seqres()
            self.assertEqual(len(pdbset), 1)
            self.assertEqual('4RFR' in pdbset, True)
        finally:
            shutil.rmtree(temp_dir)
Exemple #14
0
    def test_copy_over_tsv_files(self):
        temp_dir = tempfile.mkdtemp()
        try:
            params = D3RParameters()
            dimport = DataImportTask(temp_dir, params)
            dimport.create_dir()

            ctsv = dimport.get_crystalph_tsv()
            f = open(ctsv, 'w')
            f.write('crystal')
            f.flush()
            f.close()

            nonpoly = dimport.get_nonpolymer_tsv()
            f = open(nonpoly, 'w')
            f.write('nonpoly')
            f.flush()
            f.close()

            seq = dimport.get_sequence_tsv()
            f = open(seq, 'w')
            f.write('seq')
            f.flush()
            f.close()

            task = ChallengeDataTask(temp_dir, params)
            task.create_dir()
            challenge_dir = task._create_challenge_dir()
            self.assertEqual(os.path.isdir(challenge_dir), True)
            task._copy_over_tsv_files(challenge_dir)

            cop_ctsv = os.path.join(challenge_dir,
                                    DataImportTask.CRYSTALPH_TSV)
            self.assertEqual(os.path.isfile(cop_ctsv), True)
            f = open(cop_ctsv)
            self.assertEqual(f.readline(), 'crystal')
            f.close()

            cop_nonpoly = os.path.join(challenge_dir,
                                       DataImportTask.NONPOLYMER_TSV)
            self.assertEqual(os.path.isfile(cop_nonpoly), True)
            f = open(cop_nonpoly)
            self.assertEqual(f.readline(), 'nonpoly')
            f.close()

            cop_seq = os.path.join(challenge_dir,
                                   DataImportTask.SEQUENCE_TSV)
            self.assertEqual(os.path.isfile(cop_seq), True)
            f = open(cop_seq)
            self.assertEqual(f.readline(), 'seq')
            f.close()

        finally:
            shutil.rmtree(temp_dir)
Exemple #15
0
 def test_get_set_of_pdbid_from_crystalph_tsv_internal_exception(self):
     temp_dir = tempfile.mkdtemp()
     try:
         params = D3RParameters()
         task = DataImportTask(temp_dir, params)
         task.create_dir()
         open(task.get_crystalph_tsv(), 'a').close()
         os.chmod(task.get_crystalph_tsv(), 0o000)
         pdbid_set = task.get_set_of_pdbid_from_crystalph_tsv()
         self.assertEqual(len(pdbid_set), 0)
     finally:
         shutil.rmtree(temp_dir)
Exemple #16
0
 def test_get_set_of_pdbid_from_crystalph_tsv_empty_file(self):
     temp_dir = tempfile.mkdtemp()
     try:
         params = D3RParameters()
         task = DataImportTask(temp_dir, params)
         task.create_dir()
         open(task.get_crystalph_tsv(), 'a').close()
         pdbid_set = task.get_set_of_pdbid_in_crystalph_tsv_and_pdb_seqres()
         self.assertEqual(os.path.isfile(task.get_crystalph_tsv()), True)
         self.assertEqual(len(pdbid_set), 0)
     finally:
         shutil.rmtree(temp_dir)
Exemple #17
0
 def test_get_external_submitter_email_participant_not_found(self):
     temp_dir = tempfile.mkdtemp()
     try:
         params = D3RParameters()
         dimport = DataImportTask(temp_dir, params)
         dimport.create_dir()
         csvfile = dimport.get_participant_list_csv()
         f = open(csvfile, 'w')
         f.write('name,d3rusername,guid,email\n')
         f.write('joe,jj,123,[email protected]\n')
         f.flush()
         f.close()
         fac = ParticipantDatabaseFromCSVFactory(csvfile)
         params = D3RParameters()
         dtask = D3RTask('/foo', params)
         dtask.set_name('444' + EvaluationTask.EXT_SUBMISSION_SUFFIX)
         task = EvaluationTask(temp_dir, dtask.get_name(), dtask, params)
         emailer = EvaluationEmailer(fac.get_participant_database(), None)
         self.assertEqual(emailer._get_external_submitter_email(task), None)
         self.assertEqual(emailer.get_message_log(),
                          '\nNo participant found with guid: 444\n')
     finally:
         shutil.rmtree(temp_dir)
Exemple #18
0
    def test_download_files_with_wait_all_successful(self):
        temp_dir = tempfile.mkdtemp()
        try:
            prev_friday = util.get_previous_friday_from_date(
                datetime.now(tzlocal()))
            sat = prev_friday + timedelta(days=1)
            dse = sat - datetime(1970, 01, 01, tzinfo=tzutc())
            secs_since_epoch = self.get_total_seconds(dse)

            nonpoly = os.path.join(temp_dir, DataImportTask.NONPOLYMER_TSV)
            f = open(nonpoly, 'w')
            f.write('nonpoly\n')
            f.flush()
            f.close()
            os.utime(nonpoly, (secs_since_epoch, secs_since_epoch))

            seq = os.path.join(temp_dir, DataImportTask.SEQUENCE_TSV)
            f = open(seq, 'w')
            f.write('seq\n')
            f.flush()
            f.close()
            os.utime(seq, (secs_since_epoch, secs_since_epoch))

            oldseq = os.path.join(temp_dir, DataImportTask.OLDSEQUENCE_TSV)
            f = open(oldseq, 'w')
            f.write('oldseq\n')
            f.flush()
            f.close()
            os.utime(oldseq, (secs_since_epoch, secs_since_epoch))

            crystal = os.path.join(temp_dir, DataImportTask.CRYSTALPH_TSV)
            f = open(crystal, 'w')
            f.write('crystal\n')
            f.flush()
            f.close()
            os.utime(crystal, (secs_since_epoch, secs_since_epoch))

            comp = os.path.join(temp_dir, DataImportTask.COMPINCHI_ICH)

            f = open(comp, 'w')
            f.write('comp\n')
            f.flush()
            f.close()

            params = D3RParameters()
            params.skipimportwait = False
            params.importretry = 2
            params.importsleep = 0
            params.compinchi = 'file://' + temp_dir
            task = DataImportTask(temp_dir, params)
            task.create_dir()
            val = task._download_files('file://' + temp_dir)
            self.assertEqual(task.get_error(), None)
            self.assertTrue(val)

        finally:
            shutil.rmtree(temp_dir)
Exemple #19
0
    def test_can_run_with_complete_file(self):
        temp_dir = tempfile.mkdtemp()
        try:
            params = D3RParameters()
            task = DataImportTask(temp_dir, params)
            task.create_dir()
            open(os.path.join(task.get_dir(), D3RTask.COMPLETE_FILE),
                 'a').close()
            self.assertEquals(task.can_run(), False)
            self.assertEquals(task.get_error(), None)
            self.assertEquals(task._can_run, False)

        finally:
            shutil.rmtree(temp_dir)
Exemple #20
0
 def test_get_set_of_pdbid_from_crystalph_tsv_no_header_invalid_lines(self):
     temp_dir = tempfile.mkdtemp()
     try:
         params = D3RParameters()
         task = DataImportTask(temp_dir, params)
         task.create_dir()
         f = open(task.get_crystalph_tsv(), 'w')
         f.write('hello\nx\n')
         f.flush()
         f.close()
         pdbid_set = task.get_set_of_pdbid_from_crystalph_tsv()
         self.assertEqual(os.path.isfile(task.get_crystalph_tsv()), True)
         self.assertEqual(len(pdbid_set), 0)
     finally:
         shutil.rmtree(temp_dir)
Exemple #21
0
 def test_download_participant_list_no_filetransfer(self):
     temp_dir = tempfile.mkdtemp()
     try:
         params = D3RParameters()
         task = DataImportTask(temp_dir, params)
         task.create_dir()
         task._download_participant_list_csv()
         self.assertEqual(
             task.get_email_log(), '\nWARNING: Unable to download '
             'participant_list.csv which means '
             'external users will NOT get evaluation '
             'email : \'NoneType\' object has no attribute '
             '\'connect\'\n')
     finally:
         shutil.rmtree(temp_dir)
Exemple #22
0
 def test_run_pdbfileurl_not_set(self):
     temp_dir = tempfile.mkdtemp()
     try:
         params = D3RParameters()
         task = DataImportTask(temp_dir, params)
         task._retrysleep = 0
         task.run()
         self.assertEquals(
             task.get_error(),
             'cannot download files' + ' cause pdbfileurl not set')
     finally:
         shutil.rmtree(temp_dir)
Exemple #23
0
 def test_run_can_run_flag_false(self):
     temp_dir = tempfile.mkdtemp()
     try:
         params = D3RParameters()
         params.pdbfileurl = 'file://' + temp_dir
         params.compinchi = 'file://' + temp_dir
         task = DataImportTask(temp_dir, params)
         task._can_run = False
         task.run()
         self.assertEquals(task.get_error(), None)
     finally:
         shutil.rmtree(temp_dir)
Exemple #24
0
    def test_tar_challenge_dir(self):
        temp_dir = tempfile.mkdtemp()
        try:
            params = D3RParameters()
            dimport = DataImportTask(temp_dir, params)
            dimport.create_dir()
            open(dimport.get_sequence_tsv(), 'a').close()
            open(dimport.get_nonpolymer_tsv(), 'a').close()
            open(dimport.get_crystalph_tsv(), 'a').close()

            task = ChallengeDataTask(temp_dir, params)
            task.create_dir()
            chall_dir = task._create_challenge_dir()

            final_log = os.path.join(chall_dir, 'final.log')
            open(final_log, 'a').close()

            task._create_readme(chall_dir)
            task._copy_over_tsv_files(chall_dir)

            # make a fake candidate
            file_list = self.make_fake_candidate_dir(chall_dir, '5hib',
                                                     '2eb2', 'CSX')

            self.assertEqual(os.path.isdir(chall_dir), True)
            name = task.get_celpp_challenge_data_dir_name()
            tfile = task._tar_challenge_dir(name)
            self.assertEqual(os.path.isfile(tfile), True)

            foodir = os.path.join(temp_dir, 'foo')
            tar = tarfile.open(tfile, 'r:*')
            tar.extractall(path=foodir)
            tar.close()
            cdir = os.path.join(foodir, name)
            readme = os.path.join(cdir, ChallengeDataTask.README_TXT_FILE)
            self.assertEqual(os.path.isfile(readme), True)

            final_log = os.path.join(foodir, 'final.log')
            self.assertEqual(os.path.isfile(final_log), False)

            for fname in file_list:
                chk = os.path.join(cdir, fname)
                self.assertEqual(os.path.isfile(chk), True, chk)

        finally:
            shutil.rmtree(temp_dir)
Exemple #25
0
    def test_run_all_nonpolymer_fail(self):
        temp_dir = tempfile.mkdtemp()
        try:
            params = D3RParameters()
            params.pdbfileurl = 'file://' + temp_dir
            params.compinchi = 'file://' + temp_dir

            make_blast = MakeBlastDBTask(temp_dir, params)
            make_blast.create_dir()
            open(os.path.join(make_blast.get_dir(), D3RTask.COMPLETE_FILE),
                 'a').close()

            task = DataImportTask(temp_dir, params)
            task._retrysleep = 0
            task.run()
            self.assertEquals(
                task.get_error(), 'Unable to download file ' + 'from ' +
                params.pdbfileurl + ' to ' + task.get_nonpolymer_tsv())
        finally:
            shutil.rmtree(temp_dir)
Exemple #26
0
    def test_download_files_fail(self):
        temp_dir = tempfile.mkdtemp()
        try:
            nonpoly = os.path.join(temp_dir, DataImportTask.NONPOLYMER_TSV)
            f = open(nonpoly, 'w')
            f.write('nonpoly\n')
            f.flush()
            f.close()

            seq = os.path.join(temp_dir, DataImportTask.SEQUENCE_TSV)
            f = open(seq, 'w')
            f.write('seq\n')
            f.flush()
            f.close()

            oldseq = os.path.join(temp_dir, DataImportTask.OLDSEQUENCE_TSV)
            f = open(oldseq, 'w')
            f.write('oldseq\n')
            f.flush()
            f.close()

            crystal = os.path.join(temp_dir, DataImportTask.CRYSTALPH_TSV)
            f = open(crystal, 'w')
            f.write('crystal\n')
            f.flush()
            f.close()

            params = D3RParameters()
            params.skipimportwait = True
            params.compinchi = 'file://' + temp_dir
            task = DataImportTask(temp_dir, params)
            task.create_dir()
            val = task._download_files('file://' + temp_dir)
            self.assertEqual(
                task.get_error(), 'Unable to download file from '
                'file://' + temp_dir + ' to ' +
                task.get_components_inchi_file())
            self.assertFalse(val)

        finally:
            shutil.rmtree(temp_dir)
Exemple #27
0
 def test_get_set_of_pdbid_in_crystalph_tsv_and_pdb_seqres_no_seq(self):
     temp_dir = tempfile.mkdtemp()
     try:
         params = D3RParameters()
         task = DataImportTask(temp_dir, params)
         task.create_dir()
         f = open(task.get_crystalph_tsv(), 'w')
         f.write('PDB_ID  _exptl_crystal_grow.pH\n')
         f.write('4X09\t6.5\n')
         f.write('4rfr\t8\n')
         f.write('4XET\t6.2\n')
         f.write('4XF1\t6.2\n')
         f.write('4XF3\t6.2\n')
         f.flush()
         f.close()
         pdbset = task.get_set_of_pdbid_in_crystalph_tsv_and_pdb_seqres()
         self.assertEqual(len(pdbset), 0)
     finally:
         shutil.rmtree(temp_dir)
Exemple #28
0
    def run(self):
        """Runs blastnfilter task after verifying dataimport was good

           Method requires can_run() to be called before hand with
           successful outcome
           Otherwise method invokes D3RTask.start then this method
           creates a directory and invokes blastnfilter script and
           postanalysis script.  Upon completion results are
           analyzed and success or error status is set
           appropriately and D3RTask.end is invoked
           """
        super(BlastNFilterTask, self).run()

        if self._can_run is False:
            logger.debug(self.get_dir_name() +
                         ' cannot run cause _can_run flag '
                         'is False')
            return

        data_import = DataImportTask(self._path, self._args)

        make_blastdb = MakeBlastDBTask(self._path, self._args)

        try:
            loglevel = self.get_args().loglevel
        except AttributeError:
            logger.debug('No log level set in arguments using WARNING')
            loglevel = 'WARNING'

        # verify sequence.tsv file exists on filesystem.
        # if not fall back to oldsequence.tsv file
        sequencetsv = data_import.get_sequence_tsv()
        if not os.path.isfile(sequencetsv):
            logger.warning(sequencetsv + ' file not found. falling '
                           'back to old file')
            self.append_to_email_log('\n ' + sequencetsv + ' file not found ' +
                                     'falling back to ' +
                                     data_import.get_oldsequence_tsv() + '\n')
            sequencetsv = data_import.get_oldsequence_tsv()

        cmd_to_run = (self.get_args().blastnfilter + ' --nonpolymertsv ' +
                      data_import.get_nonpolymer_tsv() + ' --sequencetsv ' +
                      sequencetsv + ' --pdbblastdb ' + make_blastdb.get_dir() +
                      ' --compinchi ' +
                      data_import.get_components_inchi_file() +
                      ' --crystalpH ' + data_import.get_crystalph_tsv() +
                      ' --pdbdb ' + self.get_args().pdbdb + ' --log ' +
                      loglevel + ' --outdir ' + self.get_dir())

        blastnfilter_name = os.path.basename(self.get_args().blastnfilter)

        self.run_external_command(
            blastnfilter_name,
            cmd_to_run,
            False,
        )

        self.set_status(D3RTask.COMPLETE_STATUS)

        cmd_to_run = (self.get_args().postanalysis + ' --compinchi ' +
                      data_import.get_components_inchi_file() + ' ' +
                      self.get_dir())

        postanalysis_name = os.path.basename(self.get_args().postanalysis)

        self.run_external_command(postanalysis_name, cmd_to_run, False)

        try:
            # examine output to get candidate hit count DR-12
            hit_stats = self._parse_blastnfilter_output_for_hit_stats()
            if hit_stats is not None:
                self.append_to_email_log(hit_stats)
        except Exception:
            logger.exception("Error caught exception")

        # assess the result
        self.end()
Exemple #29
0
 def test_get_sequence_tsv(self):
     params = D3RParameters()
     task = DataImportTask('/foo', params)
     self.assertEqual(
         task.get_sequence_tsv(), '/foo/' + task.get_dir_name() +
         '/new_release_structure_sequence_canonical.tsv')
Exemple #30
0
 def test_get_nonpolymer_tsv(self):
     params = D3RParameters()
     task = DataImportTask('/foo', params)
     self.assertEqual(
         task.get_nonpolymer_tsv(), '/foo/' + task.get_dir_name() +
         '/new_release_structure_nonpolymer.tsv')