예제 #1
0
    def test_study_files_get_req(self):
        obs = study_files_get_req('*****@*****.**', 1, 1, 'FASTQ')
        exp = {
            'status':
            'success',
            'message':
            '',
            'remaining': ['uploaded_file.txt'],
            'file_types': [('raw_barcodes', True, []),
                           ('raw_forward_seqs', True, []),
                           ('raw_reverse_seqs', False, [])],
            'num_prefixes':
            1,
            'artifacts': [(1, 'Identification of the Microbiomes for '
                           'Cannabis Soils (1) - Raw data 1 (1)')]
        }
        self.assertEqual(obs, exp)

        obs = study_files_get_req('*****@*****.**', 1, 1, 'FASTQ')
        exp = {
            'status':
            'success',
            'message':
            '',
            'remaining': ['uploaded_file.txt'],
            'file_types': [('raw_barcodes', True, []),
                           ('raw_forward_seqs', True, []),
                           ('raw_reverse_seqs', False, [])],
            'num_prefixes':
            1,
            'artifacts': [(1, 'Identification of the Microbiomes for '
                           'Cannabis Soils (1) - Raw data 1 (1)')]
        }
        self.assertEqual(obs, exp)

        # adding a new study for further testing
        info = {
            "timeseries_type_id": 1,
            "metadata_complete": True,
            "mixs_compliant": True,
            "number_samples_collected": 25,
            "number_samples_promised": 28,
            "study_alias": "FCM",
            "study_description": "DESC",
            "study_abstract": "ABS",
            "emp_person_id": qdb.study.StudyPerson(2),
            "principal_investigator_id": qdb.study.StudyPerson(3),
            "lab_person_id": qdb.study.StudyPerson(1)
        }
        new_study = qdb.study.Study.create(qdb.user.User('*****@*****.**'),
                                           "Some New Study to get files", info)

        # check that you can't call a this function using two unrelated
        # study_id and prep_template_id
        with self.assertRaises(IncompetentQiitaDeveloperError):
            study_files_get_req('*****@*****.**', new_study.id, 1, 'FASTQ')
예제 #2
0
    def test_study_files_get_req(self):
        obs = study_files_get_req('*****@*****.**', 1, 1, 'FASTQ')
        exp = {'status': 'success',
               'message': '',
               'remaining': ['uploaded_file.txt'],
               'file_types': [('raw_barcodes', True, []),
                              ('raw_forward_seqs', True, []),
                              ('raw_reverse_seqs', False, [])],
               'num_prefixes': 1,
               'artifacts': [(1, 'Identification of the Microbiomes for '
                                 'Cannabis Soils (1) - Raw data 1 (1)')]}
        self.assertEqual(obs, exp)

        obs = study_files_get_req('*****@*****.**', 1, 1, 'FASTQ')
        exp = {'status': 'success',
               'message': '',
               'remaining': ['uploaded_file.txt'],
               'file_types': [('raw_barcodes', True, []),
                              ('raw_forward_seqs', True, []),
                              ('raw_reverse_seqs', False, [])],
               'num_prefixes': 1,
               'artifacts': [(1, 'Identification of the Microbiomes for '
                                 'Cannabis Soils (1) - Raw data 1 (1)')]}
        self.assertEqual(obs, exp)

        # adding a new study for further testing
        info = {
            "timeseries_type_id": 1,
            "metadata_complete": True,
            "mixs_compliant": True,
            "number_samples_collected": 25,
            "number_samples_promised": 28,
            "study_alias": "FCM",
            "study_description": "DESC",
            "study_abstract": "ABS",
            "emp_person_id": qdb.study.StudyPerson(2),
            "principal_investigator_id": qdb.study.StudyPerson(3),
            "lab_person_id": qdb.study.StudyPerson(1)
        }
        new_study = qdb.study.Study.create(
            qdb.user.User('*****@*****.**'), "Some New Study to get files", info)

        # check that you can't call a this function using two unrelated
        # study_id and prep_template_id
        with self.assertRaises(IncompetentQiitaDeveloperError):
            study_files_get_req('*****@*****.**', new_study.id, 1, 'FASTQ')
예제 #3
0
    def test_study_files_get_req_multiple(self):
        study_id = 1
        # adding a new prep for testing
        PREP = qdb.metadata_template.prep_template.PrepTemplate
        prep_info_dict = {
            'SKB7.640196': {
                'run_prefix': 'test_1'
            },
            'SKB8.640193': {
                'run_prefix': 'test_2'
            }
        }
        prep_info = pd.DataFrame.from_dict(prep_info_dict,
                                           orient='index',
                                           dtype=str)
        pt = npt.assert_warns(qdb.exceptions.QiitaDBWarning, PREP.create,
                              prep_info, qdb.study.Study(study_id),
                              "Metagenomic")

        # getting the upload folder so we can test
        study_upload_dir = join(
            qdb.util.get_mountpoint("uploads")[0][1], str(study_id))

        # adding just foward per sample FASTQ to the upload folder
        filenames = ['test_1.R1.fastq.gz', 'test_2.R1.fastq.gz']
        for f in filenames:
            fpt = join(study_upload_dir, f)
            open(fpt, 'w', 0).close()
            self._clean_up_files.append(fpt)
        obs = study_files_get_req('*****@*****.**', 1, pt.id,
                                  'per_sample_FASTQ')
        exp = {
            'status':
            'success',
            'num_prefixes':
            2,
            'artifacts': [],
            'remaining': ['uploaded_file.txt'],
            'message':
            '',
            'file_types': [('raw_forward_seqs', True,
                            ['test_2.R1.fastq.gz', 'test_1.R1.fastq.gz']),
                           ('raw_reverse_seqs', False, [])]
        }
        self.assertEqual(obs, exp)

        # let's add reverse
        filenames = ['test_1.R2.fastq.gz', 'test_2.R2.fastq.gz']
        for f in filenames:
            fpt = join(study_upload_dir, f)
            open(fpt, 'w', 0).close()
            self._clean_up_files.append(fpt)
        obs = study_files_get_req('*****@*****.**', 1, pt.id,
                                  'per_sample_FASTQ')
        exp = {
            'status':
            'success',
            'num_prefixes':
            2,
            'artifacts': [],
            'remaining': ['uploaded_file.txt'],
            'message':
            '',
            'file_types': [('raw_forward_seqs', True,
                            ['test_2.R1.fastq.gz', 'test_1.R1.fastq.gz']),
                           ('raw_reverse_seqs', False,
                            ['test_2.R2.fastq.gz', 'test_1.R2.fastq.gz'])]
        }
        self.assertEqual(obs, exp)

        # let's an extra file that matches
        filenames = ['test_1.R3.fastq.gz']
        for f in filenames:
            fpt = join(study_upload_dir, f)
            open(fpt, 'w', 0).close()
            self._clean_up_files.append(fpt)
        obs = study_files_get_req('*****@*****.**', 1, pt.id,
                                  'per_sample_FASTQ')
        exp = {
            'status':
            'success',
            'num_prefixes':
            2,
            'artifacts': [],
            'remaining': [
                'test_1.R1.fastq.gz', 'test_1.R2.fastq.gz',
                'test_1.R3.fastq.gz', 'uploaded_file.txt'
            ],
            'message':
            "Check these run_prefix:\n'test_1' has 3 matches.",
            'file_types': [('raw_forward_seqs', True, ['test_2.R1.fastq.gz']),
                           ('raw_reverse_seqs', False, ['test_2.R2.fastq.gz'])]
        }
        self.assertEqual(obs, exp)

        # now if we select FASTQ we have 3 columns so the extra file should go
        # to the 3rd column
        obs = study_files_get_req('*****@*****.**', 1, pt.id, 'FASTQ')
        exp = {
            'status':
            'success',
            'num_prefixes':
            2,
            'remaining': ['uploaded_file.txt'],
            'message':
            '',
            'artifacts': [(1, 'Identification of the Microbiomes for '
                           'Cannabis Soils (1) - Raw data 1 (1)')],
            'file_types': [('raw_barcodes', True,
                            ['test_2.R1.fastq.gz', 'test_1.R1.fastq.gz']),
                           ('raw_forward_seqs', True,
                            ['test_2.R2.fastq.gz', 'test_1.R2.fastq.gz']),
                           ('raw_reverse_seqs', False, ['test_1.R3.fastq.gz'])]
        }
        self.assertEqual(obs, exp)

        PREP.delete(pt.id)
예제 #4
0
    def test_study_files_get_req(self):
        obs = study_files_get_req('*****@*****.**', 1, 1, 'FASTQ')
        exp = {
            'status':
            'success',
            'message':
            '',
            'remaining': ['uploaded_file.txt'],
            'file_types': [('raw_barcodes', True, []),
                           ('raw_forward_seqs', True, []),
                           ('raw_reverse_seqs', False, [])],
            'num_prefixes':
            1,
            'artifacts': [(1, 'Identification of the Microbiomes for '
                           'Cannabis Soils (1) - Raw data 1 (1)')]
        }
        self.assertEqual(obs, exp)

        obs = study_files_get_req('*****@*****.**', 1, 1, 'FASTQ')
        exp = {
            'status':
            'success',
            'message':
            '',
            'remaining': ['uploaded_file.txt'],
            'file_types': [('raw_barcodes', True, []),
                           ('raw_forward_seqs', True, []),
                           ('raw_reverse_seqs', False, [])],
            'num_prefixes':
            1,
            'artifacts': [(1, 'Identification of the Microbiomes for '
                           'Cannabis Soils (1) - Raw data 1 (1)')]
        }
        self.assertEqual(obs, exp)

        info = {
            "timeseries_type_id": 1,
            "metadata_complete": True,
            "mixs_compliant": True,
            "number_samples_collected": 25,
            "number_samples_promised": 28,
            "study_alias": "FCM",
            "study_description": "DESC",
            "study_abstract": "ABS",
            "emp_person_id": qdb.study.StudyPerson(2),
            "principal_investigator_id": qdb.study.StudyPerson(3),
            "lab_person_id": qdb.study.StudyPerson(1)
        }

        new_study = qdb.study.Study.create(qdb.user.User('*****@*****.**'),
                                           "Some New Study to get files", [1],
                                           info)

        obs = study_files_get_req('*****@*****.**', new_study.id, 1, 'FASTQ')
        exp = {
            'status':
            'success',
            'message':
            '',
            'remaining': [],
            'file_types': [('raw_barcodes', True, []),
                           ('raw_forward_seqs', True, []),
                           ('raw_reverse_seqs', False, [])],
            'num_prefixes':
            1,
            'artifacts': [(1, 'Identification of the Microbiomes for '
                           'Cannabis Soils (1) - Raw data 1 (1)')]
        }
        self.assertEqual(obs, exp)

        obs = study_files_get_req('*****@*****.**', new_study.id, 1, 'FASTQ')
        exp = {
            'status':
            'success',
            'message':
            '',
            'remaining': [],
            'file_types': [('raw_barcodes', True, []),
                           ('raw_forward_seqs', True, []),
                           ('raw_reverse_seqs', False, [])],
            'num_prefixes':
            1,
            'artifacts': []
        }
        self.assertEqual(obs, exp)

        # Create some 'sff' files
        upload_dir = qdb.util.get_mountpoint("uploads")[0][1]
        study_upload_dir = join(upload_dir, str(new_study.id))
        fps = []

        for i in range(2):
            fd, fp = mkstemp(suffix=".sff", dir=study_upload_dir)
            close(fd)
            with open(fp, 'w') as f:
                f.write('\n')
            fps.append(fp)

        self._clean_up_files.extend(fps)

        obs = study_files_get_req('*****@*****.**', new_study.id, 1, 'SFF')
        exp = {
            'status': 'success',
            'message': '',
            'remaining': [basename(fpath) for fpath in sorted(fps)],
            'file_types': [('raw_sff', True, [])],
            'num_prefixes': 0,
            'artifacts': []
        }
        self.assertEqual(obs, exp)
예제 #5
0
    def test_study_files_get_req(self):
        obs = study_files_get_req('*****@*****.**', 1, 1, 'FASTQ')
        exp = {
            'status':
            'success',
            'message':
            '',
            'remaining': ['uploaded_file.txt'],
            'file_types': [('raw_barcodes', True, []),
                           ('raw_forward_seqs', True, []),
                           ('raw_reverse_seqs', False, [])],
            'num_prefixes':
            1,
            'artifacts': [(1, 'Identification of the Microbiomes for '
                           'Cannabis Soils (1) - Raw data 1 (1)')]
        }
        self.assertEqual(obs, exp)

        obs = study_files_get_req('*****@*****.**', 1, 1, 'FASTQ')
        exp = {
            'status':
            'success',
            'message':
            '',
            'remaining': ['uploaded_file.txt'],
            'file_types': [('raw_barcodes', True, []),
                           ('raw_forward_seqs', True, []),
                           ('raw_reverse_seqs', False, [])],
            'num_prefixes':
            1,
            'artifacts': [(1, 'Identification of the Microbiomes for '
                           'Cannabis Soils (1) - Raw data 1 (1)')]
        }
        self.assertEqual(obs, exp)

        info = {
            "timeseries_type_id": 1,
            "metadata_complete": True,
            "mixs_compliant": True,
            "number_samples_collected": 25,
            "number_samples_promised": 28,
            "study_alias": "FCM",
            "study_description": "DESC",
            "study_abstract": "ABS",
            "emp_person_id": qdb.study.StudyPerson(2),
            "principal_investigator_id": qdb.study.StudyPerson(3),
            "lab_person_id": qdb.study.StudyPerson(1)
        }

        new_study = qdb.study.Study.create(qdb.user.User('*****@*****.**'),
                                           "Some New Study", [1], info)

        obs = study_files_get_req('*****@*****.**', new_study.id, 1, 'FASTQ')
        exp = {
            'status':
            'success',
            'message':
            '',
            'remaining': [],
            'file_types': [('raw_barcodes', True, []),
                           ('raw_forward_seqs', True, []),
                           ('raw_reverse_seqs', False, [])],
            'num_prefixes':
            1,
            'artifacts': [(1, 'Identification of the Microbiomes for '
                           'Cannabis Soils (1) - Raw data 1 (1)')]
        }
        self.assertEqual(obs, exp)

        obs = study_files_get_req('*****@*****.**', new_study.id, 1, 'FASTQ')
        exp = {
            'status':
            'success',
            'message':
            '',
            'remaining': [],
            'file_types': [('raw_barcodes', True, []),
                           ('raw_forward_seqs', True, []),
                           ('raw_reverse_seqs', False, [])],
            'num_prefixes':
            1,
            'artifacts': []
        }
        self.assertEqual(obs, exp)
예제 #6
0
    def test_study_files_get_req_multiple(self):
        study_id = 1
        # adding a new prep for testing
        PREP = qdb.metadata_template.prep_template.PrepTemplate
        prep_info_dict = {
            'SKB7.640196': {'run_prefix': 'test_1'},
            'SKB8.640193': {'run_prefix': 'test_2'}
        }
        prep_info = pd.DataFrame.from_dict(prep_info_dict,
                                           orient='index', dtype=str)
        pt = npt.assert_warns(
            qdb.exceptions.QiitaDBWarning, PREP.create, prep_info,
            qdb.study.Study(study_id), "Metagenomic")

        # getting the upload folder so we can test
        study_upload_dir = join(
            qdb.util.get_mountpoint("uploads")[0][1], str(study_id))

        # adding just foward per sample FASTQ to the upload folder
        filenames = ['test_1.R1.fastq.gz', 'test_2.R1.fastq.gz']
        for f in filenames:
            fpt = join(study_upload_dir, f)
            open(fpt, 'w', 0).close()
            self._clean_up_files.append(fpt)
        obs = study_files_get_req(
            '*****@*****.**', 1, pt.id, 'per_sample_FASTQ')
        exp = {
            'status': 'success', 'num_prefixes': 2, 'artifacts': [],
            'remaining': ['uploaded_file.txt'], 'message': '',
            'file_types': [
                ('raw_forward_seqs', True,
                 ['test_2.R1.fastq.gz', 'test_1.R1.fastq.gz']),
                ('raw_reverse_seqs', False, [])]}
        self.assertEqual(obs, exp)

        # let's add reverse
        filenames = ['test_1.R2.fastq.gz', 'test_2.R2.fastq.gz']
        for f in filenames:
            fpt = join(study_upload_dir, f)
            open(fpt, 'w', 0).close()
            self._clean_up_files.append(fpt)
        obs = study_files_get_req(
            '*****@*****.**', 1, pt.id, 'per_sample_FASTQ')
        exp = {'status': 'success', 'num_prefixes': 2, 'artifacts': [],
               'remaining': ['uploaded_file.txt'], 'message': '',
               'file_types': [('raw_forward_seqs', True,
                               ['test_2.R1.fastq.gz', 'test_1.R1.fastq.gz']),
                              ('raw_reverse_seqs', False,
                              ['test_2.R2.fastq.gz', 'test_1.R2.fastq.gz'])]}
        self.assertEqual(obs, exp)

        # let's an extra file that matches
        filenames = ['test_1.R3.fastq.gz']
        for f in filenames:
            fpt = join(study_upload_dir, f)
            open(fpt, 'w', 0).close()
            self._clean_up_files.append(fpt)
        obs = study_files_get_req(
            '*****@*****.**', 1, pt.id, 'per_sample_FASTQ')
        exp = {'status': 'success', 'num_prefixes': 2, 'artifacts': [],
               'remaining': ['test_1.R1.fastq.gz', 'test_1.R2.fastq.gz',
                             'test_1.R3.fastq.gz', 'uploaded_file.txt'],
               'message':  "Check these run_prefix:\n'test_1' has 3 matches.",
               'file_types': [('raw_forward_seqs', True,
                               ['test_2.R1.fastq.gz']),
                              ('raw_reverse_seqs', False,
                              ['test_2.R2.fastq.gz'])]}
        self.assertEqual(obs, exp)

        # now if we select FASTQ we have 3 columns so the extra file should go
        # to the 3rd column
        obs = study_files_get_req(
            '*****@*****.**', 1, pt.id, 'FASTQ')
        exp = {'status': 'success', 'num_prefixes': 2,
               'remaining': ['uploaded_file.txt'],
               'message': '',
               'artifacts': [(1, 'Identification of the Microbiomes for '
                                 'Cannabis Soils (1) - Raw data 1 (1)')],
               'file_types': [
                ('raw_barcodes', True,
                 ['test_2.R1.fastq.gz', 'test_1.R1.fastq.gz']),
                ('raw_forward_seqs', True,
                 ['test_2.R2.fastq.gz', 'test_1.R2.fastq.gz']),
                ('raw_reverse_seqs', False, ['test_1.R3.fastq.gz'])]}
        self.assertEqual(obs, exp)

        PREP.delete(pt.id)
예제 #7
0
    def test_study_files_get_req(self):
        obs = study_files_get_req('*****@*****.**', 1, 1, 'FASTQ')
        exp = {'status': 'success',
               'message': '',
               'remaining': ['uploaded_file.txt'],
               'file_types': [('raw_barcodes', True, []),
                              ('raw_forward_seqs', True, []),
                              ('raw_reverse_seqs', False, [])],
               'num_prefixes': 1,
               'artifacts': [(1, 'Identification of the Microbiomes for '
                                 'Cannabis Soils (1) - Raw data 1 (1)')]}
        self.assertEqual(obs, exp)

        obs = study_files_get_req('*****@*****.**', 1, 1, 'FASTQ')
        exp = {'status': 'success',
               'message': '',
               'remaining': ['uploaded_file.txt'],
               'file_types': [('raw_barcodes', True, []),
                              ('raw_forward_seqs', True, []),
                              ('raw_reverse_seqs', False, [])],
               'num_prefixes': 1,
               'artifacts': [(1, 'Identification of the Microbiomes for '
                                 'Cannabis Soils (1) - Raw data 1 (1)')]}
        self.assertEqual(obs, exp)

        info = {
            "timeseries_type_id": 1,
            "metadata_complete": True,
            "mixs_compliant": True,
            "number_samples_collected": 25,
            "number_samples_promised": 28,
            "study_alias": "FCM",
            "study_description": "DESC",
            "study_abstract": "ABS",
            "emp_person_id": qdb.study.StudyPerson(2),
            "principal_investigator_id": qdb.study.StudyPerson(3),
            "lab_person_id": qdb.study.StudyPerson(1)
        }

        new_study = qdb.study.Study.create(
            qdb.user.User('*****@*****.**'), "Some New Study", [1],
            info)

        obs = study_files_get_req('*****@*****.**', new_study.id, 1, 'FASTQ')
        exp = {'status': 'success',
               'message': '',
               'remaining': [],
               'file_types': [('raw_barcodes', True, []),
                              ('raw_forward_seqs', True, []),
                              ('raw_reverse_seqs', False, [])],
               'num_prefixes': 1,
               'artifacts': [(1, 'Identification of the Microbiomes for '
                                 'Cannabis Soils (1) - Raw data 1 (1)')]}
        self.assertEqual(obs, exp)

        obs = study_files_get_req('*****@*****.**', new_study.id, 1, 'FASTQ')
        exp = {'status': 'success',
               'message': '',
               'remaining': [],
               'file_types': [('raw_barcodes', True, []),
                              ('raw_forward_seqs', True, []),
                              ('raw_reverse_seqs', False, [])],
               'num_prefixes': 1,
               'artifacts': []}
        self.assertEqual(obs, exp)

        # Create some 'sff' files
        upload_dir = qdb.util.get_mountpoint("uploads")[0][1]
        study_upload_dir = join(upload_dir, str(new_study.id))
        fps = []

        for i in range(2):
            fd, fp = mkstemp(suffix=".sff", dir=study_upload_dir)
            close(fd)
            with open(fp, 'w') as f:
                f.write('\n')
            fps.append(fp)

        self._clean_up_files.extend(fps)

        obs = study_files_get_req('*****@*****.**', new_study.id, 1, 'SFF')
        exp = {'status': 'success',
               'message': '',
               'remaining': [basename(fpath) for fpath in sorted(fps)],
               'file_types': [('raw_sff', True, [])],
               'num_prefixes': 0,
               'artifacts': []}
        self.assertEqual(obs, exp)
예제 #8
0
    def test_study_files_get_req(self):
        obs = study_files_get_req('*****@*****.**', 1, 1, 'FASTQ')
        exp = {'status': 'success',
               'message': '',
               'remaining': ['uploaded_file.txt'],
               'file_types': [('raw_barcodes', True, []),
                              ('raw_forward_seqs', True, []),
                              ('raw_reverse_seqs', False, [])],
               'num_prefixes': 1,
               'artifacts': [(1, 'Identification of the Microbiomes for '
                                 'Cannabis Soils (1) - Raw data 1 (1)')]}
        self.assertEqual(obs, exp)

        obs = study_files_get_req('*****@*****.**', 1, 1, 'FASTQ')
        exp = {'status': 'success',
               'message': '',
               'remaining': ['uploaded_file.txt'],
               'file_types': [('raw_barcodes', True, []),
                              ('raw_forward_seqs', True, []),
                              ('raw_reverse_seqs', False, [])],
               'num_prefixes': 1,
               'artifacts': [(1, 'Identification of the Microbiomes for '
                                 'Cannabis Soils (1) - Raw data 1 (1)')]}
        self.assertEqual(obs, exp)

        info = {
            "timeseries_type_id": 1,
            "metadata_complete": True,
            "mixs_compliant": True,
            "number_samples_collected": 25,
            "number_samples_promised": 28,
            "study_alias": "FCM",
            "study_description": "DESC",
            "study_abstract": "ABS",
            "emp_person_id": qdb.study.StudyPerson(2),
            "principal_investigator_id": qdb.study.StudyPerson(3),
            "lab_person_id": qdb.study.StudyPerson(1)
        }

        new_study = qdb.study.Study.create(
            qdb.user.User('*****@*****.**'), "Some New Study", [1],
            info)

        obs = study_files_get_req('*****@*****.**', new_study.id, 1, 'FASTQ')
        exp = {'status': 'success',
               'message': '',
               'remaining': [],
               'file_types': [('raw_barcodes', True, []),
                              ('raw_forward_seqs', True, []),
                              ('raw_reverse_seqs', False, [])],
               'num_prefixes': 1,
               'artifacts': [(1, 'Identification of the Microbiomes for '
                                 'Cannabis Soils (1) - Raw data 1 (1)')]}
        self.assertEqual(obs, exp)

        obs = study_files_get_req('*****@*****.**', new_study.id, 1, 'FASTQ')
        exp = {'status': 'success',
               'message': '',
               'remaining': [],
               'file_types': [('raw_barcodes', True, []),
                              ('raw_forward_seqs', True, []),
                              ('raw_reverse_seqs', False, [])],
               'num_prefixes': 1,
               'artifacts': []}
        self.assertEqual(obs, exp)