def test_delete_analysis(self): # adding extra filepaths to make sure the delete works as expected, we # basically want 8 -> 9 -> 10 -> 12 -> 14 # -> 11 -> 13 fd, fp10 = mkstemp(suffix='_table.biom') close(fd) fd, fp11 = mkstemp(suffix='_table.biom') close(fd) fd, fp12 = mkstemp(suffix='_table.biom') close(fd) fd, fp13 = mkstemp(suffix='_table.biom') close(fd) fd, fp14 = mkstemp(suffix='_table.biom') close(fd) with biom_open(fp10, 'w') as f: et.to_hdf5(f, "test") with biom_open(fp11, 'w') as f: et.to_hdf5(f, "test") with biom_open(fp12, 'w') as f: et.to_hdf5(f, "test") with biom_open(fp13, 'w') as f: et.to_hdf5(f, "test") with biom_open(fp14, 'w') as f: et.to_hdf5(f, "test") self._clean_up_files.extend([fp10, fp11, fp12, fp13, fp14]) # copying some processing parameters a9 = Artifact(9) pp = a9.processing_parameters # 7: BIOM a10 = Artifact.create([(fp10, 7)], "BIOM", parents=[a9], processing_parameters=pp) a11 = Artifact.create([(fp11, 7)], "BIOM", parents=[a9], processing_parameters=pp) a12 = Artifact.create([(fp12, 7)], "BIOM", parents=[a10], processing_parameters=pp) Artifact.create([(fp13, 7)], "BIOM", parents=[a11], processing_parameters=pp) Artifact.create([(fp14, 7)], "BIOM", parents=[a12], processing_parameters=pp) job = self._create_job('delete_analysis', {'analysis_id': 1}) private_task(job.id) self.assertEqual(job.status, 'success') with self.assertRaises(QiitaDBUnknownIDError): Analysis(1)
def setUp(self): # Generate some files for a root artifact fd, self.fp1 = mkstemp(suffix='_seqs.fastq') close(fd) with open(self.fp1, 'w') as f: f.write("@HWI-ST753:189:D1385ACXX:1:1101:1214:1906 1:N:0:\n" "NACGTAGGGTGCAAGCGTTGTCCGGAATNA\n" "+\n" "#1=DDFFFHHHHHJJJJJJJJJJJJGII#0\n") fd, self.fp2 = mkstemp(suffix='_barcodes.fastq') close(fd) with open(self.fp2, 'w') as f: f.write("@HWI-ST753:189:D1385ACXX:1:1101:1214:1906 2:N:0:\n" "NNNCNNNNNNNNN\n" "+\n" "#############\n") self.filepaths_root = [(self.fp1, 1), (self.fp2, 3)] # Generate some files for a processed artifact fd, self.fp3 = mkstemp(suffix='_seqs.fna') close(fd) with open(self.fp3, 'w') as f: f.write(">1.sid_r4_0 M02034:17:000000000-A5U18:1:1101:15370:1394 " "1:N:0:1 orig_bc=CATGAGCT new_bc=CATGAGCT bc_diffs=0\n" "GTGTGCCAGCAGCCGCGGTAATACGTAGGG\n") self.filepaths_processed = [(self.fp3, 4)] # Generate some file for a BIOM fd, self.fp4 = mkstemp(suffix='_table.biom') with biom_open(self.fp4, 'w') as f: et.to_hdf5(f, "test") self.filepaths_biom = [(self.fp4, 7)] # Create a new prep template metadata_dict = { 'SKB8.640193': { 'center_name': 'ANL', 'primer': 'GTGCCAGCMGCCGCGGTAA', 'barcode': 'GTCCGCAAGTTA', 'run_prefix': "s_G1_L001_sequences", 'platform': 'ILLUMINA', 'instrument_model': 'Illumina MiSeq', 'library_construction_protocol': 'AAAA', 'experiment_design_description': 'BBBB' } } metadata = pd.DataFrame.from_dict(metadata_dict, orient='index', dtype=str) self.prep_template = \ qdb.metadata_template.prep_template.PrepTemplate.create( metadata, qdb.study.Study(1), "16S") self._clean_up_files = [self.fp1, self.fp2, self.fp3, self.fp4]
def setUp(self): # Generate some files for a root artifact fd, self.fp1 = mkstemp(suffix='_seqs.fastq') close(fd) with open(self.fp1, 'w') as f: f.write("@HWI-ST753:189:D1385ACXX:1:1101:1214:1906 1:N:0:\n" "NACGTAGGGTGCAAGCGTTGTCCGGAATNA\n" "+\n" "#1=DDFFFHHHHHJJJJJJJJJJJJGII#0\n") fd, self.fp2 = mkstemp(suffix='_barcodes.fastq') close(fd) with open(self.fp2, 'w') as f: f.write("@HWI-ST753:189:D1385ACXX:1:1101:1214:1906 2:N:0:\n" "NNNCNNNNNNNNN\n" "+\n" "#############\n") self.filepaths_root = [(self.fp1, 1), (self.fp2, 3)] # Generate some files for a processed artifact fd, self.fp3 = mkstemp(suffix='_seqs.fna') close(fd) with open(self.fp3, 'w') as f: f.write(">1.sid_r4_0 M02034:17:000000000-A5U18:1:1101:15370:1394 " "1:N:0:1 orig_bc=CATGAGCT new_bc=CATGAGCT bc_diffs=0\n" "GTGTGCCAGCAGCCGCGGTAATACGTAGGG\n") self.filepaths_processed = [(self.fp3, 4)] # Generate some file for a BIOM fd, self.fp4 = mkstemp(suffix='_table.biom') with biom_open(self.fp4, 'w') as f: et.to_hdf5(f, "test") self.filepaths_biom = [(self.fp4, 7)] # Create a new prep template metadata_dict = { 'SKB8.640193': {'center_name': 'ANL', 'primer': 'GTGCCAGCMGCCGCGGTAA', 'barcode': 'GTCCGCAAGTTA', 'run_prefix': "s_G1_L001_sequences", 'platform': 'ILLUMINA', 'instrument_model': 'Illumina MiSeq', 'library_construction_protocol': 'AAAA', 'experiment_design_description': 'BBBB'}} metadata = pd.DataFrame.from_dict(metadata_dict, orient='index', dtype=str) self.prep_template = \ qdb.metadata_template.prep_template.PrepTemplate.create( metadata, qdb.study.Study(1), "16S") self._clean_up_files = [self.fp1, self.fp2, self.fp3, self.fp4]
def test_post_analysis(self): fd, fp = mkstemp(suffix='_table.biom') close(fd) with biom_open(fp, 'w') as f: et.to_hdf5(f, "test") self._clean_up_files.append(fp) data = {'filepaths': dumps([(fp, 'biom')]), 'type': "BIOM", 'name': "New biom artifact", 'analysis': 1, 'data_type': '16S'} obs = self.post('/apitest/artifact/', headers=self.header, data=data) self.assertEqual(obs.code, 200) obs = loads(obs.body) self.assertEqual(obs.keys(), ['artifact']) a = qdb.artifact.Artifact(obs['artifact']) self._clean_up_files.extend([afp for _, afp, _ in a.filepaths]) self.assertEqual(a.name, "New biom artifact")
def test_download_study(self): tmp_dir = mkdtemp() self._clean_up_files.append(tmp_dir) biom_fp = join(tmp_dir, 'otu_table.biom') smr_dir = join(tmp_dir, 'sortmerna_picked_otus') log_dir = join(smr_dir, 'seqs_otus.log') tgz = join(tmp_dir, 'sortmerna_picked_otus.tgz') with biom_open(biom_fp, 'w') as f: et.to_hdf5(f, "test") makedirs(smr_dir) with open(log_dir, 'w') as f: f.write('\n') with open(tgz, 'w') as f: f.write('\n') files_biom = [(biom_fp, 'biom'), (smr_dir, 'directory'), (tgz, 'tgz')] params = Parameters.from_default_params( next(Command(3).default_parameter_sets), {'input_data': 1}) a = Artifact.create(files_biom, "BIOM", parents=[Artifact(2)], processing_parameters=params) for x in a.filepaths: self._clean_up_files.append(x['fp']) response = self.get('/download_study_bioms/1') self.assertEqual(response.code, 200) exp = ('- 1256812 /protected/processed_data/' '1_study_1001_closed_reference_otu_table.biom processed_data/' '1_study_1001_closed_reference_otu_table.biom\n' '- [0-9]* /protected/templates/1_prep_1_[0-9]*-[0-9]*.txt ' 'mapping_files/4_mapping_file.txt\n' '- 1256812 /protected/processed_data/' '1_study_1001_closed_reference_otu_table.biom processed_data/' '1_study_1001_closed_reference_otu_table.biom\n' '- [0-9]* /protected/templates/1_prep_1_[0-9]*-[0-9]*.txt ' 'mapping_files/5_mapping_file.txt\n' '- 1256812 /protected/processed_data/1_study_1001_' 'closed_reference_otu_table_Silva.biom processed_data/' '1_study_1001_closed_reference_otu_table_Silva.biom\n' '- [0-9]* /protected/templates/1_prep_1_[0-9]*-[0-9]*.txt ' 'mapping_files/6_mapping_file.txt\n' '- 1093210 /protected/BIOM/7/biom_table.biom ' 'BIOM/7/biom_table.biom\n' '- [0-9]* /protected/templates/1_prep_2_[0-9]*-[0-9]*.txt ' 'mapping_files/7_mapping_file.txt\n' '- [0-9]* /protected/BIOM/{0}/otu_table.biom ' 'BIOM/{0}/otu_table.biom\n' '- 1 /protected/BIOM/10/sortmerna_picked_otus/seqs_otus.log ' 'BIOM/{0}/sortmerna_picked_otus/seqs_otus.log\n' '- [0-9]* /protected/templates/1_prep_1_[0-9]*-[0-9]*.txt ' 'mapping_files/{0}_mapping_file.txt\n'.format(a.id)) self.assertRegex(response.body.decode('ascii'), exp) response = self.get('/download_study_bioms/200') self.assertEqual(response.code, 405) # changing user so we can test the failures BaseHandler.get_current_user = Mock( return_value=User("*****@*****.**")) response = self.get('/download_study_bioms/1') self.assertEqual(response.code, 405) a.visibility = 'public' response = self.get('/download_study_bioms/1') # returning visibility a.visibility = 'private' self.assertEqual(response.code, 200) # we should have the same files than the previous test, except artifact # and mapping file 7: position 6 and 7; thus removing 6 twice exp = exp.split('\n') exp.pop(6) exp.pop(6) exp = '\n'.join(exp) self.assertRegex(response.body.decode('ascii'), exp)
def test_download_study(self): tmp_dir = mkdtemp() self._clean_up_files.append(tmp_dir) biom_fp = join(tmp_dir, 'otu_table.biom') smr_dir = join(tmp_dir, 'sortmerna_picked_otus') log_dir = join(smr_dir, 'seqs_otus.log') tgz = join(tmp_dir, 'sortmerna_picked_otus.tgz') with biom_open(biom_fp, 'w') as f: et.to_hdf5(f, "test") makedirs(smr_dir) with open(log_dir, 'w') as f: f.write('\n') with open(tgz, 'w') as f: f.write('\n') files_biom = [(biom_fp, 'biom'), (smr_dir, 'directory'), (tgz, 'tgz')] params = Parameters.from_default_params( Command(3).default_parameter_sets.next(), {'input_data': 1}) a = Artifact.create(files_biom, "BIOM", parents=[Artifact(2)], processing_parameters=params) for _, fp, _ in a.filepaths: self._clean_up_files.append(fp) response = self.get('/download_study_bioms/1') self.assertEqual(response.code, 200) exp = ( '- 1256812 /protected/processed_data/1_study_1001_closed_' 'reference_otu_table.biom processed_data/1_study_1001_closed_' 'reference_otu_table.biom\n' '- 36615 /protected/templates/1_prep_1_qiime_[0-9]*-' '[0-9]*.txt mapping_files/4_mapping_file.txt\n' '- 1256812 /protected/processed_data/' '1_study_1001_closed_reference_otu_table.biom processed_data/' '1_study_1001_closed_reference_otu_table.biom\n' '- 36615 /protected/templates/1_prep_1_qiime_[0-9]*-' '[0-9]*.txt mapping_files/5_mapping_file.txt\n' '- 1256812 /protected/processed_data/' '1_study_1001_closed_reference_otu_table_Silva.biom processed_data' '/1_study_1001_closed_reference_otu_table_Silva.biom\n' '- 36615 /protected/templates/1_prep_1_qiime_[0-9]*-' '[0-9]*.txt mapping_files/6_mapping_file.txt\n' '- 36615 /protected/templates/1_prep_2_qiime_[0-9]*-' '[0-9]*.txt mapping_files/7_mapping_file.txt\n' '- 39752 /protected/BIOM/{0}/otu_table.biom ' 'BIOM/{0}/otu_table.biom\n' '- 1 /protected/BIOM/{0}/sortmerna_picked_otus/seqs_otus.log ' 'BIOM/{0}/sortmerna_picked_otus/seqs_otus.log\n' '- 36615 /protected/templates/1_prep_1_qiime_[0-9]*-[0-9]*.' 'txt mapping_files/{0}_mapping_file.txt\n'.format(a.id)) self.assertRegexpMatches(response.body, exp) response = self.get('/download_study_bioms/200') self.assertEqual(response.code, 405) # changing user so we can test the failures BaseHandler.get_current_user = Mock( return_value=User("*****@*****.**")) response = self.get('/download_study_bioms/1') self.assertEqual(response.code, 405) a.visibility = 'public' response = self.get('/download_study_bioms/1') self.assertEqual(response.code, 200) exp = ('- 39752 /protected/BIOM/{0}/otu_table.biom ' 'BIOM/{0}/otu_table.biom\n' '- 1 /protected/BIOM/{0}/sortmerna_picked_otus/seqs_otus.log ' 'BIOM/{0}/sortmerna_picked_otus/seqs_otus.log\n' '- 36615 /protected/templates/1_prep_1_qiime_[0-9]*-[0-9]*.' 'txt mapping_files/{0}_mapping_file.txt\n'.format(a.id)) self.assertRegexpMatches(response.body, exp)
def test_download_study(self): tmp_dir = mkdtemp() self._clean_up_files.append(tmp_dir) biom_fp = join(tmp_dir, 'otu_table.biom') smr_dir = join(tmp_dir, 'sortmerna_picked_otus') log_dir = join(smr_dir, 'seqs_otus.log') tgz = join(tmp_dir, 'sortmerna_picked_otus.tgz') with biom_open(biom_fp, 'w') as f: et.to_hdf5(f, "test") makedirs(smr_dir) with open(log_dir, 'w') as f: f.write('\n') with open(tgz, 'w') as f: f.write('\n') files_biom = [(biom_fp, 'biom'), (smr_dir, 'directory'), (tgz, 'tgz')] params = Parameters.from_default_params( Command(3).default_parameter_sets.next(), {'input_data': 1}) a = Artifact.create(files_biom, "BIOM", parents=[Artifact(2)], processing_parameters=params) for _, fp, _ in a.filepaths: self._clean_up_files.append(fp) response = self.get('/download_study_bioms/1') self.assertEqual(response.code, 200) exp = ( '- 1256812 /protected/processed_data/1_study_1001_closed_' 'reference_otu_table.biom processed_data/1_study_1001_closed_' 'reference_otu_table.biom\n' '- 36615 /protected/templates/1_prep_1_qiime_[0-9]*-' '[0-9]*.txt mapping_files/4_mapping_file.txt\n' '- 1256812 /protected/processed_data/' '1_study_1001_closed_reference_otu_table.biom processed_data/' '1_study_1001_closed_reference_otu_table.biom\n' '- 36615 /protected/templates/1_prep_1_qiime_[0-9]*-' '[0-9]*.txt mapping_files/5_mapping_file.txt\n' '- 1256812 /protected/processed_data/' '1_study_1001_closed_reference_otu_table_Silva.biom processed_data' '/1_study_1001_closed_reference_otu_table_Silva.biom\n' '- 36615 /protected/templates/1_prep_1_qiime_[0-9]*-' '[0-9]*.txt mapping_files/6_mapping_file.txt\n' '- 36615 /protected/templates/1_prep_2_qiime_[0-9]*-' '[0-9]*.txt mapping_files/7_mapping_file.txt\n' '- [0-9]* /protected/BIOM/{0}/otu_table.biom ' 'BIOM/{0}/otu_table.biom\n' '- 1 /protected/BIOM/{0}/sortmerna_picked_otus/seqs_otus.log ' 'BIOM/{0}/sortmerna_picked_otus/seqs_otus.log\n' '- 36615 /protected/templates/1_prep_1_qiime_[0-9]*-[0-9]*.' 'txt mapping_files/{0}_mapping_file.txt\n'.format(a.id)) self.assertRegexpMatches(response.body, exp) response = self.get('/download_study_bioms/200') self.assertEqual(response.code, 405) # changing user so we can test the failures BaseHandler.get_current_user = Mock( return_value=User("*****@*****.**")) response = self.get('/download_study_bioms/1') self.assertEqual(response.code, 405) a.visibility = 'public' response = self.get('/download_study_bioms/1') self.assertEqual(response.code, 200) exp = ( '- [0-9]* /protected/BIOM/{0}/otu_table.biom ' 'BIOM/{0}/otu_table.biom\n' '- 1 /protected/BIOM/{0}/sortmerna_picked_otus/seqs_otus.log ' 'BIOM/{0}/sortmerna_picked_otus/seqs_otus.log\n' '- 36615 /protected/templates/1_prep_1_qiime_[0-9]*-[0-9]*.' 'txt mapping_files/{0}_mapping_file.txt\n'.format(a.id)) self.assertRegexpMatches(response.body, exp)
def test_download_study(self): tmp_dir = mkdtemp() self._clean_up_files.append(tmp_dir) biom_fp = join(tmp_dir, 'otu_table.biom') smr_dir = join(tmp_dir, 'sortmerna_picked_otus') log_dir = join(smr_dir, 'seqs_otus.log') tgz = join(tmp_dir, 'sortmerna_picked_otus.tgz') with biom_open(biom_fp, 'w') as f: et.to_hdf5(f, "test") makedirs(smr_dir) with open(log_dir, 'w') as f: f.write('\n') with open(tgz, 'w') as f: f.write('\n') files_biom = [(biom_fp, 'biom'), (smr_dir, 'directory'), (tgz, 'tgz')] params = Parameters.from_default_params( next(Command(3).default_parameter_sets), {'input_data': 1}) a = Artifact.create(files_biom, "BIOM", parents=[Artifact(2)], processing_parameters=params) for x in a.filepaths: self._clean_up_files.append(x['fp']) response = self.get('/download_study_bioms/1') self.assertEqual(response.code, 200) exp = ( '1579715020 1256812 /protected/processed_data/1_study_1001_closed_' 'reference_otu_table.biom processed_data/1_study_1001_closed_' 'reference_otu_table.biom\n' '- [0-9]* /protected/templates/1_prep_1_qiime_[0-9]*-' '[0-9]*.txt mapping_files/4_mapping_file.txt\n' '1579715020 1256812 /protected/processed_data/' '1_study_1001_closed_reference_otu_table.biom processed_data/' '1_study_1001_closed_reference_otu_table.biom\n' '- [0-9]* /protected/templates/1_prep_1_qiime_[0-9]*-' '[0-9]*.txt mapping_files/5_mapping_file.txt\n' '1579715020 1256812 /protected/processed_data/' '1_study_1001_closed_reference_otu_table_Silva.biom processed_data' '/1_study_1001_closed_reference_otu_table_Silva.biom\n' '- [0-9]* /protected/templates/1_prep_1_qiime_[0-9]*-' '[0-9]*.txt mapping_files/6_mapping_file.txt\n' '1756512010 1093210 /protected/BIOM/7/biom_table.biom ' 'BIOM/7/biom_table.biom\n' '- [0-9]* /protected/templates/1_prep_2_qiime_[0-9]*-' '[0-9]*.txt mapping_files/7_mapping_file.txt\n' '[0-9]* [0-9]* /protected/BIOM/{0}/otu_table.biom ' 'BIOM/{0}/otu_table.biom\n' '- 1 /protected/BIOM/{0}/sortmerna_picked_otus/seqs_otus.log ' 'BIOM/{0}/sortmerna_picked_otus/seqs_otus.log\n' '- [0-9]* /protected/templates/1_prep_1_qiime_[0-9]*-[0-9]*.' 'txt mapping_files/{0}_mapping_file.txt\n'.format(a.id)) self.assertRegex(response.body.decode('ascii'), exp) response = self.get('/download_study_bioms/200') self.assertEqual(response.code, 405) # changing user so we can test the failures BaseHandler.get_current_user = Mock( return_value=User("*****@*****.**")) response = self.get('/download_study_bioms/1') self.assertEqual(response.code, 405) a.visibility = 'public' response = self.get('/download_study_bioms/1') # returning visibility a.visibility = 'private' self.assertEqual(response.code, 200) # we should have the same files than the previous test, except artifact # and mapping file 7: position 6 and 7; thus removing 6 twice exp = exp.split('\n') exp.pop(6) exp.pop(6) exp = '\n'.join(exp) self.assertRegex(response.body.decode('ascii'), exp)
def test_post(self): # no header obs = self.post('/qiita_db/artifact/', data={}) self.assertEqual(obs.code, 400) fd, fp = mkstemp(suffix='_table.biom') close(fd) # renaming samples et.update_ids( { 'S1': '1.SKB1.640202', 'S2': '1.SKD3.640198', 'S3': '1.SKM4.640180' }, inplace=True) with biom_open(fp, 'w') as f: et.to_hdf5(f, "test") self._clean_up_files.append(fp) # no job_id or prep_id data = { 'user_email': '*****@*****.**', 'artifact_type': 'BIOM', 'command_artifact_name': 'OTU table', 'files': dumps({'biom': [fp]}) } obs = self.post('/qiita_db/artifact/', headers=self.header, data=data) self.assertEqual(obs.code, 400) self.assertIn('You need to specify a job_id or a prep_id', str(obs.error)) # both job_id and prep_id defined data['job_id'] = 'e5609746-a985-41a1-babf-6b3ebe9eb5a9' data['prep_id'] = 'prep_id' obs = self.post('/qiita_db/artifact/', headers=self.header, data=data) self.assertEqual(obs.code, 400) self.assertIn('You need to specify only a job_id or a prep_id', str(obs.error)) # make sure that all the plugins are on qdb.util.activate_or_update_plugins(update=True) # tests success by inserting a new artifact into an existing job original_job = qdb.processing_job.ProcessingJob(data['job_id']) input_artifact = original_job.input_artifacts[0] original_children = input_artifact.children self.assertEqual(len(original_children), 3) # send the new data del data['prep_id'] obs = self.post('/qiita_db/artifact/', headers=self.header, data=data) jid = obs.body.decode("utf-8") job = qdb.processing_job.ProcessingJob(jid) while job.status not in ('error', 'success'): sleep(0.5) # now the original job should have 4 children and make sure they have # the same parent and parameters children = input_artifact.children new_children = list(set(children) - set(original_children))[0] self.assertEqual(len(children), 4) for c in children[1:]: self.assertCountEqual(children[0].processing_parameters.values, c.processing_parameters.values) self.assertEqual(children[0].parents, c.parents) # making sure the new artifact is part of the descendants, which is a # different method and usage than children method self.assertIn(new_children, input_artifact.descendants.nodes) # now let's test adding an artifact directly to a new prep new_prep = qdb.metadata_template.prep_template.PrepTemplate.create( pd.DataFrame({ 'new_col': { '1.SKB1.640202': 1, '1.SKD3.640198': 2, '1.SKM4.640180': 3 } }), qdb.study.Study(1), '16S') fd, fp = mkstemp(suffix='_table.biom') close(fd) with biom_open(fp, 'w') as f: et.to_hdf5(f, "test") self._clean_up_files.append(fp) data = { 'user_email': '*****@*****.**', 'artifact_type': 'BIOM', 'prep_id': new_prep.id, 'files': dumps({'biom': [fp]}) } obs = self.post('/qiita_db/artifact/', headers=self.header, data=data) jid = obs.body.decode("utf-8") job = qdb.processing_job.ProcessingJob(jid) while job.status not in ('error', 'success'): sleep(0.5) self.assertIsNotNone(new_prep.artifact)