def abundance_filter_sample( demux_path_str: SingleLanePerSampleSingleEndFastqDirFmt, sample_id: str, new_demux_path_str: SingleLanePerSampleSingleEndFastqDirFmt): print("Commencing abundance-filtering for sample {}".format(sample_id)) demux = SingleLanePerSampleSingleEndFastqDirFmt(demux_path_str, mode='r') new_demux = SingleLanePerSampleSingleEndFastqDirFmt(new_demux_path_str, mode='r') stats_dict = {} stats_dict['sample-id'] = sample_id counts_series = sample_counts_series(demux, sample_id) stats_dict['n_input_seqs'] = counts_series.sum() threshold = abundance_filter_threshold_Wang_et_al(counts_series) stats_dict['threshold'] = threshold abundance_filtered_series = counts_series[counts_series > threshold].copy() n_seqs_kept = abundance_filtered_series.sum() stats_dict['n_seqs_kept'] = n_seqs_kept n_seqs_unique_kept = len(abundance_filtered_series.index) stats_dict['n_seqs_unique_kept'] = n_seqs_unique_kept demux_metadata_view = demux.metadata.view(YamlFormat) with open(str(demux_metadata_view)) as demux_metadata_fh: demux_metadata_dict = yaml.load(demux_metadata_fh) phred_offset = demux_metadata_dict['phred-offset'] new_fastqgz_path = new_demux.sequences.path_maker(sample_id=sample_id, barcode_id=1, lane_number=1, read_number=1) #skip writing into the gzipped file below if sequences = 0 if stats_dict['n_seqs_kept'] == 0: return str(new_fastqgz_path), stats_dict with gzip.open(str(new_fastqgz_path), mode='w') as writer: for seq in return_fastq_seqs_for_sample(demux, sample_id): if str(seq) in abundance_filtered_series.index: seq_strIO = StringIO() skbio.io.write(seq, format='fastq', phred_offset=phred_offset, into=seq_strIO) writer.write(bytes(seq_strIO.read(), encoding="UTF-8")) return str(new_fastqgz_path), stats_dict
def test_slanepsample_single_end_fastq_dir_fmt_validate_positive(self): filenames = ('single_end_data/MANIFEST', 'metadata.yml', 'Human-Kneecap_S1_L001_R1_001.fastq.gz') for filename in filenames: filepath = self.get_data_path(filename) shutil.copy(filepath, self.temp_dir.name) format = SingleLanePerSampleSingleEndFastqDirFmt(self.temp_dir.name, mode='r') format.validate()
def test_slanepsample_single_end_fastq_dir_fmt_validate_positive(self): filenames = ('single_end_data/MANIFEST', 'metadata.yml', 'Human-Kneecap_S1_L001_R1_001.fastq.gz') for filename in filenames: filepath = self.get_data_path(filename) shutil.copy(filepath, self.temp_dir.name) format = SingleLanePerSampleSingleEndFastqDirFmt( self.temp_dir.name, mode='r') format.validate()
def test_slanepsample_single_end_fastq_dir_fmt_validate_negative(self): filenames = ('single_end_data/MANIFEST', 'metadata.yml', 'not-fastq.fastq.gz') for filename in filenames: filepath = self.get_data_path(filename) shutil.copy(filepath, self.temp_dir.name) format = SingleLanePerSampleSingleEndFastqDirFmt(self.temp_dir.name, mode='r') with self.assertRaisesRegex(ValueError, 'SingleLanePerSampleSingle'): format.validate()
def test_slanepsample_single_end_fastq_dir_fmt_validate_negative(self): filenames = ('single_end_data/MANIFEST', 'metadata.yml', 'not-fastq.fastq.gz') for filename in filenames: filepath = self.get_data_path(filename) shutil.copy(filepath, self.temp_dir.name) format = SingleLanePerSampleSingleEndFastqDirFmt( self.temp_dir.name, mode='r') with self.assertRaisesRegex(ValueError, 'SingleLanePerSampleSingle'): format.validate()
def test_slanepsample_single_end_fastq_dir_fmt_validate_bad_paired(self): filenames = ('paired_end_data/MANIFEST', 'metadata.yml', 'Human-Kneecap_S1_L001_R1_001.fastq.gz', 'paired_end_data/Human-Kneecap_S1_L001_R2_001.fastq.gz') for filename in filenames: filepath = self.get_data_path(filename) shutil.copy(filepath, self.temp_dir.name) format = SingleLanePerSampleSingleEndFastqDirFmt( self.temp_dir.name, mode='r') with self.assertRaisesRegex(ValidationError, 'Forward and reverse'): format.validate()
def test_slanepsample_single_end_fastq_dir_fmt_validate_bad_paired(self): filenames = ('paired_end_data/MANIFEST', 'metadata.yml', 'Human-Kneecap_S1_L001_R1_001.fastq.gz', 'paired_end_data/Human-Kneecap_S1_L001_R2_001.fastq.gz') for filename in filenames: filepath = self.get_data_path(filename) shutil.copy(filepath, self.temp_dir.name) format = SingleLanePerSampleSingleEndFastqDirFmt(self.temp_dir.name, mode='r') with self.assertRaisesRegex(ValidationError, 'Forward and reverse'): format.validate()
def test_write_metadata(): results = SingleLanePerSampleSingleEndFastqDirFmt() _itsxpress._write_metadata(results) path = results.path metadata = os.path.join(str(path), "metadata.yml") with open(metadata, "rt") as fn: eq_("{phred-offset: 33}", fn.readline().replace("\n", ""))
def setUp(self): super().setUp() data_single = SingleLanePerSampleSingleEndFastqDirFmt( self.get_data_path('filter_samples_single_end/dir_fmt'), mode='r') self.sample_single = _PlotQualView(data_single, False) self.manifest_single = data_single.manifest.view(pd.DataFrame) self.md_single_all = Metadata.load( self.get_data_path('filter_samples_single_end/filter_all.tsv')) self.md_single_subset = Metadata.load( self.get_data_path('filter_samples_single_end/filter_subset.tsv')) self.md_single_none = Metadata.load( self.get_data_path('filter_samples_single_end/filter_none.tsv')) data_paired = SingleLanePerSamplePairedEndFastqDirFmt( self.get_data_path('filter_samples_paired_end/dir_fmt'), mode='r') self.sample_paired = _PlotQualView(data_paired, True) self.manifest_paired = data_paired.manifest.view(pd.DataFrame) self.md_paired_all = Metadata.load( self.get_data_path('filter_samples_single_end/filter_all.tsv')) self.md_paired_subset = Metadata.load( self.get_data_path('filter_samples_single_end/filter_subset.tsv')) self.md_paired_none = Metadata.load( self.get_data_path('filter_samples_single_end/filter_none.tsv'))
def test_underscore_samples(self): self.demux_seqs = SingleLanePerSampleSingleEndFastqDirFmt( self.get_data_path('underscore_samples'), 'r') with open(self.get_data_path('expected/underscore-samples.tsv')) as fh: exp_table = biom.Table.from_tsv(fh, None, None, lambda x: x) exp_rep_seqs = list( skbio.io.read( self.get_data_path('expected/underscore-samples.fasta'), 'fasta', constructor=skbio.DNA)) for seq in exp_rep_seqs: del seq.metadata['description'] exp_md = qiime2.Metadata.load( self.get_data_path('expected/underscore-samples-stats.tsv')) # Historical NOTE: default used to be `pooled`, so the data still # expects that. Since this is only testing underscores, it shouldn't # matter much and serves as a regression test to boot. table, rep_seqs, md = denoise_single(self.demux_seqs, 100, chimera_method='pooled') self.assertEqual(table, exp_table) self.assertEqual(_sort_seqs(rep_seqs), _sort_seqs(exp_rep_seqs)) self.assertEqual(md, exp_md)
def abundance_filter_pool( sequences: SingleLanePerSampleSingleEndFastqDirFmt, threads: int ) -> (SingleLanePerSampleSingleEndFastqDirFmt, pd.DataFrame): list_of_stats_dicts = [] result = SingleLanePerSampleSingleEndFastqDirFmt() sample_ids = return_sample_ids(sequences) with Pool(processes=threads) as pool: iterable = [(str(sequences), sample_id, str(result)) for sample_id in sample_ids] sample_results = pool.starmap(abundance_filter_sample, iterable) for fastqgz_pth_str, stats_dict in sample_results: list_of_stats_dicts.append(stats_dict) stats_df = pd.DataFrame(list_of_stats_dicts) stats_df = stats_df[STATS_COLUMNS] abundance_filtered_result = finalize_result(sequences, result, stats_df) return abundance_filtered_result, stats_df
def setUp(self): super().setUp() self.demux_seqs = SingleLanePerSampleSingleEndFastqDirFmt( self.get_data_path('sample_seqs_other'), 'r') self.ref_ar = Artifact.load( self.get_data_path('../../assets/test_reference.qza')) self.ref = self.ref_ar.view(DNAFASTAFormat)
def test_integer_ids(self): int_seqs = SingleLanePerSampleSingleEndFastqDirFmt( self.get_data_path('sample_seqs_integers'), 'r') obs_tab, _, stats = denoise_16S(int_seqs, 100, sample_stats=True) self.assertEqual(set(obs_tab.ids()), {'100', '101', '103', '104'}) self.assertEqual(set(stats.index.values), {'100', '101', '103', '104'})
def setUp(self): super().setUp() self.fastq_fp = \ self.get_data_path('forward.fastq.gz') self.fastq = FastqGzFormat(self.fastq_fp, mode='r') self.seqs_dir_fmt = MultiplexedSingleEndBarcodeInSequenceDirFmt() self.seqs_dir_fmt.file.write_data(self.fastq, FastqGzFormat) self.barcode_series = pd.Series(['A', 'G'], index=['sample_a', 'sample_b']) self.per_sample_dir_fmt = SingleLanePerSampleSingleEndFastqDirFmt() self.untrimmed_dir_fmt = MultiplexedSingleEndBarcodeInSequenceDirFmt()
def test_underscore_samples(self): self.demux_seqs = SingleLanePerSampleSingleEndFastqDirFmt( self.get_data_path('underscore_samples'), 'r') with open(self.get_data_path('expected/underscore-samples.tsv')) as fh: exp_table = biom.Table.from_tsv(fh, None, None, lambda x: x) exp_rep_seqs = list( skbio.io.read( self.get_data_path('expected/underscore-samples.fasta'), 'fasta', constructor=skbio.DNA)) for seq in exp_rep_seqs: del seq.metadata['description'] table, rep_seqs = denoise_single(self.demux_seqs, 100) self.assertEqual(table, exp_table) self.assertEqual(_sort_seqs(rep_seqs), _sort_seqs(exp_rep_seqs))
def test_single_sample_multiple_files(self): # Note, this case came up on the QIIME 2 Forum, due to a user running # `summarize` with demuxed sequences that all had the same Sample ID # in the internal MANIFEST file. With versions of seaborn greater than # 0.8.0 this is no longer a problem, but on prior versions, the user # would see the following error: # TypeError: len() of unsized object files = [ 'sample1_S0_L001_R1_001.fastq.gz', 'sample2_S0_L001_R1_001.fastq.gz', 'sample3_S0_L001_R1_001.fastq.gz', 'MANIFEST', 'metadata.yml' ] for f in files: shutil.copy( self.get_data_path('single_sample_multiple_files/%s' % f), self.temp_dir.name) demux_data = SingleLanePerSampleSingleEndFastqDirFmt( self.temp_dir.name, mode='r') with tempfile.TemporaryDirectory() as output_dir: # TODO: Remove _PlotQualView wrapper result = summarize(output_dir, _PlotQualView(demux_data, paired=False), n=1) self.assertTrue(result is None) index_fp = os.path.join(output_dir, 'overview.html') self.assertTrue(os.path.exists(index_fp)) self.assertTrue(os.path.getsize(index_fp) > 0) csv_fp = os.path.join(output_dir, 'per-sample-fastq-counts.csv') self.assertTrue(os.path.exists(csv_fp)) self.assertTrue(os.path.getsize(csv_fp) > 0) pdf_fp = os.path.join(output_dir, 'demultiplex-summary.pdf') self.assertTrue(os.path.exists(pdf_fp)) png_fp = os.path.join(output_dir, 'demultiplex-summary.png') self.assertTrue(os.path.exists(png_fp)) with open(index_fp, 'r') as fh: html = fh.read() self.assertIn('<td>Minimum:</td><td>1</td>', html) self.assertIn('<td>Maximum:</td><td>1</td>', html)
def abundance_filter_single_thread( sequences: SingleLanePerSampleSingleEndFastqDirFmt ) -> (SingleLanePerSampleSingleEndFastqDirFmt, pd.DataFrame): list_of_stats_dicts = [] result = SingleLanePerSampleSingleEndFastqDirFmt() sample_ids = return_sample_ids(sequences) for sample_id in sample_ids: sample_result = abundance_filter_sample(demux_path_str=str(sequences), sample_id=sample_id, new_demux_path_str=str(result)) fastqgz_pth_str, stats_dict = sample_result[0], sample_result[1] list_of_stats_dicts.append(stats_dict) stats_df = pd.DataFrame(list_of_stats_dicts) stats_df = stats_df[STATS_COLUMNS] abundance_filtered_result = finalize_result(sequences, result, stats_df) return abundance_filtered_result, stats_df
def setUp(self): super().setUp() self.demux_seqs = SingleLanePerSampleSingleEndFastqDirFmt( self.get_data_path('single-end'), mode='r') self.trimmed_seqs = CasavaOneEightSingleLanePerSampleDirFmt()
TEST_DATA_PAF = SingleLanePerSamplePairedEndFastqDirFmt(TEST_FILE_PAF, "r") # Test info 4 TEST_FILE_OUT = os.path.join(TEST_DIR, "test_data", "out", "d9955749-00d5-44ae-a628-4b2da43000e1", "data") TEST_DATA_OUT = SingleLanePerSamplePairedEndFastqDirFmt(TEST_FILE_OUT, "r") # Test info 5 TEST_FILE_SINGLEOUT = os.path.join(TEST_DIR, "test_data", "singleOut", "75aea4f5-f10e-421e-91d2-feda9fe7b2e1", "data") TEST_DATA_SINGLEOUT = SingleLanePerSamplePairedEndFastqDirFmt( TEST_FILE_SINGLEOUT, "r") # Test info 6 TEST_FILE_SINGLEIN = os.path.join(TEST_DIR, "test_data", "singleIn", "cfd0e65b-05fb-4329-9618-15ecd0aec9b3", "data") TEST_DATA_SINGLEIN = SingleLanePerSampleSingleEndFastqDirFmt( TEST_FILE_SINGLEIN, "r") # Test artifact1 ARTIFACT_TYPE_P = "SampleData[PairedEndSequencesWithQuality]" # Test artifact2 ARTIFACT_TYPE_S = "SampleData[SequencesWithQuality]" def test_taxa_prefix_to_taxa(): exp1 = _itsxpress._taxa_prefix_to_taxa(taxa_prefix="A") eq_(exp1, "Alveolata") class SetFastqsAndCheckTests(unittest.TestCase): def test_single(self): samples = TEST_DATA_SINGLEIN.manifest.view(pd.DataFrame) for sample in samples.itertuples():
def setUp(self): super().setUp() self.demux_seqs = SingleLanePerSampleSingleEndFastqDirFmt( self.get_data_path('sample_seqs_single'), 'r')
def setUp(self): super().setUp() # Reusing the single-end reads for this test suite self.demux_seqs = SingleLanePerSampleSingleEndFastqDirFmt( self.get_data_path('sample_seqs_single'), 'r')
def _join_pairs_w_command_output( demultiplexed_seqs: SingleLanePerSamplePairedEndFastqDirFmt, truncqual: int = _jp_defaults['truncqual'], minlen: int = _jp_defaults['minlen'], maxns: int = _jp_defaults['maxns'], allowmergestagger: bool = _jp_defaults['allowmergestagger'], minovlen: int = _jp_defaults['minovlen'], maxdiffs: int = _jp_defaults['maxdiffs'], minmergelen: int = _jp_defaults['minmergelen'], maxmergelen: int = _jp_defaults['maxmergelen'], maxee: float = _jp_defaults['maxee'], qmin: int = _jp_defaults['qmin'], qminout: int = _jp_defaults['qminout'], qmax: int = _jp_defaults['qmax'], qmaxout: int = _jp_defaults['qmaxout'], ) -> (List[str], SingleLanePerSampleSingleEndFastqDirFmt): # this function exists only to simplify unit testing result = SingleLanePerSampleSingleEndFastqDirFmt() manifest = pd.read_csv(os.path.join(str(demultiplexed_seqs), demultiplexed_seqs.manifest.pathspec), header=0, comment='#') manifest.filename = manifest.filename.apply( lambda x: os.path.join(str(demultiplexed_seqs), x)) phred_offset = yaml.load( open( os.path.join( str(demultiplexed_seqs), demultiplexed_seqs.metadata.pathspec)))['phred-offset'] id_to_fps = manifest.pivot(index='sample-id', columns='direction', values='filename') output_manifest = FastqManifestFormat() output_manifest_fh = output_manifest.open() output_manifest_fh.write('sample-id,filename,direction\n') output_manifest_fh.write('# direction is not meaningful in this file ' 'as these\n') output_manifest_fh.write('# data may be derived from forward, reverse, ' 'or \n') output_manifest_fh.write('# joined reads\n') for i, (sample_id, (fwd_fp, rev_fp)) in enumerate(id_to_fps.iterrows()): # The barcode id, lane number and read number are not relevant # here. We might ultimately want to use a dir format other than # SingleLanePerSampleSingleEndFastqDirFmt which doesn't care # about this information. Similarly, the direction of the read # isn't relevant here anymore. path = result.sequences.path_maker(sample_id=sample_id, barcode_id=i, lane_number=1, read_number=1) uncompressed_path = str(path).strip('.gz') cmd = [ 'vsearch', '--fastq_mergepairs', fwd_fp, '--reverse', rev_fp, '--fastqout', uncompressed_path, '--fastq_ascii', str(phred_offset), '--fastq_minlen', str(minlen), '--fastq_minovlen', str(minovlen), '--fastq_maxdiffs', str(maxdiffs), '--fastq_qmin', str(qmin), '--fastq_qminout', str(qminout), '--fastq_qmax', str(qmax), '--fastq_qmaxout', str(qmaxout), ] if truncqual is not None: cmd += ['--fastq_truncqual', str(truncqual)] if maxns is not None: cmd += ['--fastq_maxns', str(maxns)] if minmergelen is not None: cmd += ['--fastq_minmergelen', str(minmergelen)] if maxmergelen is not None: cmd += ['--fastq_maxmergelen', str(maxmergelen)] if maxee is not None: cmd += ['--fastq_maxee', str(maxee)] if allowmergestagger: cmd.append('--fastq_allowmergestagger') run_command(cmd) run_command(['gzip', uncompressed_path]) output_manifest_fh.write('%s,%s,%s\n' % (sample_id, path.name, 'forward')) output_manifest_fh.close() result.manifest.write_data(output_manifest, FastqManifestFormat) metadata = YamlFormat() metadata.path.write_text(yaml.dump({'phred-offset': phred_offset})) result.metadata.write_data(metadata, YamlFormat) return cmd, result
def main(per_sample_sequences: _SingleLanePerSampleFastqDirFmt, threads: int, taxa: str, region: str, paired: bool, cluster_id: float): """The main communication between the plugin and the ITSxpress program. Args: per_sample_sequences (SingleLanePerSampleSingleEndFastqDirFmt): The SingleLanePerSampleSingleEndFastqDirFmt type of the input. threads (int) : The number of threads to use. taxa (str): The taxa to be used for the search. region (str) : The region to be used for the search. cluster_id (float):The percent identity for clustering reads, set to 1 for exact dereplication. Returns: (SingleLanePerSampleSingleEndFastqDirFmt): The SingleLanePerSampleSingleEndFastqDirFmt type of the output. Raises: ValueError1: hmmsearch error. """ #Seeing if cluter_id is equal to 1 # Finding the artifact type. artifact_type = _view_artifact_type( per_sample_sequence=per_sample_sequences) # Setting the taxa taxa = _taxa_prefix_to_taxa(taxa) # Writing the manifest for the output qza manifest = FastqManifestFormat() manifest_fn = manifest.open() manifest_fn.write('sample-id,filename,direction\n') # Getting the sequences from the manifest sequences, single_end = _fastq_id_maker( per_sample_sequences=per_sample_sequences, artifact_type=artifact_type) barcode = 0 # Creating result dir if paired: results = SingleLanePerSamplePairedEndFastqDirFmt() else: results = SingleLanePerSampleSingleEndFastqDirFmt() # Running the for loop for each sample for sequence in sequences: # writing fastqs and there attributes and checking the files sequence_id, sobj = _set_fastqs_and_check( per_sample_sequences=per_sample_sequences, artifact_type=artifact_type, sequence=sequence, single_end=single_end, threads=threads) # Deduplicate if math.isclose(cluster_id, 1, rel_tol=1e-05): sobj.deduplicate(threads=threads) else: sobj.cluster(threads=threads, cluster_id=cluster_id) try: # HMMSearch for ITS regions hmmfile = os.path.join(ROOT_DIR, "ITSx_db", "HMMs", taxa_dict[taxa]) sobj._search(hmmfile=hmmfile, threads=threads) except (ModuleNotFoundError, FileNotFoundError, NotADirectoryError): raise ValueError( "hmmsearch was not found, make sure HMMER3 is installed and executable" ) # Parse HMMseach output. its_pos = itsxpress.ItsPosition(domtable=sobj.dom_file, region=region) # Create deduplication object. dedup_obj = itsxpress.Dedup(uc_file=sobj.uc_file, rep_file=sobj.rep_file, seq_file=sobj.seq_file, fastq=sobj.r1, fastq2=sobj.fastq2) path_forward = results.sequences.path_maker(sample_id=sequence_id, barcode_id=barcode, lane_number=1, read_number=1) path_reverse = results.sequences.path_maker(sample_id=sequence_id, barcode_id=barcode, lane_number=1, read_number=2) manifest_fn.write("{},{},forward\n".format(sequence_id, path_forward.name)) # Create trimmed sequences. if paired: dedup_obj.create_paired_trimmed_seqs(str(path_forward), str(path_reverse), gzipped=True, itspos=its_pos) else: dedup_obj.create_trimmed_seqs(str(path_forward), gzipped=True, itspos=its_pos) # Deleting the temp files. shutil.rmtree(sobj.tempdir) # Adding one to the barcode barcode += 1 # Writing out the results. manifest_fn.close() _write_metadata(results=results) results.manifest.write_data(manifest, FastqManifestFormat) return results
def test_integer_ids_with_underscores(self): bad_seqs = SingleLanePerSampleSingleEndFastqDirFmt( self.get_data_path('sample_seqs_integers_underscore'), 'r') with self.assertRaisesRegex(ValueError, 'Deblur cannot.*100_100.'): denoise_16S(bad_seqs, 100)
def emp_single( seqs: BarcodeSequenceFastqIterator, barcodes: qiime2.MetadataCategory, rev_comp_barcodes: bool = False, rev_comp_mapping_barcodes: bool = False ) -> SingleLanePerSampleSingleEndFastqDirFmt: result = SingleLanePerSampleSingleEndFastqDirFmt() barcode_map, barcode_len = _make_barcode_map(barcodes, rev_comp_mapping_barcodes) manifest = FastqManifestFormat() manifest_fh = manifest.open() manifest_fh.write('sample-id,filename,direction\n') manifest_fh.write('# direction is not meaningful in this file as these\n') manifest_fh.write('# data may be derived from forward, reverse, or \n') manifest_fh.write('# joined reads\n') per_sample_fastqs = {} for barcode_record, sequence_record in seqs: barcode_read = barcode_record[1] if rev_comp_barcodes: barcode_read = str(skbio.DNA(barcode_read).reverse_complement()) barcode_read = barcode_read[:barcode_len] try: sample_id = barcode_map[barcode_read] except KeyError: # TODO: this should ultimately be logged, but we don't currently # have support for that. continue if sample_id not in per_sample_fastqs: # The barcode id, lane number and read number are not relevant # here. We might ultimately want to use a dir format other than # SingleLanePerSampleSingleEndFastqDirFmt which doesn't care # about this information. Similarly, the direction of the read # isn't relevant here anymore. barcode_id = len(per_sample_fastqs) + 1 path = result.sequences.path_maker(sample_id=sample_id, barcode_id=barcode_id, lane_number=1, read_number=1) _maintain_open_fh_count(per_sample_fastqs) per_sample_fastqs[sample_id] = gzip.open(str(path), mode='a') manifest_fh.write('%s,%s,%s\n' % (sample_id, path.name, 'forward')) if per_sample_fastqs[sample_id].closed: _maintain_open_fh_count(per_sample_fastqs) per_sample_fastqs[sample_id] = gzip.open( per_sample_fastqs[sample_id].name, mode='a') fastq_lines = '\n'.join(sequence_record) + '\n' fastq_lines = fastq_lines.encode('utf-8') per_sample_fastqs[sample_id].write(fastq_lines) if len(per_sample_fastqs) == 0: raise ValueError('No sequences were mapped to samples. Check that ' 'your barcodes are in the correct orientation (see ' 'the rev_comp_barcodes and/or ' 'rev_comp_mapping_barcodes options).') for fh in per_sample_fastqs.values(): fh.close() manifest_fh.close() result.manifest.write_data(manifest, FastqManifestFormat) _write_metadata_yaml(result) return result
def q_score(demux: SingleLanePerSampleSingleEndFastqDirFmt, min_quality: int = _default_params['min_quality'], quality_window: int = _default_params['quality_window'], min_length_fraction: float = _default_params['min_length_fraction'], max_ambiguous: int = _default_params['max_ambiguous']) \ -> (SingleLanePerSampleSingleEndFastqDirFmt, pd.DataFrame): result = SingleLanePerSampleSingleEndFastqDirFmt() manifest = FastqManifestFormat() manifest_fh = manifest.open() manifest_fh.write('sample-id,filename,direction\n') manifest_fh.write('# direction is not meaningful in this file as these\n') manifest_fh.write('# data may be derived from forward, reverse, or \n') manifest_fh.write('# joined reads\n') log_records_truncated_counts = {} log_records_max_ambig_counts = {} log_records_tooshort_counts = {} log_records_totalread_counts = {} log_records_totalkept_counts = {} metadata_view = demux.metadata.view(YamlFormat).open() phred_offset = yaml.load(metadata_view)['phred-offset'] demux_manifest = demux.manifest.view(demux.manifest.format) demux_manifest = pd.read_csv(demux_manifest.open(), dtype=str) demux_manifest.set_index('filename', inplace=True) iterator = demux.sequences.iter_views(FastqGzFormat) for bc_id, (fname, fp) in enumerate(iterator): sample_id = demux_manifest.loc[str(fname)]['sample-id'] log_records_truncated_counts[sample_id] = 0 log_records_max_ambig_counts[sample_id] = 0 log_records_tooshort_counts[sample_id] = 0 log_records_totalread_counts[sample_id] = 0 log_records_totalkept_counts[sample_id] = 0 # per q2-demux, barcode ID, lane number and read number are not # relevant here path = result.sequences.path_maker(sample_id=sample_id, barcode_id=bc_id, lane_number=1, read_number=1) # we do not open a writer by default in the event that all sequences # for a sample are filtered out; an empty fastq file is not a valid # fastq file. writer = None for sequence_record in _read_fastq_seqs(str(fp), phred_offset): log_records_totalread_counts[sample_id] += 1 # determine the length of the runs below quality threshold # NOTE: QIIME 1.x used <= the quality threshold and the parameter # -q was interpreted as the maximum unacceptable PHRED score. In # QIIME 2.x, we're now interpreting this as the minimum # acceptable score. qual_below_threshold = sequence_record[4] < min_quality run_starts, run_lengths = _runs_of_ones(qual_below_threshold) bad_windows = np.argwhere(run_lengths > quality_window) # if there is a run of sufficient size, truncate it if bad_windows.size > 0: log_records_truncated_counts[sample_id] += 1 full_length = len(sequence_record[1]) sequence_record = _truncate(sequence_record, run_starts[bad_windows[0]][0]) trunc_length = len(sequence_record[1]) # do not keep the read if it is too short following truncation if round(trunc_length / full_length, 3) <= min_length_fraction: log_records_tooshort_counts[sample_id] += 1 continue # do not keep the read if there are too many ambiguous bases if sequence_record[1].count('N') > max_ambiguous: log_records_max_ambig_counts[sample_id] += 1 continue fastq_lines = '\n'.join(sequence_record[:4]) + '\n' fastq_lines = fastq_lines.encode('utf-8') if writer is None: writer = gzip.open(str(path), mode='w') writer.write(fastq_lines) log_records_totalkept_counts[sample_id] += 1 if writer is not None: manifest_fh.write('%s,%s,%s\n' % (sample_id, path.name, 'forward')) writer.close() if set(log_records_totalkept_counts.values()) == {0, }: raise ValueError("All sequences from all samples were filtered out. " "The parameter choices may be too stringent for the " "data.") manifest_fh.close() result.manifest.write_data(manifest, FastqManifestFormat) metadata = YamlFormat() metadata.path.write_text(yaml.dump({'phred-offset': phred_offset})) result.metadata.write_data(metadata, YamlFormat) columns = ['sample-id', 'total-input-reads', 'total-retained-reads', 'reads-truncated', 'reads-too-short-after-truncation', 'reads-exceeding-maximum-ambiguous-bases'] stats = [] for id_, _ in sorted(log_records_truncated_counts.items()): stats.append([id_, log_records_totalread_counts[id_], log_records_totalkept_counts[id_], log_records_truncated_counts[id_], log_records_tooshort_counts[id_], log_records_max_ambig_counts[id_]]) stats = pd.DataFrame(stats, columns=columns).set_index('sample-id') return result, stats
def emp_single( seqs: BarcodeSequenceFastqIterator, barcodes: qiime2.CategoricalMetadataColumn, golay_error_correction: bool = True, rev_comp_barcodes: bool = False, rev_comp_mapping_barcodes: bool = False, ignore_description_mismatch: bool = False ) -> (SingleLanePerSampleSingleEndFastqDirFmt, ErrorCorrectionDetailsFmt): seqs.ignore_description_mismatch = ignore_description_mismatch result = SingleLanePerSampleSingleEndFastqDirFmt() barcode_map, barcode_len = _make_barcode_map(barcodes, rev_comp_mapping_barcodes) if golay_error_correction: decoder = GolayDecoder() manifest = FastqManifestFormat() manifest_fh = manifest.open() manifest_fh.write('sample-id,filename,direction\n') manifest_fh.write('# direction is not meaningful in this file as these\n') manifest_fh.write('# data may be derived from forward, reverse, or \n') manifest_fh.write('# joined reads\n') per_sample_fastqs = {} ec_details_fmt = ErrorCorrectionDetailsFmt() ec_details = ECDetails(ec_details_fmt) for i, (barcode_record, sequence_record) in enumerate(seqs, start=1): barcode_read = barcode_record[1] if rev_comp_barcodes: barcode_read = str(skbio.DNA(barcode_read).reverse_complement()) raw_barcode_read = barcode_read[:barcode_len] if golay_error_correction: # A three bit filter is implicitly used by the decoder. See Hamady # and Knight 2009 Genome Research for the justification: # # https://genome.cshlp.org/content/19/7/1141.full # # Specifically that "...Golay codes of 12 bases can correct all # triple-bit errors and detect all quadruple-bit errors." barcode_read, ecc_errors = decoder.decode(raw_barcode_read) golay_stats = [barcode_read, ecc_errors] else: barcode_read = raw_barcode_read golay_stats = [None, None] sample_id = barcode_map.get(barcode_read) record = [ f'record-{i}', sample_id, barcode_record[0], raw_barcode_read, ] ec_details.write(record + golay_stats) if sample_id is None: continue if sample_id not in per_sample_fastqs: # The barcode id, lane number and read number are not relevant # here. We might ultimately want to use a dir format other than # SingleLanePerSampleSingleEndFastqDirFmt which doesn't care # about this information. Similarly, the direction of the read # isn't relevant here anymore. barcode_id = len(per_sample_fastqs) + 1 path = result.sequences.path_maker(sample_id=sample_id, barcode_id=barcode_id, lane_number=1, read_number=1) _maintain_open_fh_count(per_sample_fastqs) per_sample_fastqs[sample_id] = gzip.open(str(path), mode='a') manifest_fh.write('%s,%s,%s\n' % (sample_id, path.name, 'forward')) if per_sample_fastqs[sample_id].closed: _maintain_open_fh_count(per_sample_fastqs) per_sample_fastqs[sample_id] = gzip.open( per_sample_fastqs[sample_id].name, mode='a') fastq_lines = '\n'.join(sequence_record) + '\n' fastq_lines = fastq_lines.encode('utf-8') per_sample_fastqs[sample_id].write(fastq_lines) if len(per_sample_fastqs) == 0: raise ValueError('No sequences were mapped to samples. Check that ' 'your barcodes are in the correct orientation (see ' 'the rev_comp_barcodes and/or ' 'rev_comp_mapping_barcodes options). If barcodes are ' 'NOT Golay format set golay_error_correction ' 'to False.') for fh in per_sample_fastqs.values(): fh.close() manifest_fh.close() result.manifest.write_data(manifest, FastqManifestFormat) _write_metadata_yaml(result) return result, ec_details_fmt
def join_pairs(demultiplexed_seqs: SingleLanePerSamplePairedEndFastqDirFmt, threads: int = 1) -> SingleLanePerSampleSingleEndFastqDirFmt: result = SingleLanePerSampleSingleEndFastqDirFmt() manifest = pd.read_csv(os.path.join(str(demultiplexed_seqs), demultiplexed_seqs.manifest.pathspec), header=0, comment='#') manifest.filename = manifest.filename.apply( lambda x: os.path.join(str(demultiplexed_seqs), x)) phred_offset = yaml.load( open( os.path.join( str(demultiplexed_seqs), demultiplexed_seqs.metadata.pathspec)))['phred-offset'] id_to_fps = manifest.pivot(index='sample-id', columns='direction', values='filename') output_manifest = FastqManifestFormat() output_manifest_fh = output_manifest.open() output_manifest_fh.write('sample-id,filename,direction\n') output_manifest_fh.write('# direction is not meaningful in this file ' 'as these\n') output_manifest_fh.write('# data may be derived from forward, reverse, ' 'or \n') output_manifest_fh.write('# joined reads\n') for i, (sample_id, (fwd_fp, rev_fp)) in enumerate(id_to_fps.iterrows()): # The barcode id, lane number and read number are not relevant # here. We might ultimately want to use a dir format other than # SingleLanePerSampleSingleEndFastqDirFmt which doesn't care # about this information. Similarly, the direction of the read # isn't relevant here anymore. path = result.sequences.path_maker(sample_id=sample_id, barcode_id=i, lane_number=1, read_number=1) uncompressed_path = str(path).strip('.gz') parent_pth = Path(path).parent sample_id_path = str(parent_pth / sample_id) assembled_pth = parent_pth / "{}.assembled.fastq".format(sample_id) discarded_pth = parent_pth / "{}.discarded.fastq".format(sample_id) unassembled_fwd_pth = parent_pth / "{}.unassembled.forward.fastq".format( sample_id) unassembled_rev_pth = parent_pth / "{}.unassembled.reverse.fastq".format( sample_id) cmd = [ 'pear', '-f', fwd_fp, '-r', rev_fp, '-o', sample_id_path, '--threads', str(threads) ] run_command(cmd) assembled_pth.rename(Path(uncompressed_path)) run_command(['gzip', uncompressed_path]) #delete extra files extra_files = [discarded_pth, unassembled_fwd_pth, unassembled_rev_pth] for f_pth in extra_files: try: os.remove(str(f_pth)) except: pass output_manifest_fh.write('%s,%s,%s\n' % (sample_id, Path(path).name, 'forward')) output_manifest_fh.close() result.manifest.write_data(output_manifest, FastqManifestFormat) metadata = YamlFormat() metadata.path.write_text(yaml.dump({'phred-offset': phred_offset})) result.metadata.write_data(metadata, YamlFormat) return result
def main(per_sample_sequences, threads, taxa, region): """The main communtion between the pluin and the ITSxpress program. Args: per_sample_sequences (SingleLanePerSampleSingleEndFastqDirFmt): The SingleLanePerSampleSingleEndFastqDirFmt type of the input. threads (int) : The number of threads to use. taxa (str): The taxa to be used for the search. region (str) : The region to be used for the search. Returns: (SingleLanePerSampleSingleEndFastqDirFmt): The SingleLanePerSampleSingleEndFastqDirFmt type of the output. Raises: ValueError1: BBTools error or fastq format issue. ValueError2: BBmerge error. ValueError3: hmmsearch error. """ # Setting a temp folder dirt = tempfile.tempdir # Setting the current dir os.chdir(str(per_sample_sequences.path)) # Finding the artifact type. artifactType = _view_artifact_type() # Setting the taxa taxa = _taxa_prefix_to_taxa(taxa) # Writing the manifest for the output qza manifest = FastqManifestFormat() manifest_fn = manifest.open() manifest_fn.write('sample-id,filename,direction\n') sequences,singleEnd = _fastq_id_maker(per_sample_sequences, artifactType) sequenceList = set(sequences) barcode = 0 # Creating result dir results = SingleLanePerSampleSingleEndFastqDirFmt() # Running the for loop for each sample for sequence in sequenceList: # Setting the fastq files and if singleEnd is used. fastq = os.path.join(str(per_sample_sequences.path),str(sequence[0])) if "SampleData[PairedEndSequencesWithQuality]" in artifactType: fastq2 = os.path.join(str(per_sample_sequences.path),str(sequence[1])) else: fastq2 = sequence[1] sequenceID = sequence[2] # Running the main ITSxpress program. try: itsx._check_fastqs(fastq, fastq2) # Parse input types paired_end, interleaved = itsx._is_paired(fastq, fastq2, singleEnd) except: raise ValueError("There is a problem with the fastq file(s) you selected or\n" "BBtools was not found. check that the BBtools reformat.sh package is executable.") # Create SeqSample objects and merge if needed. try: if paired_end and interleaved: sobj = itsx.SeqSamplePairedInterleaved(fastq=fastq, tempdir=dirt) sobj._merge_reads(threads=threads) elif paired_end and not interleaved: sobj = itsx.SeqSamplePairedNotInterleaved(fastq=fastq, fastq2=fastq2, tempdir=dirt) sobj._merge_reads(threads=threads) elif not paired_end and not interleaved: sobj = itsx.SeqSampleNotPaired(fastq=fastq, tempdir=dirt) except: raise ValueError("BBmerge was not found. check that the BBmerge reformat.sh package is executible") # Deduplicate sobj._deduplicate(threads=threads) try: # HMMSearch for ITS regions hmmfile = os.path.join(ROOT_DIR, "ITSx_db", "HMMs", taxa_dict[taxa]) sobj._search(hmmfile=hmmfile, threads=threads) except: raise ValueError("hmmsearch was not found, make sure HMMER3 is installed and executible") # Parse HMMseach output. its_pos = itsx.ItsPosition(domtable=sobj.dom_file, region=region) # Create deduplication object. dedup_obj = itsx.Dedup(uc_file=sobj.uc_file, rep_file=sobj.rep_file, seq_file=sobj.seq_file) pathForward = results.sequences.path_maker(sample_id=sequenceID, barcode_id=barcode, lane_number=1, read_number=1) manifest_fn.write("{},{},forward\n".format(sequenceID,pathForward.name)) # Create trimmed sequences. dedup_obj.create_trimmed_seqs(str(pathForward), gzipped=True, itspos=its_pos) # Deleting the temp files. itsx.shutil.rmtree(sobj.tempdir) barcode += 1 #Writing out the results. manifest_fn.close() _write_metadata(results) results.manifest.write_data(manifest, FastqManifestFormat) return results
def main(fastq, fastq2, singleEnd, threads, taxa, region): """The main communtion between the pluin and the ITSxpress program. Args: fastq (str) : The first fastq location. fastq2 (str) : The second fastq location. singleEnd (bool) : boolean for if singleEnd is used or not. threads (int) : The number of threads to use. taxa (str): The taxa to be used for the search. region (str) : The region to be used for the search. Returns: (SingleLanePerSampleSingleEndFastqDirFmt): The SingleLanePerSampleSingleEndFastqDirFmt type of the output. Raises: ValueError1: BBTools error or fastq format issue. ValueError2: BBmerge error. ValueError3: hmmsearch error. """ dirt = "/tmp" try: itsx._check_fastqs(fastq, fastq2) # Parse input types paired_end, interleaved = itsx._is_paired(fastq, fastq2, singleEnd) except: raise ValueError("There is a problem with the fastq file(s) you selected or\n" "BBtools was not found. check that the BBtools reformat.sh package is executable.") # Create SeqSample objects and merge if needed. try: if paired_end and interleaved: sobj = itsx.SeqSamplePairedInterleaved(fastq=fastq, tempdir=dirt) sobj._merge_reads(threads=threads) elif paired_end and not interleaved: sobj = itsx.SeqSamplePairedNotInterleaved(fastq=fastq, fastq2=fastq2, tempdir=dirt) sobj = itsx.SeqSampleNotPaired(fastq=fastq, tempdir=dirt) except: raise ValueError("BBmerge was not found. check that the BBmerge reformat.sh package is executible") # Deduplicate sobj._deduplicate(threads=threads) try: # HMMSearch for ITS regions hmmfile = os.path.join(ROOT_DIR, "ITSx_db", "HMMs", taxa_dict[taxa]) sobj._search(hmmfile=hmmfile, threads=threads) except: raise ValueError("hmmsearch was not found, make sure HMMER3 is installed and executible") # Parse HMMseach output. its_pos = itsx.ItsPosition(domtable=sobj.dom_file, region=region) # Create deduplication object. dedup_obj = itsx.Dedup(uc_file=sobj.uc_file, rep_file=sobj.rep_file, seq_file=sobj.seq_file) results = SingleLanePerSampleSingleEndFastqDirFmt() path = results.sequences.path_maker(sample_id="seq", barcode_id=1, lane_number=1, read_number=1) # Writing the manifest for the output qza manifest = FastqManifestFormat() manifest_fn = manifest.open() manifest_fn.write('sample-id,filename,direction\n') manifest_fn.write("seq,{},reverse".format(path)) manifest_fn.close() # Create trimmed sequences. dedup_obj.create_trimmed_seqs(str(path), gzipped=True, itspos=its_pos) # Writing out the results. _write_metadata(results) results.manifest.write_data(manifest, FastqManifestFormat) # Deleting the temp files. itsx.shutil.rmtree(sobj.tempdir) return results