def process_barcode_single_end_data(read1_data, output_bc_fastq, output_fastq1, bc1_len=6, rev_comp_bc1=False): """ Processes, writes single-end barcode data, parsed sequence read1_data: list of header, read, quality scores output_bc_fastq: open output fastq filepath output_fastq1: open output fastq reads filepath bc1_len: length of barcode to remove from beginning of data rev_comp_bc1: reverse complement barcode before writing. """ header_index = 0 sequence_index = 1 quality_index = 2 bc_read = read1_data[sequence_index][:bc1_len] bc_qual = read1_data[quality_index][:bc1_len] if rev_comp_bc1: bc_read = str(DNA(bc_read).rc()) bc_qual = bc_qual[::-1] bc_lines = format_fastq_record(read1_data[header_index], bc_read, bc_qual) output_bc_fastq.write(bc_lines) seq_lines = format_fastq_record(read1_data[header_index], read1_data[sequence_index][bc1_len:], read1_data[quality_index][bc1_len:]) output_fastq1.write(seq_lines) return
def write_synced_barcodes_fastq(joined_fp, index_fp): """Writes new index file based on surviving assembled paired-ends. -joined_fp : file path to paired-end assembled fastq file -index_fp : file path to index / barcode reads fastq file This function iterates through the joined reads file and index file. Only those index-reads within the file at index_fp, that have headers matching those within the joined-pairs at joined_fp, are written to file. WARNING: Assumes reads are in the same order in both files, except for cases in which the corresponding read in the joined_fp file is missing (i.e. pairs failed to assemble). """ # open files (handles normal / gzipped data) jh = qiime_open(joined_fp) ih = qiime_open(index_fp) # base new index file name on joined paired-end file name: j_path, ext = os.path.splitext(joined_fp) filtered_bc_outfile_path = j_path + '_barcodes.fastq' fbc_fh = open(filtered_bc_outfile_path, 'w') # Set up iterators index_fastq_iter = parse_fastq(ih, strict=False) joined_fastq_iter = parse_fastq(jh, strict=False) # Write barcodes / index reads that we observed within # the joined paired-ends. Warn if index and joined data # are not in order. for joined_label, joined_seq, joined_qual in joined_fastq_iter: index_label, index_seq, index_qual = index_fastq_iter.next() while joined_label != index_label: try: index_label, index_seq, index_qual = index_fastq_iter.next() except StopIteration: raise StopIteration( "\n\nReached end of index-reads file" + " before iterating through joined paired-end-reads file!" + " Except for missing paired-end reads that did not survive" + " assembly, your index and paired-end reads files must be in" + " the same order! Also, check that the index-reads and" + " paired-end reads have identical headers. The last joined" + " paired-end ID processed was:\n\'%s\'\n" % (joined_label)) else: fastq_string = format_fastq_record(index_label, index_seq, index_qual) fbc_fh.write(fastq_string) ih.close() jh.close() fbc_fh.close() return filtered_bc_outfile_path
def write_synced_barcodes_fastq(joined_fp, index_fp): """Writes new index file based on surviving assembled paired-ends. -joined_fp : file path to paired-end assembled fastq file -index_fp : file path to index / barcode reads fastq file This function iterates through the joined reads file and index file. Only those index-reads within the file at index_fp, that have headers matching those within the joined-pairs at joined_fp, are written to file. WARNING: Assumes reads are in the same order in both files, except for cases in which the corresponding read in the joined_fp file is missing (i.e. pairs failed to assemble). """ # open files (handles normal / gzipped data) jh = qiime_open(joined_fp) ih = qiime_open(index_fp) # base new index file name on joined paired-end file name: j_path, ext = os.path.splitext(joined_fp) filtered_bc_outfile_path = j_path + '_barcodes.fastq' fbc_fh = open(filtered_bc_outfile_path, 'w') # Set up iterators index_fastq_iter = parse_fastq(ih, strict=False) joined_fastq_iter = parse_fastq(jh, strict=False) # Write barcodes / index reads that we observed within # the joined paired-ends. Warn if index and joined data # are not in order. for joined_label, joined_seq, joined_qual in joined_fastq_iter: index_label, index_seq, index_qual = index_fastq_iter.next() while joined_label != index_label: try: index_label, index_seq, index_qual = index_fastq_iter.next() except StopIteration: raise StopIteration("\n\nReached end of index-reads file" + " before iterating through joined paired-end-reads file!" + " Except for missing paired-end reads that did not survive" + " assembly, your index and paired-end reads files must be in" + " the same order! Also, check that the index-reads and" + " paired-end reads have identical headers. The last joined" + " paired-end ID processed was:\n\'%s\'\n" % (joined_label)) else: fastq_string = format_fastq_record(index_label, index_seq, index_qual) fbc_fh.write(fastq_string) ih.close() jh.close() fbc_fh.close() return filtered_bc_outfile_path
def process_barcode_in_label(read1_data, read2_data, output_bc_fastq, bc1_len=6, bc2_len=6, rev_comp_bc1=False, rev_comp_bc2=False, char_delineator=":"): """ Reads data from one or two fastq labels, writes output barcodes file. read1_data: list of header, read, quality scores read2_data: list of header, read, quality scores, False if no read 2. output_bc_fastq: open output fastq filepath bc1_len: length of barcode to remove from beginning of read1 data bc2_len: length of barcode to remove from beginning of read2 data rev_comp_bc1: reverse complement barcode 1 before writing. rev_comp_bc2: reverse complement barcode 2 before writing. char_delineator: Specify character that immediately precedes the barcode for input_type of barcode_in_label. """ header_index = 0 # Check for char_delineator in sequence try: bc1_read = read1_data[header_index].split( char_delineator)[-1][0:bc1_len] # If there is an index error, it means the char_delineator wasn't found except IndexError: raise IndexError("Found sequence lacking character delineator. " "Sequence header %s, character delineator %s" % (read1_data[header_index], char_delineator)) # Create fake quality scores, using 6 here to match the existing qual fake # qual scores that were all F. bc1_qual = np.ones(len(bc1_read), dtype=np.int8) * 6 if rev_comp_bc1: bc1_read = str(DNA(bc1_read).rc()) if read2_data: bc2_read =\ read2_data[header_index].strip().split( char_delineator)[-1][0:bc2_len] bc2_qual = np.ones(len(bc2_read), dtype=np.int8) * 6 if rev_comp_bc2: bc2_read = str(DNA(bc2_read).rc()) else: bc2_read = "" bc2_qual = np.array([], dtype=np.int8) if not bc1_read and not bc2_read: raise ValueError("Came up with empty barcode sequence, please check " "character delineator with -s, and fastq label " "%s" % read1_data[header_index]) bc_lines = format_fastq_record(read1_data[header_index], bc1_read + bc2_read, np.hstack([bc1_qual, bc2_qual])) output_bc_fastq.write(bc_lines) return
def process_barcode_paired_stitched(read_data, output_bc_fastq, output_fastq, bc1_len=6, bc2_len=6, rev_comp_bc1=False, rev_comp_bc2=False, attempt_read_orientation=False, forward_primers=None, reverse_primers=None, output_bc_not_oriented=None, fastq_out_not_oriented=None, switch_bc_order=False): """ Processes stitched barcoded reads, writes barcode, parsed stitched read read_data: list of header, read, quality scores output_bc_fastq: open output fastq filepath output_fastq: open output fastq reads filepath bc1_len: length of barcode to remove from beginning of read1 stitched data bc2_len: length of barcode to remove from end of read2 stitched data rev_comp_bc1: reverse complement barcode 1 before writing. rev_comp_bc2: reverse complement barcode 2 before writing. attempt_read_orientation: If True, will attempt to orient the reads according to the forward primers in the mapping file. If primer is detected in current orientation, leave the read as is, but if reverse complement is detected (or ReversePrimer is detected in the current orientation) the read will either be written to the forward (read 1) or reverse (read 2) reads for the case of paired files, or the read will be reverse complemented in the case of stitched reads. forward_primers: list of regular expression generators, forward primers reverse_primers: list of regular expression generators, reverse primers output_bc_not_oriented: Barcode output from reads that are not oriented fastq_out_not_oriented: Open filepath to write reads where primers can't be found when attempt_read_orientation is True. switch_bc_order: Normally, barcode 1 will be written first, followed by barcode 2 in a combined output fastq file. If True, the order will be reversed. Only applies to stitched reads processing, as other barcode orders are dictated by the the parameter chosen for the fastq files. """ header_index = 0 sequence_index = 1 quality_index = 2 read_seq = read_data[sequence_index] read_qual = read_data[quality_index] found_primer_match = False # Break from orientation search as soon as a match is found if attempt_read_orientation: for curr_primer in forward_primers: if curr_primer.search(read_data[sequence_index]): found_primer_match = True break if not found_primer_match: for curr_primer in reverse_primers: if curr_primer.search(read_data[sequence_index]): read_seq = str(DNA(read_seq).rc()) read_qual = read_qual[::-1] found_primer_match = True break if not found_primer_match and attempt_read_orientation: output_bc = output_bc_not_oriented output_read = fastq_out_not_oriented else: output_bc = output_bc_fastq output_read = output_fastq bc_read1 = read_seq[0:bc1_len] bc_read2 = read_seq[-bc2_len:] bc_qual1 = read_qual[0:bc1_len] bc_qual2 = read_qual[-bc2_len:] if rev_comp_bc1: bc_read1 = str(DNA(bc_read1).rc()) bc_qual1 = bc_qual1[::-1] if rev_comp_bc2: bc_read2 = str(DNA(bc_read2).rc()) bc_qual2 = bc_qual2[::-1] if switch_bc_order: bc_read1, bc_read2 = bc_read2, bc_read1 bc_qual1, bc_qual2 = bc_qual2, bc_qual1 bc_lines = format_fastq_record(read_data[header_index], bc_read1 + bc_read2, np.hstack([bc_qual1, bc_qual2])) output_bc.write(bc_lines) seq_lines = format_fastq_record(read_data[header_index], read_seq[bc1_len:-bc2_len], read_qual[bc1_len:-bc2_len]) output_read.write(seq_lines) return
def process_barcode_paired_end_data(read1_data, read2_data, output_bc_fastq, output_fastq1, output_fastq2, bc1_len=6, bc2_len=6, rev_comp_bc1=False, rev_comp_bc2=False, attempt_read_orientation=False, forward_primers=None, reverse_primers=None, output_bc_not_oriented=None, fastq1_out_not_oriented=None, fastq2_out_not_oriented=None): """ Processes, writes paired-end barcode data, parsed sequences read1_data: list of header, read, quality scores read2_data: list of header, read, quality scores output_bc_fastq: open output fastq filepath output_fastq1: open output fastq reads 1 filepath output_fastq2: open output fastq reads 2 filepath bc1_len: length of barcode to remove from beginning of read1 data bc2_len: length of barcode to remove from beginning of read2 data rev_comp_bc1: reverse complement barcode 1 before writing. rev_comp_bc2: reverse complement barcode 2 before writing. attempt_read_orientation: If True, will attempt to orient the reads according to the forward primers in the mapping file. If primer is detected in current orientation, leave the read as is, but if reverse complement is detected (or ReversePrimer is detected in the current orientation) the read will either be written to the forward (read 1) or reverse (read 2) reads for the case of paired files, or the read will be reverse complemented in the case of stitched reads. forward_primers: list of regular expression generators, forward primers reverse_primers: list of regular expression generators, reverse primers output_bc_not_oriented: Barcode output from reads that are not oriented fastq1_out_not_oriented: Open filepath to write reads 1 where primers can't be found when attempt_read_orientation is True. fastq2_out_not_oriented: Open filepath to write reads 2 where primers can't be found when attempt_read_orientation is True. """ header_index = 0 sequence_index = 1 quality_index = 2 found_primer_match = False # Break from orientation search as soon as a match is found if attempt_read_orientation: # First check forward primers for curr_primer in forward_primers: if curr_primer.search(read1_data[sequence_index]): read1 = read1_data read2 = read2_data found_primer_match = True break if curr_primer.search(read2_data[sequence_index]): read1 = read2_data read2 = read1_data found_primer_match = True break # Check reverse primers if forward primers not found if not found_primer_match: for curr_primer in reverse_primers: if curr_primer.search(read1_data[sequence_index]): read1 = read2_data read2 = read1_data found_primer_match = True break if curr_primer.search(read2_data[sequence_index]): read1 = read1_data read2 = read2_data found_primer_match = True break else: read1 = read1_data read2 = read2_data if not found_primer_match and attempt_read_orientation: read1 = read1_data read2 = read2_data output_bc = output_bc_not_oriented output_read1 = fastq1_out_not_oriented output_read2 = fastq2_out_not_oriented else: output_bc = output_bc_fastq output_read1 = output_fastq1 output_read2 = output_fastq2 bc_read1 = read1[sequence_index][0:bc1_len] bc_read2 = read2[sequence_index][0:bc2_len] bc_qual1 = read1[quality_index][0:bc1_len] bc_qual2 = read2[quality_index][0:bc2_len] if rev_comp_bc1: bc_read1 = str(DNA(bc_read1).rc()) bc_qual1 = bc_qual1[::-1] if rev_comp_bc2: bc_read2 = str(DNA(bc_read2).rc()) bc_qual2 = bc_qual2[::-1] bc_lines = format_fastq_record(read1[header_index], bc_read1 + bc_read2, np.hstack([bc_qual1, bc_qual2])) output_bc.write(bc_lines) seq1_lines = format_fastq_record(read1[header_index], read1[sequence_index][bc1_len:], read1[quality_index][bc1_len:]) output_read1.write(seq1_lines) seq2_lines = format_fastq_record(read2[header_index], read2[sequence_index][bc2_len:], read2[quality_index][bc2_len:]) output_read2.write(seq2_lines) return
def test_format_fastq_record(self): """Construt a FASTQ record""" exp = b"@abc\ndef\n+\nfgh\n" obs = format_fastq_record(b'abc', b'def', np.array([38, 39, 40], dtype=np.int8), 64) self.assertEqual(obs, exp)
def test_format_fastq_record(self): """Construt a FASTQ record""" exp = "@abc\ndef\n+\nfgh\n" obs = format_fastq_record('abc', 'def', np.array([38, 39, 40], dtype=np.int8), 64) self.assertEqual(obs, exp)