Esempio n. 1
0
def trim_mismatches_from_start(mapping, region_fetcher, type_counts):
    ''' Remove all consecutive Q30+ mismatches from the beginning of alignments,
        under the assumption that these represent untemplated additions during
        reverse transcription.
        Characterize the mismatches into type_counts.
    '''
    if sam.contains_indel_pysam(mapping) or mapping.is_unmapped:
        set_nongenomic_length(mapping, 0)
        return mapping

    if mapping.is_reverse:
        aligned_pairs = mapping.aligned_pairs[::-1]
        index_lookup = utilities.base_to_complement_index
    else:
        aligned_pairs = mapping.aligned_pairs
        index_lookup = utilities.base_to_index

    decoded_qual = fastq.decode_sanger(mapping.qual)

    bases_to_trim = 0
    found_trim_point = False
    first_ref_index = None
    for read_index, ref_index in aligned_pairs:
        if read_index == None:
            # This shouldn't be able to be triggered since alignments
            # containing indels are ruled out above.
            continue

        if mapping.is_reverse:
            corrected_read_index = mapping.qlen - 1 - read_index
        else:
            corrected_read_index = read_index

        ref_base = region_fetcher(mapping.tid, ref_index, ref_index + 1)
        read_base = mapping.seq[read_index]
        read_qual = decoded_qual[read_index]
        coords = (
            mapping.qlen,
            corrected_read_index,
            read_qual,
            index_lookup[ref_base],
            index_lookup[read_base],
        )
        type_counts[coords] += 1

        if not found_trim_point:
            if read_base != ref_base and read_qual >= 30:
                bases_to_trim += 1
            else:
                first_ref_index = ref_index
                found_trim_point = True

    if first_ref_index == None:
        raise ValueError('first_ref_index not set')

    if bases_to_trim == 0:
        trimmed_mapping = mapping
    else:
        trimmed_mapping = pysam.AlignedRead()
        trimmed_mapping.qname = mapping.qname
        trimmed_mapping.tid = mapping.tid

        # first_ref_index has been set above to the be index of the
        # reference base aligned to the first non-trimmed base in the
        # read. If the mapping is forward, this will be the new pos.
        # If the mapping is reverse, the pos won't change.
        if mapping.is_reverse:
            first_ref_index = mapping.pos
        trimmed_mapping.pos = first_ref_index

        trimmed_mapping.is_reverse = mapping.is_reverse
        trimmed_mapping.is_secondary = mapping.is_secondary
        trimmed_mapping.mapq = mapping.mapq

        if mapping.is_reverse:
            # bases_to_trim is never zero here, so there is no danger
            # of minus zero
            trimmed_slice = slice(None, -bases_to_trim)
        else:
            trimmed_slice = slice(bases_to_trim, None)

        trimmed_mapping.seq = mapping.seq[trimmed_slice]
        trimmed_mapping.qual = mapping.qual[trimmed_slice]
        trimmed_mapping.rnext = -1
        trimmed_mapping.pnext = -1

        trimmed_length = len(mapping.seq) - bases_to_trim
        if mapping.is_reverse:
            # Remove blocks from the end
            trimmed_cigar = sam.truncate_cigar_blocks_up_to(
                mapping.cigar, trimmed_length)
        else:
            # Remove blocks from the beginning
            trimmed_cigar = sam.truncate_cigar_blocks_from_beginning(
                mapping.cigar, trimmed_length)

        trimmed_mapping.cigar = trimmed_cigar

    return trimmed_mapping
Esempio n. 2
0
def trim_nongenomic_polyA_from_end(mapping, region_fetcher):
    ''' If a mapping ends in a polyA stretch, soft clip from the first
    nongenomic A onward.
    '''
    if sam.contains_indel_pysam(mapping) or mapping.is_unmapped:
        return mapping

    first_ref_index = None
    if mapping.is_reverse:
        bases_to_trim = 0
        poly_T_end = find_poly_T(mapping.seq)
        for read_index, ref_index in mapping.aligned_pairs[::-1]:
            if read_index == None:
                # indels are filtered out above, so this can only be 
                # a skip from splicing
                continue
            if read_index > poly_T_end:
                first_ref_index = ref_index
                continue
            ref_base = region_fetcher(mapping.tid, ref_index, ref_index + 1)
            if ref_base != 'T':
                bases_to_trim = read_index + 1
                break
            else:
                # first_ref_index needs to be set to the last position
                # that passed that 'are you genomic?' test
                first_ref_index = ref_index
    else:
        first_ref_index = mapping.pos
        bases_to_trim = 0
        poly_A_start = find_poly_A(mapping.seq)
        for read_index, ref_index in mapping.aligned_pairs:
            if read_index < poly_A_start:
                continue
            ref_base = region_fetcher(mapping.tid, ref_index, ref_index + 1)
            if ref_base != 'A':
                bases_to_trim = len(mapping.seq) - read_index
                break
    
    if first_ref_index == None:
        print mapping
        raise ValueError('first_ref_index not set')

    if bases_to_trim > 0:
        mapping.pos = first_ref_index

        trimmed_length = len(mapping.seq) - bases_to_trim
        soft_clipped_block = [(sam.BAM_CSOFT_CLIP, bases_to_trim)]
        if mapping.is_reverse:
            # Remove blocks from the beginning.
            trimmed_cigar = sam.truncate_cigar_blocks_from_beginning(mapping.cigar, trimmed_length)
            updated_cigar = soft_clipped_block + trimmed_cigar
        else:
            # Remove blocks from the end.
            trimmed_cigar = sam.truncate_cigar_blocks_up_to(mapping.cigar, trimmed_length)
            updated_cigar = trimmed_cigar + soft_clipped_block
        
        mapping.cigar = updated_cigar
    
    if mapping.tags:
        # Clear the MD tag since the possible removal of bases to the
        # alignment may have made it inaccurate. 
        # TODO: now have machinery to make it accurate.
        filtered_tags = filter(lambda t: t[0] != 'MD', mapping.tags)
        mapping.tags = filtered_tags

    set_nongenomic_length(mapping, bases_to_trim)

    return mapping
Esempio n. 3
0
def combine_paired_mappings(R1_mapping, R2_mapping, verbose=False):
    ''' Takes two pysam mappings representing opposite ends of a fragment and
    combines them into one mapping, (ab)using BAM_CREF_SKIP to bridge the gap
    (if any) between them.
    '''
    R1_strand = sam.get_strand(R1_mapping)

    if R1_strand == '+':
        left_mapping, right_mapping = R1_mapping, R2_mapping
    elif R1_strand == '-':
        left_mapping, right_mapping = R2_mapping, R1_mapping

    left_md = dict(left_mapping.tags)['MD']
    right_md = dict(right_mapping.tags)['MD']

    right_aligned_pairs = sam.cigar_to_aligned_pairs(
        right_mapping.cigar, right_mapping.reference_start)

    right_after_overlap_pair_index = len(right_aligned_pairs)
    for i, (read, ref) in enumerate(right_aligned_pairs):
        if ref != None and ref >= left_mapping.aend:
            right_after_overlap_pair_index = i
            break

    right_overlap_pairs = right_aligned_pairs[:right_after_overlap_pair_index]
    right_after_overlap_pairs = right_aligned_pairs[
        right_after_overlap_pair_index:]

    right_reads_after = [
        read for read, ref in right_after_overlap_pairs
        if read != None and read != 'N'
    ]
    right_refs_after = [
        ref for read, ref in right_after_overlap_pairs if ref != None
    ]

    right_overlap_cigar = sam.aligned_pairs_to_cigar(right_overlap_pairs)
    right_after_overlap_cigar = sam.aligned_pairs_to_cigar(
        right_after_overlap_pairs)
    right_after_overlap_md = sam.truncate_md_string_from_beginning(
        right_md, len(right_refs_after))

    right_after_overlap_read_start = len(
        right_mapping.seq) - len(right_reads_after)

    right_overlap_seq = right_mapping.seq[:right_after_overlap_read_start]
    right_overlap_qual = right_mapping.qual[:right_after_overlap_read_start]

    right_after_overlap_seq = right_mapping.seq[
        right_after_overlap_read_start:]
    right_after_overlap_qual = right_mapping.qual[
        right_after_overlap_read_start:]

    left_aligned_pairs = sam.cigar_to_aligned_pairs(
        left_mapping.cigar, left_mapping.reference_start)

    left_before_overlap_pair_index = -1
    for i, (read, ref) in list(enumerate(left_aligned_pairs))[::-1]:
        if ref != None and ref < right_mapping.pos:
            left_before_overlap_pair_index = i
            break

    left_overlap_pairs = left_aligned_pairs[left_before_overlap_pair_index +
                                            1:]
    left_before_overlap_pairs = left_aligned_pairs[:
                                                   left_before_overlap_pair_index
                                                   + 1]

    left_reads_before = [
        read for read, ref in left_before_overlap_pairs
        if read != None and read != 'N'
    ]
    left_refs_before = [
        ref for read, ref in left_before_overlap_pairs if ref != None
    ]

    left_overlap_cigar = sam.aligned_pairs_to_cigar(left_overlap_pairs)
    left_before_overlap_cigar = sam.aligned_pairs_to_cigar(
        left_before_overlap_pairs)
    left_before_overlap_md = sam.truncate_md_string_up_to(
        left_md, len(left_refs_before))

    left_overlap_read_start = len(left_reads_before)
    left_overlap_seq = left_mapping.seq[left_overlap_read_start:]
    left_overlap_qual = left_mapping.qual[left_overlap_read_start:]

    left_before_overlap_seq = left_mapping.seq[:left_overlap_read_start]
    left_before_overlap_qual = left_mapping.qual[:left_overlap_read_start]

    if left_overlap_pairs or right_overlap_pairs:
        gap_length = 0

        left_has_splicing = sam.contains_splicing(left_mapping)
        right_has_splicing = sam.contains_splicing(right_mapping)

        if left_overlap_cigar == right_overlap_cigar:
            # If the two mappings agree about the location of indels in their overlap,
            # use the seq from the mapping with the higher average quality in the
            # overlap.
            left_mean_qual = np.mean(fastq.decode_sanger(left_overlap_qual))
            right_mean_qual = np.mean(fastq.decode_sanger(right_overlap_qual))

            if left_mean_qual > right_mean_qual:
                use_overlap_from = 'left'
            else:
                use_overlap_from = 'right'
        elif left_has_splicing != right_has_splicing:
            # A temporary(?) heuristic - if one read has splicing and the other
            # doesn't, use the overlap from the one with splicing under the
            # assumption that the other just has a few bases overhanging the
            # splice junction.
            if left_has_splicing:
                use_overlap_from = 'left'
            else:
                use_overlap_from = 'right'
        else:
            # If the two mappings disagree about the location of indels in their overlap,
            # we need a heuristic for picking which mapping we believe reflects the
            # true structure of the input fragment. The most innocuous explanation
            # is that a 'true' indel happened to lie close to the edge of one of the
            # mappings. A more problematic situation is a 'false' indel (that is,
            # produced during cluster generation or sequencing-by-synthesis, NOT
            # template production). Our strategy is: realign the overlapping part of
            # left mapping starting from the left edge of the overlap according to the
            # cigar of the right mapping and realign the overlapping part of the right
            # mapping starting from the right edge of the overlap according to the cigar
            # of the left mapping. Count the number of mismatches produced by each.
            # If the left overlap can accomodate the right cigar with fewer mismatches,
            # use the right cigar and seq. If the right overlap can accomodate the left
            # cigar with fewer mismatches, use the left cigar and seq.

            # The leftmost aligned_pair from the right mapping is guaranteed by the
            # mapping process to not involve a gap.
            _, overlap_ref_start = right_overlap_pairs[0]
            # Similarly, the rightmost aligned_pair from the left mapping can't be a
            # gap.
            _, overlap_ref_end = left_overlap_pairs[-1]

            realigned_left_cigar = sam.truncate_cigar_blocks_up_to(
                right_mapping.cigar, len(left_overlap_seq))
            realigned_right_cigar = sam.truncate_cigar_blocks_from_beginning(
                left_mapping.cigar, len(right_overlap_seq))

            ref_dict = sam.merge_ref_dicts(
                sam.ref_dict_from_mapping(left_mapping),
                sam.ref_dict_from_mapping(right_mapping),
            )

            try:
                left_using_right_mismatches = realigned_mismatches(
                    left_overlap_seq, overlap_ref_start, realigned_left_cigar,
                    ref_dict)
                right_using_left_mismatches = realigned_mismatches_backwards(
                    right_overlap_seq, overlap_ref_end, realigned_right_cigar,
                    ref_dict)
            except ValueError:
                print left_mapping
                print right_mapping
                raise

            if verbose:
                logging.info('disagreements in {0}'.format(left_mapping.qname))
                logging.info('left overlap cigar is  {0}'.format(
                    str(left_overlap_cigar)))
                logging.info('right overlap cigar is {0}'.format(
                    str(right_overlap_cigar)))
                logging.info('left_using_right_mismatches - {0}'.format(
                    len(left_using_right_mismatches)))
                logging.info('right_using_left_mismatches - {0}'.format(
                    len(right_using_left_mismatches)))

            if len(left_using_right_mismatches) < len(
                    right_using_left_mismatches):
                use_overlap_from = 'right'
            elif len(right_using_left_mismatches) < len(
                    left_using_right_mismatches):
                use_overlap_from = 'left'
            else:
                logging.info('disagreements in {0}'.format(left_mapping.qname))
                logging.info('left overlap cigar is  {0}'.format(
                    str(left_overlap_cigar)))
                logging.info('right overlap cigar is {0}'.format(
                    str(right_overlap_cigar)))
                logging.info('left_using_right_mismatches - {0}'.format(
                    len(left_using_right_mismatches)))
                logging.info('right_using_left_mismatches - {0}'.format(
                    len(right_using_left_mismatches)))
                logging.info('ambiguous disagreement')
                return False

    else:
        gap_length = right_mapping.pos - left_mapping.aend
        # It doesn't matter what use_overlap_from is set to; there is no overlap
        use_overlap_from = 'left'

    combined_mapping = pysam.AlignedRead()
    combined_mapping.qname = left_mapping.qname
    combined_mapping.tid = left_mapping.tid
    combined_mapping.mapq = min(left_mapping.mapq, right_mapping.mapq)
    combined_mapping.rnext = -1
    combined_mapping.pnext = -1
    combined_mapping.pos = left_mapping.pos

    if R1_strand == '-':
        combined_mapping.is_reverse = True

    gap_cigar = [(sam.BAM_CREF_SKIP, gap_length)]

    if use_overlap_from == 'left':
        combined_mapping.seq = left_mapping.seq + right_after_overlap_seq
        combined_mapping.qual = left_mapping.qual + right_after_overlap_qual
        combined_mapping.cigar = left_mapping.cigar + gap_cigar + right_after_overlap_cigar

        combined_md = sam.combine_md_strings(left_md, right_after_overlap_md)
        combined_mapping.setTag('MD', combined_md)

        overlap_seq_tag = right_overlap_seq
        overlap_qual_tag = right_overlap_qual

    elif use_overlap_from == 'right':
        combined_mapping.seq = left_before_overlap_seq + right_mapping.seq
        combined_mapping.qual = left_before_overlap_qual + right_mapping.qual
        combined_mapping.cigar = left_before_overlap_cigar + gap_cigar + right_mapping.cigar

        combined_md = sam.combine_md_strings(left_before_overlap_md, right_md)
        combined_mapping.setTag('MD', combined_md)

        overlap_seq_tag = left_overlap_seq
        overlap_qual_tag = left_overlap_qual

    if len(overlap_seq_tag) > 0:
        # Having empty tags causes problems, so don't create them.
        combined_mapping.setTag('Xs', overlap_seq_tag)
        combined_mapping.setTag('Xq', overlap_qual_tag)
        combined_mapping.setTag('Xw', use_overlap_from)

    return combined_mapping
Esempio n. 4
0
def trim_mismatches_from_start(mapping, region_fetcher, type_counts):
    ''' Remove all consecutive Q30+ mismatches from the beginning of alignments,
        under the assumption that these represent untemplated additions during
        reverse transcription.
        Characterize the mismatches into type_counts.
    '''
    if sam.contains_indel_pysam(mapping) or mapping.is_unmapped:
        set_nongenomic_length(mapping, 0)
        return mapping

    if mapping.is_reverse:
        aligned_pairs = mapping.aligned_pairs[::-1]
        index_lookup = utilities.base_to_complement_index
    else:
        aligned_pairs = mapping.aligned_pairs
        index_lookup = utilities.base_to_index

    decoded_qual = fastq.decode_sanger(mapping.qual)
    
    bases_to_trim = 0
    found_trim_point = False
    first_ref_index = None
    for read_index, ref_index in aligned_pairs:
        if read_index == None:
            # This shouldn't be able to be triggered since alignments
            # containing indels are ruled out above.
            continue

        if mapping.is_reverse:
            corrected_read_index = mapping.qlen - 1 - read_index
        else:
            corrected_read_index = read_index

        ref_base = region_fetcher(mapping.tid, ref_index, ref_index + 1)
        read_base = mapping.seq[read_index]
        read_qual = decoded_qual[read_index]
        coords = (mapping.qlen,
                  corrected_read_index,
                  read_qual,
                  index_lookup[ref_base],
                  index_lookup[read_base],
                 )
        type_counts[coords] += 1

        if not found_trim_point:
            if read_base != ref_base and read_qual >= 30:
                bases_to_trim += 1
            else:
                first_ref_index = ref_index
                found_trim_point = True

    if first_ref_index == None:
        raise ValueError('first_ref_index not set')

    if bases_to_trim == 0:
        trimmed_mapping = mapping
    else:
        trimmed_mapping = pysam.AlignedRead()
        trimmed_mapping.qname = mapping.qname
        trimmed_mapping.tid = mapping.tid
        
        # first_ref_index has been set above to the be index of the
        # reference base aligned to the first non-trimmed base in the
        # read. If the mapping is forward, this will be the new pos.
        # If the mapping is reverse, the pos won't change.
        if mapping.is_reverse:
            first_ref_index = mapping.pos
        trimmed_mapping.pos = first_ref_index

        trimmed_mapping.is_reverse = mapping.is_reverse
        trimmed_mapping.is_secondary = mapping.is_secondary
        trimmed_mapping.mapq = mapping.mapq

        if mapping.is_reverse:
            # bases_to_trim is never zero here, so there is no danger
            # of minus zero
            trimmed_slice = slice(None, -bases_to_trim)
        else:
            trimmed_slice = slice(bases_to_trim, None)

        trimmed_mapping.seq = mapping.seq[trimmed_slice]
        trimmed_mapping.qual = mapping.qual[trimmed_slice]
        trimmed_mapping.rnext = -1
        trimmed_mapping.pnext = -1

        trimmed_length = len(mapping.seq) - bases_to_trim
        if mapping.is_reverse:
            # Remove blocks from the end
            trimmed_cigar = sam.truncate_cigar_blocks_up_to(mapping.cigar, trimmed_length)
        else:
            # Remove blocks from the beginning
            trimmed_cigar = sam.truncate_cigar_blocks_from_beginning(mapping.cigar, trimmed_length)
        
        trimmed_mapping.cigar = trimmed_cigar

    return trimmed_mapping