示例#1
0
def main():
    outfolder, genome_fasta, normalization_file_name = sys.argv[1:4]
    experimental_file_names = sys.argv[4:]

    control_dict = mod_utils.unPickle(normalization_file_name)
    rescaled_control_dict = normalize_dict_to_max(control_dict)
    norm_name = '.'.join(os.path.basename(normalization_file_name).split('.')[:-1])
    experimental_dict_names = ['.'.join(os.path.basename(file_name).split('.')[:-1]) for file_name in experimental_file_names]
    experimental_dicts = [mod_utils.unPickle(file_name) for file_name in experimental_file_names]
    rescaled_experimental_dicts = [normalize_dict_to_max(exp_dict) for exp_dict in experimental_dicts]

    print experimental_dict_names, norm_name

    normed_mutation_rate_histogram(rescaled_experimental_dicts, experimental_dict_names, os.path.join(outfolder, '%s_rescaled_mutation_rate_histogram' % norm_name), title='mutation rate, rescaled to max', xlim = (0, 0.1), min = 0, max =1, step = 0.001)
    comparisons = []
    rescaled_comparisons = []
    write_wig(control_dict, norm_name, os.path.join(outfolder, norm_name))
    for i in range(len(experimental_dict_names)):
        write_wig(rescaled_experimental_dicts[i], experimental_dict_names[i], os.path.join(outfolder, experimental_dict_names[i]))
        comparison_log2_ratios = compare_to_control(experimental_dicts[i], control_dict)
        rescaled_comparison_log2_ratios = compare_to_control(rescaled_experimental_dicts[i], rescaled_control_dict)
        comparisons.append(comparison_log2_ratios)
        rescaled_comparisons.append(rescaled_comparison_log2_ratios)
        mod_utils.makePickle(comparison_log2_ratios, os.path.join(outfolder, experimental_dict_names[i]+'_comparison_log2.pkl'))
        #mod_utils.makePickle(rescaled_comparison_log2_ratios, os.path.join(outfolder, experimental_dict_names[i]+'_rescaled_comparison_log2.pkl'))
        write_wig(comparison_log2_ratios, experimental_dict_names[i]+'_comparison_log2', os.path.join(outfolder, experimental_dict_names[i]+'_comparison_log2'))
        #write_wig(rescaled_comparison_log2_ratios, experimental_dict_names[i]+'_rescaled)comparison_log2', os.path.join(outfolder, experimental_dict_names[i]+'_rescaled_comparison_log2'))
        #try:
        #    plot_weighted_nts_pie(background_subtracted, genome_fasta, '%s backround-subtracted fractions' % experimental_dict_names[i], os.path.join(outfolder, experimental_dict_names[i]+'_sub_pie'))
        #except:
        #    pass
    #print comparisons
    #print rescaled_comparisons
    normed_mutation_rate_histogram(comparisons, experimental_dict_names, os.path.join(outfolder, '%s_comparison_histogram' % norm_name), title='log2 experiment/control', xlim = (-10, 10), min = -100, max =100, step = 0.1)
def main():
    outprefix, bundle1, bundle2, bundle3, bundle4, bundle5, datafile_name  = sys.argv[1:8]

    bundles = [bundle1, bundle2, bundle3, bundle4, bundle5]
    reactivities = mod_utils.unPickle(datafile_name)
    for i in range(1,6):
        infile = open(bundles[i-1])
        outfile = open(outprefix+'_bundle'+str(i)+'.pdb' ,'w')

        for line in infile:
            if line.startswith('ATOM'):
                chain = line[21]
                resi = int(line[22:28].strip())
                if i in rRNA_assignments and chain in rRNA_assignments[i] and resi in reactivities['+'][rRNA_assignments[i][chain]]:
                    new_line = '%s%6.3f%s' % (line[:60], reactivities['+'][rRNA_assignments[i][chain]][resi], line[66:])
                    assert len(line) == len(new_line)
                else:
                    new_line = '%s%6.4f%s' % (line[:60], 0.0, line[66:])
                    assert len(line) == len(new_line)

            elif line.startswith("ANISOU"):
                new_line = '' #remove the anisotropic b factors, I don't need them
            else:
                new_line = line

            outfile.write(new_line)
        infile.close()
        outfile.close()
def main():
    read_5p_ends_file, genome_fasta, outprefix = sys.argv[1:4]
    tp_tn_annotations = sys.argv[4:]#true positive and true negative annotations
    genome_dict = mod_utils.convertFastaToDict(genome_fasta)
    read_5p_ends = mod_utils.unPickle(read_5p_ends_file)
    normed_density_array = winsorize_norm_chromosome_data(read_5p_ends, 'S.c.18S_rRNA', '+', genome_dict, 'ACTG')
    real_tp_tn_data = []
    for filename in tp_tn_annotations:
        real_tp, real_tn = get_tp_tn(filename)
        real_tp_tn_data.append((os.path.basename(filename), real_tp, real_tn))

    roc_curves = {}
    for entry in real_tp_tn_data:
        roc_curves[entry[0]] = [[],[]]#x and y value arrays for each

    stepsize = 0.0001
    for cutoff in numpy.arange(0,1.+5*stepsize, stepsize):
        called_p = call_positives(normed_density_array, 'S.c.18S_rRNA', '+', genome_dict, 'AC', cutoff)
        for entry in real_tp_tn_data:
            #print called_p.intersection(entry[1])

            num_tp_called = len(called_p.intersection(entry[1]))#how many true positives called at this cutoff
            num_fp_called = len(called_p.intersection(entry[2]))#how many fp positives called at this cutoff
            roc_curves[entry[0]][0].append(100.*num_fp_called/float(len(entry[2])))#FP rate on x axis
            roc_curves[entry[0]][1].append(100.*num_tp_called/float(len(entry[1])))#TP rate on y axis

    plot_ROC_curves(roc_curves, outprefix)
def main():
    tp_tn_annotations, genome_fasta, outprefix = sys.argv[1:4]
    density_files = sys.argv[4:]
    sample_names = [os.path.basename(filename).split("_back_")[0] for filename in density_files]
    mutation_densities = [mod_utils.unPickle(pickled_density) for pickled_density in density_files]

    genome_dict = mod_utils.convertFastaToDict(genome_fasta)
    normed_density_arrays = [
        winsorize_norm_chromosome_data(mutation_density, "S.c.25S__rRNA", genome_dict, "AC")
        for mutation_density in mutation_densities
    ]
    real_tp, real_tn = get_tp_tn(tp_tn_annotations)
    roc_curves = {}
    for sample_name in sample_names:
        roc_curves[sample_name] = [[], []]  # x and y value arrays for each

    stepsize = 0.0001
    for cutoff in numpy.arange(0, 1.0 + 5 * stepsize, stepsize):
        for i in range(len(sample_names)):
            # the fasta file should be the EXACT one used for the pipeline, and the chromosome name below should match
            # the one in the FASTA file exactly
            called_p = call_positives(normed_density_arrays[i], "S.c.25S__rRNA", genome_dict, "AC", cutoff)
            num_tp_called = len(called_p.intersection(real_tp))  # how many true positives called at this cutoff
            num_fp_called = len(called_p.intersection(real_tn))  # how many fp positives called at this cutoff
            roc_curves[sample_names[i]][1].append(100.0 * num_tp_called / float(len(real_tp)))  # TP rate on y axis
            roc_curves[sample_names[i]][0].append(100.0 * num_fp_called / float(len(real_tn)))  # FP rate on x axis

    plot_ROC_curves(roc_curves, "S.c.25S__rRNA", outprefix)
def main():
    all_counts_file, all_depths_file, min_mutations, output_prefix = sys.argv[1:5]
    min_mutations = int(min_mutations)
    all_counts = mod_utils.unPickle(all_counts_file)
    all_depths = mod_utils.unPickle(all_depths_file)

    comparisons = (pair.split(',') for pair in sys.argv[5:])

    for comparison in comparisons:
        subtracted_rates, subtraction_errors = subtraction_norm(all_counts, all_depths, min_mutations, comparison)
        divided_rates, division_errors = division_norm(all_counts, all_depths, min_mutations, comparison)
        mod_utils.makePickle(subtracted_rates, '%s_%s_%s_sub_norm.pkl' % (output_prefix, comparison[0], comparison[1]))
        mod_utils.makePickle(subtraction_errors, '%s_%s_%s_sub_err.pkl' % (output_prefix, comparison[0], comparison[1]))
        mod_utils.makePickle(divided_rates, '%s_%s_%s_div_norm.pkl' % (output_prefix, comparison[0], comparison[1]))
        mod_utils.makePickle(division_errors, '%s_%s_%s_div_err.pkl' % (output_prefix, comparison[0], comparison[1]))
        for rRNA in subtracted_rates:
            write_out_counts(subtracted_rates, subtraction_errors, divided_rates, division_errors, rRNA, '%s_%s_%s_%s.txt' % (output_prefix, comparison[0], comparison[1], rRNA))
示例#6
0
def generate_single_mutation_rates_dict(chromosome, start, stop, folder, file_names, strip_suffix):
    combined_mutation_rates = {}
    for file_name in file_names:
        dataset_label = file_name.rstrip(strip_suffix)
        mutation_rates = mod_utils.unPickle(os.path.join(folder, file_name))

        mutation_array = [float(mutation_rates[chromosome][position]) if position in mutation_rates[chromosome] else 0.0 for position in range(start, stop+1)]
        combined_mutation_rates[dataset_label] = mutation_array
    return combined_mutation_rates
def main():
    outfolder, genome_fasta, normalization_file_name = sys.argv[1:4]
    experimental_file_names = sys.argv[4:]
    mod_utils.make_dir(outfolder)
    normalization_dict = mod_utils.unPickle(normalization_file_name)
    norm_name = '.'.join(os.path.basename(normalization_file_name).split('.')[:-2])
    experimental_dict_names = ['.'.join(os.path.basename(file_name).split('.')[:-2]) for file_name in experimental_file_names]
    experimental_dicts = [mod_utils.unPickle(file_name) for file_name in experimental_file_names]

    normed_mutation_rate_histogram(experimental_dicts, experimental_dict_names, os.path.join(outfolder, 'mutation_rate_histogram'), title='nonzero positions')
    background_subtracted_sets = []
    write_wig(normalization_dict, norm_name, os.path.join(outfolder, norm_name))
    for i in range(len(experimental_dict_names)):
        write_wig(experimental_dicts[i], experimental_dict_names[i], os.path.join(outfolder, experimental_dict_names[i]))
        background_subtracted = subtract_background(experimental_dicts[i], normalization_dict)
        background_subtracted_sets.append(background_subtracted)
        mod_utils.makePickle(background_subtracted, os.path.join(outfolder, experimental_dict_names[i]+'_subtracted.pkl'))
        write_wig(background_subtracted, experimental_dict_names[i]+'_subtracted', os.path.join(outfolder, experimental_dict_names[i]+'_subtracted'))
        try:
            plot_weighted_nts_pie(background_subtracted, genome_fasta, '%s backround-subtracted fractions' % experimental_dict_names[i], os.path.join(outfolder, experimental_dict_names[i]+'_sub_pie'))
        except:
            pass
    normed_mutation_rate_histogram(background_subtracted_sets, experimental_dict_names, os.path.join(outfolder, 'back_subtracted_mutation_rate_histogram'), title = 'nonzero positions, background subtracted')
def main():
    tp_tn_annotations, genome_fasta, outprefix = sys.argv[1:4]
    density_files = sys.argv[4:]
    sample_names = [os.path.basename(filename) for filename in density_files]
    mutation_densities = [mod_utils.unPickle(pickled_density) for pickled_density in density_files]

    genome_dict = mod_utils.convertFastaToDict(genome_fasta)
    normed_density_arrays = [winsorize_norm_chromosome_data(mutation_density, 'S.c.18S_rRNA', '+', genome_dict, 'ACTG') for mutation_density in mutation_densities]
    real_tp, real_tn = get_tp_tn(tp_tn_annotations)
    roc_curves = {}
    for sample_name in sample_names:
        roc_curves[sample_name] = [[],[]]#x and y value arrays for each

    stepsize = 0.0001
    for cutoff in numpy.arange(0,1.+5*stepsize, stepsize):
        for i in range(len(sample_names)):
            called_p = call_positives(normed_density_arrays[i], 'S.c.18S_rRNA', '+', genome_dict, 'AC', cutoff)
            num_tp_called = len(called_p.intersection(real_tp))#how many true positives called at this cutoff
            num_fp_called = len(called_p.intersection(real_tn))#how many fp positives called at this cutoff
            roc_curves[sample_names[i]][1].append(100.*num_tp_called/float(len(real_tp)))#TP rate on y axis
            roc_curves[sample_names[i]][0].append(100.*num_fp_called/float(len(real_tn)))#FP rate on x axis

    plot_ROC_curves(roc_curves, outprefix)
def count_reads(lib_settings):
    """
    """

    # Create empty dicts for storing counts data
    srt_dict = createStrandDict(strands) # Counts for 5' end of read our standard data format
    cov_dict = createStrandDict(strands) # Counts of times covered by a read
    mut_dict = createStrandDict(strands) # Counts of mismatches at a position
    read_mutations = defaultdict(int) #counts different types of mutations relative to read
    genome_mutations = defaultdict(int) #counts different types of mutations relative to genome
    mutations_by_read_position = defaultdict(dict)
    read_position_coverage = defaultdict(float)
    mutations_by_genome_position = defaultdict(dict)
    genome_position_coverage = defaultdict(float)
    mutated_nts = defaultdict(float)
    read_insertion_sizes = []
    genomic_deletion_sizes = []

    with gzip.open(lib_settings.get_mapped_reads_sam_gz(), 'r') as f:
        for line in f: # Iterate through SAM file lines
            if not line.startswith('@'):
                # Parse line into relevant strings

                fields = line.strip().split('\t')
                ID = fields[0] #the first field in the mapped file corresponds to a unique id number for that read- these should correspond to the names in the raw_seqs dictionary
                flag = int(fields[1])
                '''
                The flag field provides a lot of info about the read, it is the decimal representation of a bit string, each digit of which is true or false

                Bit 0 = The read was part of a pair during sequencing
                Bit 1 = The read is mapped in a pair
                Bit 2 = The query sequence is unmapped
                Bit 3 = The mate is unmapped
                Bit 4 = Strand of query (0=forward 1=reverse)
                So, to see if a flag represents a read on the  - strand, we evaluate (16 & 'flag'), where & is the bitwise and operator,
                which will be non-zero (True) only if this read is on the - strand
                '''
                if (4&flag):#if this is an unmapped read, don't bother
                    continue

                if (16&flag):
                    strand = '-'
                else:
                    strand = '+'
                chrom = fields[2]
                MAPQ = int(fields[4])
                if int(MAPQ) >= lib_settings.get_property('min_mapping_quality'):
                    cigarString = fields[5]
                    seq = fields[9]
                    mappingLength = len(seq)
                    qScores = fields[10]
                    # Some lines seem to lack some strings this throws of indexing of NM:i, MD:Z, and NH:i strings
                    NHstr = checkTag('NH:i:',fields)
                    NMstr = checkTag('NM:i:',fields)
                    MDstr = checkTag('MD:Z:',fields)
                    assert 'NM:i' in NMstr
                    assert 'MD:Z' in MDstr
                    assert 'NH:i' in NHstr
                    multiplicity = float(NHstr.split(':')[2])

                    fields = line.strip().split('\t')
                    counts = float(1.0/multiplicity) # Weight of read
                    MDzString = MDstr

                    # Add subdicts for chromosome if needed
                    if chrom not in srt_dict[strand]:
                        srt_dict[strand][chrom] = defaultdict(float)
                    if chrom not in cov_dict[strand]:
                        cov_dict[strand][chrom] = defaultdict(float)
                    if chrom not in mut_dict[strand]:
                        mut_dict[strand][chrom] = defaultdict(float)

                    # Parse cigar string, get genome mapping span, and relative genomic positions covered by read
                    rel_genomic_event_positions, rel_genome_coverage, mutations_rel_genome, mutations_rel_read, readMappingSpan, genomeMappingSpan = parse_MDz_and_cigar(cigarString, MDzString, mappingLength, seq)

                    for pos in range(len(mutations_rel_genome)):
                        genome_position_coverage[pos] += counts
                        event = mutations_rel_genome[pos]
                        if not event == 'M': #count if it's not a match
                            assert event[0] != 'I'
                            if event[0] == 'D':
                                genomic_deletion_sizes.append(event[1])
                                event = event[0]
                            if event not in mutations_by_genome_position[pos]:
                                 mutations_by_genome_position[pos][event] = 0
                            mutations_by_genome_position[pos][event] += counts
                            genome_mutations[event] += counts
                            if event[0] in 'ATCG':
                                mutated_nts[event[0]] += counts

                    for pos in range(len(mutations_rel_read)):
                        read_position_coverage[pos] += counts
                        event = mutations_rel_read[pos]
                        if not event == 'M': #count if it's not a match
                            assert event[0] != 'D'
                            if event[0] == 'I':
                                read_insertion_sizes.append(event[1])
                                event = event[0]
                            if event not in mutations_by_read_position[pos]:
                                 mutations_by_read_position[pos][event] = 0
                            mutations_by_read_position[pos][event] += counts
                            read_mutations[event] += counts

                    # Set start position of read
                    if strand== '+':
                        start=int(fields[3])
                    else:
                        #When a read maps to the minus strand, bowtie returns the reverse complement, and indicates
                        # where this reverse mapped on the + strand. Thus the original 5' end of the read actually
                        # was x nt downstream on the + strand
                        start=int(fields[3])+genomeMappingSpan-1

                    # translate relative positions to absolute positions
                    genome_cov = readGenomicCoverage(rel_genome_coverage, strand, start) # get genome coverage

                    srt_dict[strand][chrom][start] += counts #just add the number of counts to that start position
                    for pos in genome_cov: # Increment positions for coverage dict
                        cov_dict[strand][chrom][pos] += counts

                    # If mismatches need to parse, get the absolute genomic pos, and increment counters
                    genMismatches = readGenomicCoverage(rel_genomic_event_positions, strand, start)
                    for event_position in genMismatches:
                        mut_dict[strand][chrom][event_position] += counts




    mod_utils.makePickle(srt_dict, lib_settings.get_read_5p_counts())
    mod_utils.makePickle(cov_dict, lib_settings.get_positional_coverage())
    mod_utils.makePickle(mut_dict, lib_settings.get_mutation_counts())

    mod_utils.makePickle(genome_mutations, lib_settings.get_counting_prefix() + '.genome_mutations.pkl')
    mod_utils.makePickle(mutations_by_genome_position, lib_settings.get_counting_prefix() + '.genome_position_mutations.pkl')
    mod_utils.makePickle(genome_position_coverage, lib_settings.get_counting_prefix() + '.genome_position_coverage.pkl')

    mod_utils.makePickle(mutated_nts, lib_settings.get_counting_prefix() + '.nt_mutations.pkl')

    mod_utils.makePickle(read_mutations, lib_settings.get_counting_prefix() + '.read_mutations.pkl')
    mod_utils.makePickle(mutations_by_read_position, lib_settings.get_counting_prefix() + '.read_position_mutations.pkl')
    mod_utils.makePickle(read_position_coverage, lib_settings.get_counting_prefix() + '.read_position_coverage.pkl')

    mod_utils.makePickle(genomic_deletion_sizes, lib_settings.get_counting_prefix() + '.deletion_sizes.pkl')

    mod_utils.makePickle(read_insertion_sizes, lib_settings.get_counting_prefix() + '.insertion_sizes.pkl')

    normalized_mutations = normalized_mutation_rates(mod_utils.unPickle(lib_settings.get_mutation_counts()), mod_utils.unPickle(lib_settings.get_positional_coverage()))
    mod_utils.makePickle(normalized_mutations, lib_settings.get_normalized_mutation_counts())



    plot_mutated_nts_pie(mod_utils.unPickle(lib_settings.get_counting_prefix() + '.nt_mutations.pkl'), 'mutated rRNA nts in ' + lib_settings.sample_name, lib_settings.get_counting_prefix()+'.mutated_nts' )
    plot_full_mutation_stats(mod_utils.unPickle(lib_settings.get_counting_prefix() + '.read_mutations.pkl'), mod_utils.unPickle(lib_settings.get_counting_prefix() + '.insertion_sizes.pkl'),
                             mod_utils.unPickle(lib_settings.get_counting_prefix() + '.read_position_mutations.pkl'), mod_utils.unPickle(lib_settings.get_counting_prefix() + '.read_position_coverage.pkl'), 'mutations wrt reads', "insertion size",
                             lib_settings.get_counting_prefix()+'.read_mutations')
    plot_full_mutation_stats(mod_utils.unPickle(lib_settings.get_counting_prefix() + '.genome_mutations.pkl'), mod_utils.unPickle(lib_settings.get_counting_prefix() + '.deletion_sizes.pkl'), mod_utils.unPickle(lib_settings.get_counting_prefix() + '.genome_position_mutations.pkl'),
                             mod_utils.unPickle(lib_settings.get_counting_prefix() + '.genome_position_coverage.pkl'), 'mutations wrt genome', "deletion size",
                             lib_settings.get_counting_prefix()+'.genome_mutations')
    pie_read_5p_ends(mod_utils.unPickle(lib_settings.get_read_5p_counts()), mod_utils.convertFastaToDict(lib_settings.experiment_settings.get_rRNA_fasta()), lib_settings.get_counting_prefix())
    normed_mutation_rate_histogram(mod_utils.unPickle(lib_settings.get_normalized_mutation_counts()), lib_settings.sample_name, lib_settings.get_counting_prefix())