def merge_DBD_regions(path): """Merge all available DBD regions in BED format. """ for t in os.listdir(path): if os.path.isdir(os.path.join(path, t)): dbd_pool = GenomicRegionSet(t) for rna in os.listdir(os.path.join(path,t)): f = os.path.join(path, t, rna, "DBD_"+rna+".bed") if os.path.exists(f): dbd = GenomicRegionSet(rna) dbd.read_bed(f) for r in dbd: r.name = rna+"_"+r.name dbd_pool.combine(dbd) dbd_pool.write_bed(os.path.join(path, t, "DBD_"+t+".bed"))
def get_bc_signal(arguments): (mpbs_name, mpbs_file1, mpbs_file2, reads_file1, reads_file2, organism, window_size, forward_shift, reverse_shift, bias_table1, bias_table2) = arguments mpbs1 = GenomicRegionSet("Motif Predicted Binding Sites of Condition1") mpbs1.read(mpbs_file1) mpbs2 = GenomicRegionSet("Motif Predicted Binding Sites of Condition2") mpbs2.read(mpbs_file2) mpbs = mpbs1.combine(mpbs2, output=True) mpbs.sort() bam1 = Samfile(reads_file1, "rb") bam2 = Samfile(reads_file2, "rb") genome_data = GenomeData(organism) fasta = Fastafile(genome_data.get_genome()) signal_1 = np.zeros(window_size) signal_2 = np.zeros(window_size) motif_len = None pwm = dict([("A", [0.0] * window_size), ("C", [0.0] * window_size), ("G", [0.0] * window_size), ("T", [0.0] * window_size), ("N", [0.0] * window_size)]) mpbs_regions = mpbs.by_names([mpbs_name]) num_motif = len(mpbs_regions) # Fetch bias corrected signal for region in mpbs_regions: if motif_len is None: motif_len = region.final - region.initial mid = (region.final + region.initial) / 2 p1 = mid - window_size / 2 p2 = mid + window_size / 2 if p1 <= 0: continue # Fetch raw signal signal1 = bias_correction(chrom=region.chrom, start=p1, end=p2, bam=bam1, bias_table=bias_table1, genome_file_name=genome_data.get_genome(), forward_shift=forward_shift, reverse_shift=reverse_shift) signal2 = bias_correction(chrom=region.chrom, start=p1, end=p2, bam=bam2, bias_table=bias_table2, genome_file_name=genome_data.get_genome(), forward_shift=forward_shift, reverse_shift=reverse_shift) if len(signal1) != len(signal_1) or len(signal2) != len(signal_2): continue # smooth the signal signal_1 = np.add(signal_1, np.array(signal1)) signal_2 = np.add(signal_2, np.array(signal2)) update_pwm(pwm, fasta, region, p1, p2) return signal_1, signal_2, motif_len, pwm, num_motif
def get_raw_signal(arguments): (mpbs_name, mpbs_file1, mpbs_file2, reads_file1, reads_file2, organism, window_size, forward_shift, reverse_shift) = arguments mpbs1 = GenomicRegionSet("Motif Predicted Binding Sites of Condition1") mpbs1.read(mpbs_file1) mpbs2 = GenomicRegionSet("Motif Predicted Binding Sites of Condition2") mpbs2.read(mpbs_file2) mpbs = mpbs1.combine(mpbs2, output=True) mpbs.sort() bam1 = Samfile(reads_file1, "rb") bam2 = Samfile(reads_file2, "rb") genome_data = GenomeData(organism) fasta = Fastafile(genome_data.get_genome()) signal_1 = np.zeros(window_size) signal_2 = np.zeros(window_size) motif_len = None pwm = dict([("A", [0.0] * window_size), ("C", [0.0] * window_size), ("G", [0.0] * window_size), ("T", [0.0] * window_size), ("N", [0.0] * window_size)]) mpbs_regions = mpbs.by_names([mpbs_name]) num_motif = len(mpbs_regions) for region in mpbs_regions: if motif_len is None: motif_len = region.final - region.initial mid = (region.final + region.initial) / 2 p1 = mid - window_size / 2 p2 = mid + window_size / 2 if p1 <= 0: continue # Fetch raw signal for read in bam1.fetch(region.chrom, p1, p2): # check if the read is unmapped, according to issue #112 if read.is_unmapped: continue if not read.is_reverse: cut_site = read.pos + forward_shift if p1 <= cut_site < p2: signal_1[cut_site - p1] += 1.0 else: cut_site = read.aend + reverse_shift - 1 if p1 <= cut_site < p2: signal_1[cut_site - p1] += 1.0 for read in bam2.fetch(region.chrom, p1, p2): # check if the read is unmapped, according to issue #112 if read.is_unmapped: continue if not read.is_reverse: cut_site = read.pos + forward_shift if p1 <= cut_site < p2: signal_2[cut_site - p1] += 1.0 else: cut_site = read.aend + reverse_shift - 1 if p1 <= cut_site < p2: signal_2[cut_site - p1] += 1.0 update_pwm(pwm, fasta, region, p1, p2) return signal_1, signal_2, motif_len, pwm, num_motif
def diff_analysis_run(args): # Initializing Error Handler err = ErrorHandler() output_location = os.path.join(args.output_location, "Lineplots") try: if not os.path.isdir(output_location): os.makedirs(output_location) except Exception: err.throw_error("MM_OUT_FOLDER_CREATION") # Check if the index file exists base_name1 = "{}.bai".format(args.reads_file1) if not os.path.exists(base_name1): pysam.index(args.reads_file1) base_name2 = "{}.bai".format(args.reads_file2) if not os.path.exists(base_name2): pysam.index(args.reads_file2) mpbs1 = GenomicRegionSet("Motif Predicted Binding Sites of Condition1") mpbs1.read(args.mpbs_file1) mpbs2 = GenomicRegionSet("Motif Predicted Binding Sites of Condition2") mpbs2.read(args.mpbs_file2) mpbs = mpbs1.combine(mpbs2, output=True) mpbs.sort() mpbs.remove_duplicates() mpbs_name_list = list(set(mpbs.get_names())) signal_dict_by_tf_1 = dict() signal_dict_by_tf_2 = dict() motif_len_dict = dict() motif_num_dict = dict() pwm_dict_by_tf = dict() pool = Pool(processes=args.nc) # differential analysis using bias corrected signal if args.bc: hmm_data = HmmData() table_F = hmm_data.get_default_bias_table_F_ATAC() table_R = hmm_data.get_default_bias_table_R_ATAC() bias_table1 = BiasTable().load_table(table_file_name_F=table_F, table_file_name_R=table_R) bias_table2 = BiasTable().load_table(table_file_name_F=table_F, table_file_name_R=table_R) mpbs_list = list() for mpbs_name in mpbs_name_list: mpbs_list.append((mpbs_name, args.mpbs_file1, args.mpbs_file2, args.reads_file1, args.reads_file2, args.organism, args.window_size, args.forward_shift, args.reverse_shift, bias_table1, bias_table2)) try: res = pool.map(get_bc_signal, mpbs_list) except Exception: logging.exception("get bias corrected signal failed") # differential analysis using raw signal else: mpbs_list = list() for mpbs_name in mpbs_name_list: mpbs_list.append((mpbs_name, args.mpbs_file1, args.mpbs_file2, args.reads_file1, args.reads_file2, args.organism, args.window_size, args.forward_shift, args.reverse_shift)) try: res = pool.map(get_raw_signal, mpbs_list) except Exception: logging.exception("get raw signal failed") for idx, mpbs_name in enumerate(mpbs_name_list): signal_dict_by_tf_1[mpbs_name] = res[idx][0] signal_dict_by_tf_2[mpbs_name] = res[idx][1] motif_len_dict[mpbs_name] = res[idx][2] pwm_dict_by_tf[mpbs_name] = res[idx][3] motif_num_dict[mpbs_name] = res[idx][4] if args.factor1 is None or args.factor2 is None: args.factor1, args.factor2 = compute_factors(signal_dict_by_tf_1, signal_dict_by_tf_2) output_factor(args, args.factor1, args.factor2) if args.output_profiles: output_profiles(mpbs_name_list, signal_dict_by_tf_1, output_location, args.condition1) output_profiles(mpbs_name_list, signal_dict_by_tf_2, output_location, args.condition2) ps_tc_results_by_tf = dict() plots_list = list() for mpbs_name in mpbs_name_list: plots_list.append((mpbs_name, motif_num_dict[mpbs_name], signal_dict_by_tf_1[mpbs_name], signal_dict_by_tf_2[mpbs_name], args.factor1, args.factor2, args.condition1, args.condition2, pwm_dict_by_tf[mpbs_name], output_location, args.window_size, args.standardize)) pool.map(line_plot, plots_list) for mpbs_name in mpbs_name_list: res = get_ps_tc_results(signal_dict_by_tf_1[mpbs_name], signal_dict_by_tf_2[mpbs_name], args.factor1, args.factor2, motif_num_dict[mpbs_name], motif_len_dict[mpbs_name]) # # # only use the factors whose protection scores are greater than 0 # if res[0] > 0 and res[1] < 0: ps_tc_results_by_tf[mpbs_name] = res # # stat_results_by_tf = get_stat_results(ps_tc_results_by_tf) ps_tc_results_by_tf = scatter_plot(args, ps_tc_results_by_tf) output_stat_results(args, ps_tc_results_by_tf, motif_num_dict)
def posi2region(self, regions, p): all = range(len(regions)) new_r = GenomicRegionSet(name="") for r in p: new_r.combine(regions[r]) return new_r
def diff_analysis_run(args): # Initializing Error Handler err = ErrorHandler() output_location = os.path.join(args.output_location, "Lineplots") try: if not os.path.isdir(output_location): os.makedirs(output_location) except Exception: err.throw_error("MM_OUT_FOLDER_CREATION") mpbs1 = GenomicRegionSet("Motif Predicted Binding Sites of Condition1") mpbs1.read(args.mpbs_file1) mpbs2 = GenomicRegionSet("Motif Predicted Binding Sites of Condition2") mpbs2.read(args.mpbs_file2) mpbs = mpbs1.combine(mpbs2, output=True) mpbs.sort() mpbs_name_list = list(set(mpbs.get_names())) signal_dict_by_tf_1 = dict() signal_dict_by_tf_2 = dict() motif_len_dict = dict() motif_num_dict = dict() pwm_dict_by_tf = dict() pool = Pool(processes=args.nc) # differential analysis using bias corrected signal if args.bc: hmm_data = HmmData() table_F = hmm_data.get_default_bias_table_F_ATAC() table_R = hmm_data.get_default_bias_table_R_ATAC() bias_table1 = BiasTable().load_table(table_file_name_F=table_F, table_file_name_R=table_R) bias_table2 = BiasTable().load_table(table_file_name_F=table_F, table_file_name_R=table_R) mpbs_list = list() for mpbs_name in mpbs_name_list: mpbs_list.append( (mpbs_name, args.mpbs_file1, args.mpbs_file2, args.reads_file1, args.reads_file2, args.organism, args.window_size, args.forward_shift, args.reverse_shift, bias_table1, bias_table2)) try: res = pool.map(get_bc_signal, mpbs_list) except Exception: logging.exception("get bias corrected signal failed") # differential analysis using raw signal else: mpbs_list = list() for mpbs_name in mpbs_name_list: mpbs_list.append( (mpbs_name, args.mpbs_file1, args.mpbs_file2, args.reads_file1, args.reads_file2, args.organism, args.window_size, args.forward_shift, args.reverse_shift)) try: res = pool.map(get_raw_signal, mpbs_list) except Exception: logging.exception("get raw signal failed") for idx, mpbs_name in enumerate(mpbs_name_list): signal_dict_by_tf_1[mpbs_name] = res[idx][0] signal_dict_by_tf_2[mpbs_name] = res[idx][1] motif_len_dict[mpbs_name] = res[idx][2] pwm_dict_by_tf[mpbs_name] = res[idx][3] motif_num_dict[mpbs_name] = res[idx][4] if args.factor1 is None or args.factor2 is None: args.factor1, args.factor2 = compute_factors(signal_dict_by_tf_1, signal_dict_by_tf_2) output_factor(args, args.factor1, args.factor2) if args.output_profiles: output_profiles(mpbs_name_list, signal_dict_by_tf_1, output_location, args.condition1) output_profiles(mpbs_name_list, signal_dict_by_tf_2, output_location, args.condition2) ps_tc_results_by_tf = dict() plots_list = list() for mpbs_name in mpbs_name_list: plots_list.append( (mpbs_name, motif_num_dict[mpbs_name], signal_dict_by_tf_1[mpbs_name], signal_dict_by_tf_2[mpbs_name], args.factor1, args.factor2, args.condition1, args.condition2, pwm_dict_by_tf[mpbs_name], output_location, args.window_size, args.standardize)) pool.map(line_plot, plots_list) for mpbs_name in mpbs_name_list: res = get_ps_tc_results(signal_dict_by_tf_1[mpbs_name], signal_dict_by_tf_2[mpbs_name], args.factor1, args.factor2, motif_num_dict[mpbs_name], motif_len_dict[mpbs_name]) # # # only use the factors whose protection scores are greater than 0 # if res[0] > 0 and res[1] < 0: ps_tc_results_by_tf[mpbs_name] = res # stat_results_by_tf = get_stat_results(ps_tc_results_by_tf) scatter_plot(args, stat_results_by_tf) output_stat_results(args, stat_results_by_tf, motif_num_dict)
def get_bc_signal(arguments): (mpbs_name, mpbs_file1, mpbs_file2, reads_file1, reads_file2, organism, window_size, forward_shift, reverse_shift, bias_table1, bias_table2) = arguments mpbs1 = GenomicRegionSet("Motif Predicted Binding Sites of Condition1") mpbs1.read(mpbs_file1) mpbs2 = GenomicRegionSet("Motif Predicted Binding Sites of Condition2") mpbs2.read(mpbs_file2) mpbs = mpbs1.combine(mpbs2, output=True) mpbs.sort() bam1 = Samfile(reads_file1, "rb") bam2 = Samfile(reads_file2, "rb") genome_data = GenomeData(organism) fasta = Fastafile(genome_data.get_genome()) signal_1 = np.zeros(window_size) signal_2 = np.zeros(window_size) motif_len = None pwm = dict([("A", [0.0] * window_size), ("C", [0.0] * window_size), ("G", [0.0] * window_size), ("T", [0.0] * window_size), ("N", [0.0] * window_size)]) mpbs_regions = mpbs.by_names([mpbs_name]) num_motif = len(mpbs_regions) # Fetch bias corrected signal for region in mpbs_regions: if motif_len is None: motif_len = region.final - region.initial mid = (region.final + region.initial) / 2 p1 = mid - window_size / 2 p2 = mid + window_size / 2 if p1 <= 0: continue # Fetch raw signal signal1 = bias_correction(chrom=region.chrom, start=p1, end=p2, bam=bam1, bias_table=bias_table1, genome_file_name=genome_data.get_genome(), forward_shift=forward_shift, reverse_shift=reverse_shift) signal2 = bias_correction(chrom=region.chrom, start=p1, end=p2, bam=bam2, bias_table=bias_table2, genome_file_name=genome_data.get_genome(), forward_shift=forward_shift, reverse_shift=reverse_shift) if len(signal1) != len(signal_1) or len(signal2) != len(signal_2): continue signal_1 = np.add(signal_1, np.array(signal1)) signal_2 = np.add(signal_2, np.array(signal2)) update_pwm(pwm, fasta, region, p1, p2) return signal_1, signal_2, motif_len, pwm, num_motif
def get_raw_signal(arguments): (mpbs_name, mpbs_file1, mpbs_file2, reads_file1, reads_file2, organism, window_size, forward_shift, reverse_shift) = arguments mpbs1 = GenomicRegionSet("Motif Predicted Binding Sites of Condition1") mpbs1.read(mpbs_file1) mpbs2 = GenomicRegionSet("Motif Predicted Binding Sites of Condition2") mpbs2.read(mpbs_file2) mpbs = mpbs1.combine(mpbs2, output=True) mpbs.sort() bam1 = Samfile(reads_file1, "rb") bam2 = Samfile(reads_file2, "rb") genome_data = GenomeData(organism) fasta = Fastafile(genome_data.get_genome()) signal_1 = np.zeros(window_size) signal_2 = np.zeros(window_size) motif_len = None pwm = dict([("A", [0.0] * window_size), ("C", [0.0] * window_size), ("G", [0.0] * window_size), ("T", [0.0] * window_size), ("N", [0.0] * window_size)]) mpbs_regions = mpbs.by_names([mpbs_name]) num_motif = len(mpbs_regions) for region in mpbs_regions: if motif_len is None: motif_len = region.final - region.initial mid = (region.final + region.initial) / 2 p1 = mid - window_size / 2 p2 = mid + window_size / 2 if p1 <= 0: continue # Fetch raw signal for read in bam1.fetch(region.chrom, p1, p2): if not read.is_reverse: cut_site = read.pos + forward_shift if p1 <= cut_site < p2: signal_1[cut_site - p1] += 1.0 else: cut_site = read.aend + reverse_shift - 1 if p1 <= cut_site < p2: signal_1[cut_site - p1] += 1.0 for read in bam2.fetch(region.chrom, p1, p2): if not read.is_reverse: cut_site = read.pos + forward_shift if p1 <= cut_site < p2: signal_2[cut_site - p1] += 1.0 else: cut_site = read.aend + reverse_shift - 1 if p1 <= cut_site < p2: signal_2[cut_site - p1] += 1.0 update_pwm(pwm, fasta, region, p1, p2) return signal_1, signal_2, motif_len, pwm, num_motif
class Boxplot: """ input: exps: input experimental matrix title: Default = boxplot groupby: Group the data by the given factor in the header of experimental matrix output: parameters: list of records figs: a list of figure(s) """ def __init__(self, EMpath, fields, title="boxplot", df=False): # Read the Experimental Matrix self.title = title self.exps = ExperimentalMatrix() self.exps.read(EMpath) for f in fields: if f not in ["None", "reads", "regions", "factor"]: self.exps.match_ms_tags(f) self.exps.remove_name() self.beds = self.exps.get_regionsets() # A list of GenomicRegionSets self.bednames = self.exps.get_regionsnames() self.reads = self.exps.get_readsfiles() self.readsnames = self.exps.get_readsnames() self.fieldsDict = self.exps.fieldsDict self.parameter = [] self.df = df def combine_allregions(self): self.all_bed = GenomicRegionSet("All regions") for bed in self.beds: self.all_bed.combine(bed) self.all_bed.remove_duplicates() # all_bed is sorted!! def bedCoverage(self): """ Return coverage matrix of multiple reads on one bed. bed --> GenomicRegionSet """ c = [] for rp in self.reads: print(" processing: ..." + rp[-45:]) r = os.path.abspath(rp) # Here change the relative path into absolute path cov = CoverageSet(r, self.all_bed) cov.coverage_from_genomicset(r) cov.normRPM() c.append(cov.coverage) self.all_table = numpy.transpose(c) def quantile_normalization(self): """ Return the np.array which contains the normalized values """ rank_matrix = [] for c in range(self.all_table.shape[1]): col = self.all_table[:, c] rank_col = mstats.rankdata(col) rank_matrix.append(rank_col) ranks = numpy.array(rank_matrix) trans_rank = numpy.transpose(ranks) # Calculate for means of ranks print(" Calculating for the mean of ranked data...") sort_matrix = numpy.sort(self.all_table, axis=0) means = [] for r in range(self.all_table.shape[0]): row = [x for x in sort_matrix[r, :]] means.append(numpy.mean(row)) # Replace the value by new means print(" Replacing the data value by normalized mean...") normalized_table = numpy.around(trans_rank) for i, v in enumerate(means): normalized_table[normalized_table == i + 1] = v # print(rounded_rank) self.norm_table = normalized_table def tables_for_plot(self): """ Return a Dict which stores all tables for each bed with file name as its key. """ self.tableDict = OrderedDict() # Storage all tables for each bed with bedname as the key conList = [] # Store containers of beds iterList = [] for i, bed in enumerate(self.beds): self.tableDict[bed.name] = [] bed.sort() conList.append(bed.__iter__()) iterList.append(conList[-1].next()) for i, r in enumerate(self.all_bed.sequences): for j in range(len(self.beds)): while r > iterList[j]: try: iterList[j] = conList[j].next() except: break if r == iterList[j]: self.tableDict[self.beds[j].name].append(self.norm_table[i]) elif r < iterList[j]: continue def print_plot_table(self, directory, folder): for i, bed in enumerate(self.tableDict.keys()): # table = [] # header = ["chrom", "initial", "final"] # for rp in self.reads: # header.append(os.path.basename(rp)) # table.append(header) # for j, re in enumerate(self.beds[i]): # table.append([re.chrom, re.initial, re.final] + self.tableDict[bed][j].tolist()) # output_array(table, directory, folder, filename="table_" + bed + ".txt") output_array(self.tableDict[bed], directory, folder, filename="table_" + bed + ".txt") def group_tags(self, groupby, sortby, colorby): """Generate the tags for the grouping of plot Parameters: groupby = 'reads','regions','cell',or 'factor' colorby = 'reads','regions','cell',or 'factor' sortby = 'reads','regions','cell',or 'factor' """ self.tag_type = [groupby, sortby, colorby] if groupby == "None": self.group_tags = [""] else: self.group_tags = gen_tags(self.exps, groupby) if sortby == "None": self.sort_tags = [""] else: self.sort_tags = gen_tags(self.exps, sortby) if colorby == "None": self.color_tags = [""] else: self.color_tags = gen_tags(self.exps, colorby) def group_data(self, directory, folder, log=False): plotDict = OrderedDict() # Extracting the data from different bed_bams file cuesbed = OrderedDict() # Storing the cues for back tracking cuesbam = OrderedDict() for bedname in self.tableDict.keys(): plotDict[bedname] = OrderedDict() mt = numpy.array(self.tableDict[bedname]) cuesbed[bedname] = set(tag_from_r(self.exps, self.tag_type, bedname)) # cuesbed[bedname] = [tag for tag in self.exps.get_types(bedname) if tag in self.group_tags + self.sort_tags + self.color_tags] for i, readname in enumerate(self.readsnames): plotDict[bedname][readname] = mt[:, i] # print(plotDict[bedname][readname]) cuesbam[readname] = set(tag_from_r(self.exps, self.tag_type, readname)) # cuesbam[readname] = [tag for tag in self.exps.get_types(readname) if tag in self.group_tags + self.sort_tags + self.color_tags] sortDict = OrderedDict() # Storing the data by sorting tags for g in self.group_tags: # print(" "+g) sortDict[g] = OrderedDict() for a in self.sort_tags: # print(" "+a) sortDict[g][a] = OrderedDict() for c in self.color_tags: # sortDict[g][a][c] = None # print(" "+c) for i, bed in enumerate(cuesbed.keys()): if set([g, a, c]) >= cuesbed[bed]: sortDict[g][a][c] = [] for bam in cuesbam.keys(): if set([g, a, c]) >= cuesbam[bam]: if self.df: sortDict[g][a][c].append(plotDict[bed][bam]) if len(sortDict[g][a][c]) == 2: if log: sortDict[g][a][c][0] = numpy.log(sortDict[g][a][c][0]+1) sortDict[g][a][c][1] = numpy.log(sortDict[g][a][c][1]+1) sortDict[g][a][c] = numpy.subtract(sortDict[g][a][c][0], sortDict[g][a][c][1]).tolist() else: sortDict[g][a][c] = numpy.subtract(sortDict[g][a][c][0], sortDict[g][a][c][1]).tolist() else: sortDict[g][a][c] = plotDict[bed][bam] self.sortDict = sortDict def color_map(self, colorby, definedinEM): self.colors = colormap(self.exps, colorby, definedinEM) def print_table(self, directory, folder): self.printtable = OrderedDict() table = [] table.append(["#group_tag", "sort_tag", "color_tag", "Signals"]) for i, g in enumerate(self.group_tags): for k, a in enumerate(self.sort_tags): for j, c in enumerate(self.color_tags): table.append([g, a, c] + [str(x) for x in self.sortDict[g][a][c]]) # print(table) output_array(table, directory, folder, filename="output_table.txt") def plot(self, title, scol, logT=False, ylim=False, pw=3, ph=4): """ Return boxplot from the given tables. """ self.xtickrotation, self.xtickalign = 0, "center" if len(self.group_tags) < 2: ticklabelsize = pw * 1.5 else: ticklabelsize = pw * 6 tw = len(self.group_tags) * pw th = ph f, axarr = plt.subplots(1, len(self.group_tags), dpi=300, sharey=scol, figsize=(tw, th)) # f, axarr = plt.subplots(1, len(self.group_tags), dpi=300, sharey = scol) # nm = len(self.group_tags) * len(self.color_tags) * len(self.sort_tags) # if nm > 30: # f.set_size_inches(nm * 0.25 ,nm * 0.15) # legend_x = 1.2 # self.xtickrotation, self.xtickalign = 70,"right" canvas = FigureCanvas(f) canvas.set_window_title(title) try: axarr = axarr.reshape(-1) except: axarr = [axarr] # plt.subplots_adjust(bottom=0.3) if logT: if self.df: axarr[0].set_ylabel("Read number difference (log)", fontsize=ticklabelsize + 1) else: axarr[0].set_ylabel("Read number (log)", fontsize=ticklabelsize + 1) else: if self.df: axarr[0].set_ylabel("Read number difference", fontsize=ticklabelsize + 1) else: axarr[0].set_ylabel("Read number", fontsize=ticklabelsize + 1) for i, g in enumerate(self.sortDict.keys()): # if self.df: # axarr[i].set_title(g + "_df", y=1.02, fontsize=ticklabelsize + 2) # else: axarr[i].set_title(g, y=1.02, fontsize=ticklabelsize + 2) if logT and not self.df: axarr[i].set_yscale('log') else: axarr[i].locator_params(axis='y', nbins=4) axarr[i].tick_params(axis='y', direction='out') axarr[i].yaxis.tick_left() axarr[i].yaxis.grid(True, linestyle='-', which='major', color='lightgrey', alpha=0.7, zorder=1) if ylim: axarr[i].set_ylim([-ylim, ylim]) d = [] # Store data within group color_t = [] # Store tag for coloring boxes x_ticklabels = [] # Store ticklabels for j, a in enumerate(self.sortDict[g].keys()): # if len(a) > 10: # print(a) self.xtickrotation = 70 self.xtickalign = "right" for k, c in enumerate(self.sortDict[g][a].keys()): if self.sortDict[g][a][c] == None: # When there is no matching data, skip it continue else: if self.df: d.append(self.sortDict[g][a][c]) else: d.append([x + 1 for x in self.sortDict[g][a][c]]) color_t.append(self.colors[k]) x_ticklabels.append(a) # + "." + c # Fine tuning boxplot # print(d) bp = axarr[i].boxplot(d, notch=False, sym='o', vert=True, whis=1.5, positions=None, widths=None, patch_artist=True, bootstrap=None) z = 10 # zorder for boxplot plt.setp(bp['whiskers'], color='black', linestyle='-', linewidth=0.8, zorder=z) plt.setp(bp['fliers'], markerfacecolor='gray', color='white', alpha=0.3, markersize=1.8, zorder=z) plt.setp(bp['caps'], color='white', zorder=z) plt.setp(bp['medians'], color='black', linewidth=1.5, zorder=z + 1) legends = [] for patch, color in zip(bp['boxes'], color_t): patch.set_facecolor(color) # When missing the data, the color patch will exceeds patch.set_edgecolor("none") patch.set_zorder(z) legends.append(patch) # Fine tuning subplot axarr[i].set_xticks([len(self.color_tags) * n + 1 + (len(self.color_tags) - 1) / 2 for n, s in enumerate(self.sortDict[g].keys())]) # plt.xticks(xlocations, sort_tags, rotation=90, fontsize=10) axarr[i].set_xticklabels(self.sortDict[g].keys(), rotation=self.xtickrotation, ha=self.xtickalign) # axarr[i].set_xticklabels(self.sortDict[g].keys(), rotation=70, ha=self.xtickalign, fontsize=10) # axarr[i].set_ylim(bottom=0.95) for spine in ['top', 'right', 'left', 'bottom']: axarr[i].spines[spine].set_visible(False) axarr[i].tick_params(axis='x', which='both', bottom='off', top='off', labelbottom='on') axarr[i].tick_params(labelsize=ticklabelsize + 1) if scol: # plt.setp(axarr[i].get_yticklabels(),visible=False) axarr[i].minorticks_off() # axarr[i].tick_params(axis='y', which='both', left='off', right='off', labelbottom='off') else: plt.setp(axarr[i].get_yticklabels(), visible=True) axarr[i].tick_params(axis='y', which='both', left='on', right='off', labelbottom='on') # plt.setp(axarr[i].get_yticks(),visible=False) axarr[-1].legend(legends[0:len(self.color_tags)], self.color_tags, loc='center left', handlelength=1, handletextpad=1, columnspacing=2, borderaxespad=0., prop={'size': ticklabelsize + 1}, bbox_to_anchor=(1.05, 0.5)) # f.tight_layout(pad=2, h_pad=None, w_pad=None) # f.tight_layout() self.fig = f def gen_html(self, directory, title, align=50): dir_name = os.path.basename(directory) # check_dir(directory) html_header = title link_d = OrderedDict() link_d["Boxplot"] = "index.html" link_d["Parameters"] = "parameters.html" html = Html(name=html_header, links_dict=link_d, fig_rpath="../style", RGT_header=False, other_logo="viz", homepage="../index.html") # fp = os.path.join(dir,outputname,title) html.add_figure("boxplot.png", align="center") type_list = 'ssssssssssssssssssssssssssssssssssssssssssssss' #### Calculate p value #### plist = {} for g in self.sortDict.keys(): plist[g] = {} for s1 in self.sortDict[g].keys(): for c1 in self.sortDict[g][s1].keys(): data1 = self.sortDict[g][s1][c1] plist[g][s1 + c1] = {} for s2 in self.sortDict[g].keys(): for c2 in self.sortDict[g][s2].keys(): if s2 == s1 and c2 == c1: pass else: data2 = self.sortDict[g][s2][c2] u, p_value = mannwhitneyu(data1, data2) plist[g][s1 + c1][s2 + c2] = p_value print("Multiple test correction.") multiple_correction(plist) for g in self.sortDict.keys(): html.add_heading(g, size=4, bold=False) data_table = [] col_size_list = [15] header_list = ["p-value"] for s in self.sortDict[g].keys(): for c in self.sortDict[g][s1].keys(): header_list.append(s + "\n" + c) col_size_list.append(15) for s1 in self.sortDict[g].keys(): for c1 in self.sortDict[g][s1].keys(): row = [s1 + "\n" + c1] for s2 in self.sortDict[g].keys(): for c2 in self.sortDict[g][s2].keys(): if s2 == s1 and c2 == c1: row.append("-") else: p = plist[g][s1 + c1][s2 + c2] if p > 0.05: row.append(value2str(p)) else: row.append("<font color=\"red\">" + value2str(p) + "</font>") data_table.append(row) html.add_zebra_table(header_list, col_size_list, type_list, data_table, align=align + 50) # html.add_fixed_rank_sortable() html.write(os.path.join(directory, title, "index.html")) ## Parameters html = Html(name=html_header, links_dict=link_d, fig_rpath="../style", RGT_header=False, other_logo="viz", homepage="../index.html") header_list = ["Assumptions and hypothesis"] col_size_list = [50] data_table = [['All the regions among different BED files are normalized by quantile normalization.'], [ 'If there is any grouping problem, please check all the optional columns in input experimental matrix.']] html.add_zebra_table(header_list, col_size_list, type_list, data_table, align=align, cell_align="left") html.add_free_content(['<a href="parameters.txt" style="margin-left:100">See parameters</a>']) html.add_free_content(['<a href="experimental_matrix.txt" style="margin-left:100">See experimental matrix</a>']) html.write(os.path.join(directory, title, "parameters.html"))
class Boxplot: """ input: exps: input experimental matrix title: Default = boxplot groupby: Group the data by the given factor in the header of experimental matrix output: parameters: list of records figs: a list of figure(s) """ def __init__(self, EMpath, fields, title="boxplot", df=False): # Read the Experimental Matrix self.title = title self.exps = ExperimentalMatrix() self.exps.read(EMpath) for f in fields: if f not in ["None", "reads", "regions", "factor"]: self.exps.match_ms_tags(f) self.exps.remove_name() self.beds = self.exps.get_regionsets() # A list of GenomicRegionSets self.bednames = self.exps.get_regionsnames() self.reads = self.exps.get_readsfiles() self.readsnames = self.exps.get_readsnames() self.fieldsDict = self.exps.fieldsDict self.parameter = [] self.df = df def combine_allregions(self): self.all_bed = GenomicRegionSet("All regions") for bed in self.beds: self.all_bed.combine(bed) self.all_bed.remove_duplicates() # all_bed is sorted!! def bedCoverage(self): """ Return coverage matrix of multiple reads on one bed. bed --> GenomicRegionSet """ c = [] for rp in self.reads: print(" processing: ..." + rp[-45:]) r = os.path.abspath( rp) # Here change the relative path into absolute path cov = CoverageSet(r, self.all_bed) cov.coverage_from_genomicset(r) cov.normRPM() c.append(cov.coverage) self.all_table = numpy.transpose(c) def quantile_normalization(self): """ Return the np.array which contains the normalized values """ rank_matrix = [] for c in range(self.all_table.shape[1]): col = self.all_table[:, c] rank_col = mstats.rankdata(col) rank_matrix.append(rank_col) ranks = numpy.array(rank_matrix) trans_rank = numpy.transpose(ranks) # Calculate for means of ranks print(" Calculating for the mean of ranked data...") sort_matrix = numpy.sort(self.all_table, axis=0) means = [] for r in range(self.all_table.shape[0]): row = [x for x in sort_matrix[r, :]] means.append(numpy.mean(row)) # Replace the value by new means print(" Replacing the data value by normalized mean...") normalized_table = numpy.around(trans_rank) for i, v in enumerate(means): normalized_table[normalized_table == i + 1] = v # print(rounded_rank) self.norm_table = normalized_table def tables_for_plot(self): """ Return a Dict which stores all tables for each bed with file name as its key. """ self.tableDict = OrderedDict( ) # Storage all tables for each bed with bedname as the key conList = [] # Store containers of beds iterList = [] for i, bed in enumerate(self.beds): self.tableDict[bed.name] = [] bed.sort() conList.append(bed.__iter__()) iterList.append(conList[-1].next()) for i, r in enumerate(self.all_bed.sequences): for j in range(len(self.beds)): while r > iterList[j]: try: iterList[j] = conList[j].next() except: break if r == iterList[j]: self.tableDict[self.beds[j].name].append( self.norm_table[i]) elif r < iterList[j]: continue def print_plot_table(self, directory, folder): for i, bed in enumerate(self.tableDict.keys()): # table = [] # header = ["chrom", "initial", "final"] # for rp in self.reads: # header.append(os.path.basename(rp)) # table.append(header) # for j, re in enumerate(self.beds[i]): # table.append([re.chrom, re.initial, re.final] + self.tableDict[bed][j].tolist()) # output_array(table, directory, folder, filename="table_" + bed + ".txt") output_array(self.tableDict[bed], directory, folder, filename="table_" + bed + ".txt") def group_tags(self, groupby, sortby, colorby): """Generate the tags for the grouping of plot Parameters: groupby = 'reads','regions','cell',or 'factor' colorby = 'reads','regions','cell',or 'factor' sortby = 'reads','regions','cell',or 'factor' """ self.tag_type = [groupby, sortby, colorby] if groupby == "None": self.group_tags = [""] else: self.group_tags = gen_tags(self.exps, groupby) if sortby == "None": self.sort_tags = [""] else: self.sort_tags = gen_tags(self.exps, sortby) if colorby == "None": self.color_tags = [""] else: self.color_tags = gen_tags(self.exps, colorby) def group_data(self, directory, folder, log=False): plotDict = OrderedDict( ) # Extracting the data from different bed_bams file cuesbed = OrderedDict() # Storing the cues for back tracking cuesbam = OrderedDict() for bedname in self.tableDict.keys(): plotDict[bedname] = OrderedDict() mt = numpy.array(self.tableDict[bedname]) cuesbed[bedname] = set( tag_from_r(self.exps, self.tag_type, bedname)) # cuesbed[bedname] = [tag for tag in self.exps.get_types(bedname) if tag in self.group_tags + self.sort_tags + self.color_tags] for i, readname in enumerate(self.readsnames): plotDict[bedname][readname] = mt[:, i] # print(plotDict[bedname][readname]) cuesbam[readname] = set( tag_from_r(self.exps, self.tag_type, readname)) # cuesbam[readname] = [tag for tag in self.exps.get_types(readname) if tag in self.group_tags + self.sort_tags + self.color_tags] sortDict = OrderedDict() # Storing the data by sorting tags for g in self.group_tags: # print(" "+g) sortDict[g] = OrderedDict() for a in self.sort_tags: # print(" "+a) sortDict[g][a] = OrderedDict() for c in self.color_tags: # sortDict[g][a][c] = None # print(" "+c) for i, bed in enumerate(cuesbed.keys()): if set([g, a, c]) >= cuesbed[bed]: sortDict[g][a][c] = [] for bam in cuesbam.keys(): if set([g, a, c]) >= cuesbam[bam]: if self.df: sortDict[g][a][c].append( plotDict[bed][bam]) if len(sortDict[g][a][c]) == 2: if log: sortDict[g][a][c][ 0] = numpy.log( sortDict[g][a][c][0] + 1) sortDict[g][a][c][ 1] = numpy.log( sortDict[g][a][c][1] + 1) sortDict[g][a][ c] = numpy.subtract( sortDict[g][a][c][0], sortDict[g][a][c] [1]).tolist() else: sortDict[g][a][ c] = numpy.subtract( sortDict[g][a][c][0], sortDict[g][a][c] [1]).tolist() else: sortDict[g][a][c] = plotDict[bed][bam] self.sortDict = sortDict def color_map(self, colorby, definedinEM): self.colors = colormap(self.exps, colorby, definedinEM) def print_table(self, directory, folder): self.printtable = OrderedDict() table = [] table.append(["#group_tag", "sort_tag", "color_tag", "Signals"]) for i, g in enumerate(self.group_tags): for k, a in enumerate(self.sort_tags): for j, c in enumerate(self.color_tags): table.append([g, a, c] + [str(x) for x in self.sortDict[g][a][c]]) # print(table) output_array(table, directory, folder, filename="output_table.txt") def plot(self, title, scol, logT=False, ylim=False, pw=3, ph=4): """ Return boxplot from the given tables. """ self.xtickrotation, self.xtickalign = 0, "center" if len(self.group_tags) < 2: ticklabelsize = pw * 1.5 else: ticklabelsize = pw * 6 tw = len(self.group_tags) * pw th = ph f, axarr = plt.subplots(1, len(self.group_tags), dpi=300, sharey=scol, figsize=(tw, th)) # f, axarr = plt.subplots(1, len(self.group_tags), dpi=300, sharey = scol) # nm = len(self.group_tags) * len(self.color_tags) * len(self.sort_tags) # if nm > 30: # f.set_size_inches(nm * 0.25 ,nm * 0.15) # legend_x = 1.2 # self.xtickrotation, self.xtickalign = 70,"right" canvas = FigureCanvas(f) canvas.set_window_title(title) try: axarr = axarr.reshape(-1) except: axarr = [axarr] # plt.subplots_adjust(bottom=0.3) if logT: if self.df: axarr[0].set_ylabel("Read number difference (log)", fontsize=ticklabelsize + 1) else: axarr[0].set_ylabel("Read number (log)", fontsize=ticklabelsize + 1) else: if self.df: axarr[0].set_ylabel("Read number difference", fontsize=ticklabelsize + 1) else: axarr[0].set_ylabel("Read number", fontsize=ticklabelsize + 1) for i, g in enumerate(self.sortDict.keys()): # if self.df: # axarr[i].set_title(g + "_df", y=1.02, fontsize=ticklabelsize + 2) # else: axarr[i].set_title(g, y=1.02, fontsize=ticklabelsize + 2) if logT and not self.df: axarr[i].set_yscale('log') else: axarr[i].locator_params(axis='y', nbins=4) axarr[i].tick_params(axis='y', direction='out') axarr[i].yaxis.tick_left() axarr[i].yaxis.grid(True, linestyle='-', which='major', color='lightgrey', alpha=0.7, zorder=1) if ylim: axarr[i].set_ylim([-ylim, ylim]) d = [] # Store data within group color_t = [] # Store tag for coloring boxes x_ticklabels = [] # Store ticklabels for j, a in enumerate(self.sortDict[g].keys()): # if len(a) > 10: # print(a) self.xtickrotation = 70 self.xtickalign = "right" for k, c in enumerate(self.sortDict[g][a].keys()): if self.sortDict[g][a][ c] == None: # When there is no matching data, skip it continue else: if self.df: d.append(self.sortDict[g][a][c]) else: d.append([x + 1 for x in self.sortDict[g][a][c]]) color_t.append(self.colors[k]) x_ticklabels.append(a) # + "." + c # Fine tuning boxplot # print(d) bp = axarr[i].boxplot(d, notch=False, sym='o', vert=True, whis=1.5, positions=None, widths=None, patch_artist=True, bootstrap=None) z = 10 # zorder for boxplot plt.setp(bp['whiskers'], color='black', linestyle='-', linewidth=0.8, zorder=z) plt.setp(bp['fliers'], markerfacecolor='gray', color='white', alpha=0.3, markersize=1.8, zorder=z) plt.setp(bp['caps'], color='white', zorder=z) plt.setp(bp['medians'], color='black', linewidth=1.5, zorder=z + 1) legends = [] for patch, color in zip(bp['boxes'], color_t): patch.set_facecolor( color ) # When missing the data, the color patch will exceeds patch.set_edgecolor("none") patch.set_zorder(z) legends.append(patch) # Fine tuning subplot axarr[i].set_xticks([ len(self.color_tags) * n + 1 + (len(self.color_tags) - 1) / 2 for n, s in enumerate(self.sortDict[g].keys()) ]) # plt.xticks(xlocations, sort_tags, rotation=90, fontsize=10) axarr[i].set_xticklabels(self.sortDict[g].keys(), rotation=self.xtickrotation, ha=self.xtickalign) # axarr[i].set_xticklabels(self.sortDict[g].keys(), rotation=70, ha=self.xtickalign, fontsize=10) # axarr[i].set_ylim(bottom=0.95) for spine in ['top', 'right', 'left', 'bottom']: axarr[i].spines[spine].set_visible(False) axarr[i].tick_params(axis='x', which='both', bottom='off', top='off', labelbottom='on') axarr[i].tick_params(labelsize=ticklabelsize + 1) if scol: # plt.setp(axarr[i].get_yticklabels(),visible=False) axarr[i].minorticks_off() # axarr[i].tick_params(axis='y', which='both', left='off', right='off', labelbottom='off') else: plt.setp(axarr[i].get_yticklabels(), visible=True) axarr[i].tick_params(axis='y', which='both', left='on', right='off', labelbottom='on') # plt.setp(axarr[i].get_yticks(),visible=False) axarr[-1].legend(legends[0:len(self.color_tags)], self.color_tags, loc='center left', handlelength=1, handletextpad=1, columnspacing=2, borderaxespad=0., prop={'size': ticklabelsize + 1}, bbox_to_anchor=(1.05, 0.5)) # f.tight_layout(pad=2, h_pad=None, w_pad=None) # f.tight_layout() self.fig = f def gen_html(self, directory, title, align=50): dir_name = os.path.basename(directory) # check_dir(directory) html_header = title link_d = OrderedDict() link_d["Boxplot"] = "index.html" link_d["Parameters"] = "parameters.html" html = Html(name=html_header, links_dict=link_d, fig_rpath="../style", RGT_header=False, other_logo="viz", homepage="../index.html") # fp = os.path.join(dir,outputname,title) html.add_figure("boxplot.png", align="center") type_list = 'ssssssssssssssssssssssssssssssssssssssssssssss' #### Calculate p value #### plist = {} for g in self.sortDict.keys(): plist[g] = {} for s1 in self.sortDict[g].keys(): for c1 in self.sortDict[g][s1].keys(): data1 = self.sortDict[g][s1][c1] plist[g][s1 + c1] = {} for s2 in self.sortDict[g].keys(): for c2 in self.sortDict[g][s2].keys(): if s2 == s1 and c2 == c1: pass else: data2 = self.sortDict[g][s2][c2] u, p_value = mannwhitneyu(data1, data2) plist[g][s1 + c1][s2 + c2] = p_value print("Multiple test correction.") multiple_correction(plist) for g in self.sortDict.keys(): html.add_heading(g, size=4, bold=False) data_table = [] col_size_list = [15] header_list = ["p-value"] for s in self.sortDict[g].keys(): for c in self.sortDict[g][s1].keys(): header_list.append(s + "\n" + c) col_size_list.append(15) for s1 in self.sortDict[g].keys(): for c1 in self.sortDict[g][s1].keys(): row = [s1 + "\n" + c1] for s2 in self.sortDict[g].keys(): for c2 in self.sortDict[g][s2].keys(): if s2 == s1 and c2 == c1: row.append("-") else: p = plist[g][s1 + c1][s2 + c2] if p > 0.05: row.append(value2str(p)) else: row.append("<font color=\"red\">" + value2str(p) + "</font>") data_table.append(row) html.add_zebra_table(header_list, col_size_list, type_list, data_table, align=align + 50) # html.add_fixed_rank_sortable() html.write(os.path.join(directory, title, "index.html")) ## Parameters html = Html(name=html_header, links_dict=link_d, fig_rpath="../style", RGT_header=False, other_logo="viz", homepage="../index.html") header_list = ["Assumptions and hypothesis"] col_size_list = [50] data_table = [ [ 'All the regions among different BED files are normalized by quantile normalization.' ], [ 'If there is any grouping problem, please check all the optional columns in input experimental matrix.' ] ] html.add_zebra_table(header_list, col_size_list, type_list, data_table, align=align, cell_align="left") html.add_free_content([ '<a href="parameters.txt" style="margin-left:100">See parameters</a>' ]) html.add_free_content([ '<a href="experimental_matrix.txt" style="margin-left:100">See experimental matrix</a>' ]) html.write(os.path.join(directory, title, "parameters.html"))