def collapse_region_map(region_map_file, name="", control_bams=False): """Take a region_map file and collapse signal into a single column. Also fix any stupid start/stop sorting issues. Need to take into account whether or not controls were used. """ region_map = utils.parse_table(region_map_file, "\t") for n, line in enumerate(region_map): if n == 0: # new header if len(name) == 0: name = "MERGED_SIGNAL" region_map[n] = line[0:6] + [name] else: new_line = list(line[0:6]) if control_bams: signal_line = [float(x) for x in line[6:]] rankby_indexes = range(0, len(signal_line) // 2, 1) control_indexes = range( len(signal_line) // 2, len(signal_line), 1) meta_vector = [] for i, j in zip(rankby_indexes, control_indexes): # min signal is 0 meta_vector.append(max(0, signal_line[i] - signal_line[j])) meta_signal = numpy.mean(meta_vector) else: meta_signal = numpy.mean([float(x) for x in line[6:]]) region_map[n] = new_line + [meta_signal] output_file = region_map_file.replace("REGION", "META") utils.unparse_table(region_map, output_file, "\t") return output_file
def map_gff_line_to_bed(gff_line, out_folder, n_bins, bed_collection, header=""): """For every line produces a file with all of the rectangles to draw.""" if not header: gff_string = "{}_{}_{}_{}".format( gff_line[0], gff_line[6], gff_line[3], gff_line[4] ) else: gff_string = header diagram_table = [[0, 0, 0, 0]] name_table = [["", 0, 0]] gff_locus = utils.Locus( gff_line[0], int(gff_line[3]), int(gff_line[4]), gff_line[6], gff_line[1], ) scale_factor = n_bins / gff_locus.len() overlap_loci = bed_collection.get_overlap(gff_locus, sense="both") print( "IDENTIFIED {} OVERLAPPING BED LOCI FOR REGION {}".format( str(len(overlap_loci)), gff_line, ) ) # since beds come from multiple sources, we want to figure out how to offset them offset_dict = {} # this will store each ID name bed_names_list = utils.uniquify([locus.id for locus in overlap_loci]) bed_names_list.sort() for i in range(len(bed_names_list)): offset_dict[bed_names_list[i]] = ( 2 * i ) # offsets different categories of bed regions if gff_line[6] == "-": ref_point = int(gff_line[4]) else: ref_point = int(gff_line[3]) # fill out the name table for name in bed_names_list: offset = offset_dict[name] name_table.append([name, 0, 0.0 - offset]) for bed_locus in overlap_loci: offset = offset_dict[bed_locus.id] [start, stop] = [abs(x - ref_point) * scale_factor for x in bed_locus.coords()] diagram_table.append([start, -0.5 - offset, stop, 0.5 - offset]) utils.unparse_table( diagram_table, os.path.join(out_folder, "{}_bedDiagramTemp.txt".format(gff_string)), "\t", ) utils.unparse_table( name_table, os.path.join(out_folder, "{}_bedNameTemp.txt".format(gff_string)), "\t", )
def make_enhancer_signal_table(name_dict, merged_region_map, median_dict, analysis_name, genome, output_folder): """Makes a signal table. Each row is an enhancer and each column is the log2 background corrected signal vs. median. """ # load in the region map region_map = utils.parse_table(merged_region_map, "\t") names_list = list(name_dict.keys()) names_list.sort() signal_table = [[ "REGION_ID", "CHROM", "START", "STOP", "NUM_LOCI", "CONSTITUENT_SIZE" ] + names_list] print("len of {} for names_list".format(len(names_list))) print(names_list) for line in region_map[1:]: new_line = line[0:6] # a little tricky here to add datasets sequentially i = 6 # start w/ the first column w/ data for name in names_list: if name_dict[name]["background"] is True: enhancer_index = int(i) i += 1 control_index = int(i) i += 1 try: enhancer_signal = float(line[enhancer_index]) - float( line[control_index]) except IndexError: print(line) print(len(line)) print(enhancer_index) print(control_index) sys.exit() else: enhancer_index = int(i) i += 1 enhancer_signal = float(line[enhancer_index]) if enhancer_signal < 0: enhancer_signal = 0 enhancer_signal = enhancer_signal / median_dict[name] new_line.append(enhancer_signal) signal_table.append(new_line) output_file = os.path.join( output_folder, "{}_{}_signal_table.txt".format(genome, analysis_name)) print("WRITING MEDIAN NORMALIZED SIGNAL TABLE TO {}".format(output_file)) utils.unparse_table(signal_table, output_file, "\t") return output_file
def make_signal_table( names_list, gff_file, mapped_folder, median_norm=False, output="" ): """For each sample, make a dictionary keyed by locus ID.""" signal_dict = {} for name in names_list: signal_dict[name] = defaultdict(float) # now start filling in the signal dict gff_name = os.path.basename(gff_file).split(".")[0] print(gff_name) for name in names_list: print("MAKING SIGNAL DICT FOR %s" % (name)) # try opening the batch mapping output first mapped_file = os.path.join( mapped_folder, gff_name, "{}_{}.txt".format(gff_name, name) ) if utils.check_output(mapped_file, 0.02, 0.02): print("FOUND MAPPED FILE FOR {} AT {}".format(name, mapped_file)) else: mapped_file = os.path.join( mapped_folder, gff_name, "{}_{}.txt".format(gff_name, name), ) if utils.check_output(mapped_file, 0.02, 0.02): print("FOUND MAPPED FILE FOR {} AT {}".format(name, mapped_file)) else: print("ERROR NO MAPPED FILE FOUND FOR {}".format(name)) sys.exit() mapped_table = utils.parse_table(mapped_file, "\t") if median_norm: median_signal = numpy.median([float(line[2]) for line in mapped_table[1:]]) else: median_signal = 1 for line in mapped_table[1:]: signal_dict[name][line[1]] = float(line[2]) / median_signal # now make the signal table signal_table = [] header = ["GENE_ID", "locusLine"] + names_list signal_table.append(header) for line in mapped_table[1:]: locus_id = line[1] sig_line = line[0:2] + [signal_dict[name][locus_id] for name in names_list] signal_table.append(sig_line) if not output: return signal_table else: utils.unparse_table(signal_table, output, "\t") return signal_table
def merge_collections(name_dict, analysis_name, output="", super_only=True): """Merge them collections.""" all_loci = [] names_list = list(name_dict.keys()) for name in names_list: se_collection = make_se_collection(name_dict[name]["enhancer_file"], name, super_only) if super_only: print("DATASET: {} HAS {} SUPERENHANCERS".format( name, str(len(se_collection)))) else: print("DATASET: {} HAS {} ENHANCERS".format( name, str(len(se_collection)))) all_loci += se_collection.get_loci() print(str(len(all_loci))) merged_collection = utils.LocusCollection(all_loci, 50) # stitch the collection together stitched_collection = merged_collection.stitch_collection() stitched_loci = stitched_collection.get_loci() print("IDENTIFIED {} CONSENSUS ENHANCER REGIONS".format( str(len(stitched_loci)))) # sort by size and provide a unique ID size_list = [locus.len() for locus in stitched_loci] size_order = utils.order(size_list, decreasing=True) ordered_loci = [stitched_loci[i] for i in size_order] for i in range(len(ordered_loci)): ordered_loci[i].id = "merged_{}_{}".format(analysis_name, str(i + 1)) merged_gff = [] for locus in ordered_loci: new_line = [ locus.chr, locus.id, "", locus.start, locus.end, "", locus.sense, "", locus.id, ] merged_gff.append(new_line) if len(output) == 0: return merged_gff else: print("writing merged gff to {}".format(output)) utils.unparse_table(merged_gff, output, "\t") return output
def format_data_table(data_file): """Formats the data_file and rewrite. First 3 columns are required for every line. If they aren't there the line is deleted. """ print("reformatting data table") data_table = utils.parse_table(data_file, "\t") new_data_table = [ [ "FILE_PATH", "UNIQUE_ID", "GENOME", "NAME", "BACKGROUND", "ENRICHED_REGION", "ENRICHED_MACS", "COLOR", "FASTQ_FILE", ] ] # first check to make sure the table is formatted correctly for line in data_table[1:]: if len(line) < 3: continue # this spots header lines that may be out of place if line[0] == "FILE_PATH": continue # check if it at least has the first 3 columns filled in if len(line[0]) == 0 or len(line[1]) == 0 or len(line[2]) == 0: print("ERROR required fields missing in line") print(line) # if the first three are filled in, check to make sure there are 8 columns else: if len(line) > 3 and len(line) < 9: new_line = line + (8 - len(line)) * [""] + ["NA"] new_data_table.append(new_line) elif len(line) >= 9: new_line = line[0:9] new_data_table.append(new_line) # lower case all of the genomes # make the color 0,0,0 for blank lines and strip out any " marks for i in range(1, len(new_data_table)): new_data_table[i][2] = new_data_table[i][2].lower() color = new_data_table[i][7] if len(color) == 0: new_data_table[i][7] = "0,0,0" utils.unparse_table(new_data_table, data_file, "\t") return new_data_table
def assign_enhancer_rank(enhancer_to_gene_file, enhancer_file1, enhancer_file2, name1, name2, rank_output=""): """Assign enhancer rank to genes. For all genes in the enhancer_to_gene table, assign the highest overlapping ranked enhancer in the other tables. """ enhancer_to_gene = utils.parse_table(enhancer_to_gene_file, "\t") enhancer_collection1 = make_se_collection(enhancer_file1, name1, False) enhancer_collection2 = make_se_collection(enhancer_file2, name2, False) enhancer_dict1 = make_se_dict(enhancer_file1, name1, False) enhancer_dict2 = make_se_dict(enhancer_file2, name2, False) # we're going to update the enhancer_to_gene_table enhancer_to_gene[0] += ["{}_rank".format(name1), "{}_rank".format(name2)] for i in range(1, len(enhancer_to_gene)): line = enhancer_to_gene[i] locus_line = utils.Locus(line[1], line[2], line[3], ".", line[0]) # if the enhancer doesn't exist, its ranking is dead last on the enhancer list enhancer1_overlap = enhancer_collection1.get_overlap( locus_line, "both") if len(enhancer1_overlap) == 0: enhancer1_rank = len(enhancer_collection1) else: rank_list1 = [ enhancer_dict1[x.id]["rank"] for x in enhancer1_overlap ] enhancer1_rank = min(rank_list1) enhancer2_overlap = enhancer_collection2.get_overlap( locus_line, "both") if len(enhancer2_overlap) == 0: enhancer2_rank = len(enhancer_collection2) else: rank_list2 = [ enhancer_dict2[x.id]["rank"] for x in enhancer2_overlap ] enhancer2_rank = min(rank_list2) enhancer_to_gene[i] += [enhancer1_rank, enhancer2_rank] if len(rank_output) == 0: return enhancer_to_gene else: utils.unparse_table(enhancer_to_gene, rank_output, "\t")
def merge_collections(super_file1, super_file2, name1, name2, output=""): """Merge them collections.""" con_super_collection = make_se_collection(super_file1, name1) tnf_super_collection = make_se_collection(super_file2, name2) # now merge them merged_loci = con_super_collection.get_loci( ) + tnf_super_collection.get_loci() merged_collection = utils.LocusCollection(merged_loci, 50) # stitch the collection together stitched_collection = merged_collection.stitch_collection() stitched_loci = stitched_collection.get_loci() # loci that are in both get renamed with a new unique identifier renamed_loci = [] ticker = 1 for locus in stitched_loci: if len(con_super_collection.get_overlap(locus)) > 0 and len( tnf_super_collection.get_overlap(locus)): new_id = "CONSERVED_{}".format(str(ticker)) ticker += 1 locus.id = new_id else: locus.id = locus.id[2:] renamed_loci.append(locus) # now we turn this into a gff and write it out gff = utils.locus_collection_to_gff(utils.LocusCollection( renamed_loci, 50)) if len(output) == 0: return gff else: print("writing merged gff to {}".format(output)) utils.unparse_table(gff, output, "\t") return output
def finish_rank_output( data_file, rank_output, genome, merge_folder, merge_name, name1, name2, cut_off=1.5, window=100000, super_only=True, plot_bam=True, ): """Finish rank output. Clean up the rank output table. Make a gff of all of the gained/lost supers beyond a certain cut_off w/ a window. Make a list of gained genes and lost genes. Make a bed of gained loss. """ data_dict = pipeline_utils.load_data_table(data_file) # making sure window and cut_off are int/float cut_off = float(cut_off) window = int(window) genome = genome.upper() # make the output folder output_folder = utils.format_folder(os.path.join(merge_folder, "output"), True) # bring in the old rank table rank_enhancer_table = utils.parse_table(rank_output, "\t") # make a new formatted table header = rank_enhancer_table[0] header[-4] = "DELTA RANK" header[-3] = "IS_SUPER" formatted_rank_table = [header] # the gffs gained_gff = [] lost_gff = [] gained_window_gff = [] lost_window_gff = [] if super_only: enhancer_type = "SUPERS" else: enhancer_type = "ENHANCERS" # the beds if super_only: gained_track_header = ( 'track name="{} {} only SEs" description="{} super enhancers that are found only in ' '{} vs {}" itemRGB=On color=255,0,0'.format( genome, name2, genome, name2, name1)) gained_bed = [[gained_track_header]] conserved_track_header = ( 'track name="{} {} and {} SEs" description="{} super enhancers that are found in both' ' {} vs {}" itemRGB=On color=0,0,0'.format(genome, name1, name2, genome, name1, name2)) conserved_bed = [[conserved_track_header]] lost_track_header = ( 'track name="{} {} only SEs" description="{} super enhancers that are found only in ' '{} vs {}" itemRGB=On color=0,255,0'.format( genome, name1, genome, name1, name2)) lost_bed = [[lost_track_header]] else: gained_track_header = ( 'track name="{} {} only enhancers" description="{} enhancers that are found only in ' '{} vs {}" itemRGB=On color=255,0,0'.format( genome, name2, genome, name2, name1)) gained_bed = [[gained_track_header]] conserved_track_header = ( 'track name="{} {} and {} enhancers" description="{} enhancers that are found in both' ' {} vs {}" itemRGB=On color=0,0,0'.format(genome, name1, name2, genome, name1, name2)) conserved_bed = [[conserved_track_header]] lost_track_header = ( 'track name="{} {} only enhancers" description="{} enhancers that are found only in ' '{} vs {}" itemRGB=On color=0,255,0'.format( genome, name1, genome, name1, name2)) lost_bed = [[lost_track_header]] # the genes gene_table = [[ "GENE", "ENHANCER_ID", "ENHANCER_CHROM", "ENHANCER_START", "ENHANCER_STOP", header[6], header[7], header[8], "STATUS", ]] for line in rank_enhancer_table[1:]: # fixing the enhancer ID line[0] = line[0].replace("_lociStitched", "") formatted_rank_table.append(line) # getting the genes gene_list = [] gene_list += line[9].split(",") gene_list += line[10].split(",") gene_list += line[11].split(",") gene_list = [x for x in gene_list if len(x) > 0] gene_list = utils.uniquify(gene_list) gene_string = ",".join(gene_list) bed_line = [line[1], line[2], line[3], line[0], line[-4]] # for gained if float(line[6]) > cut_off: gff_line = [ line[1], line[0], "", line[2], line[3], "", ".", "", gene_string, ] gff_window_line = [ line[1], line[0], "", int(line[2]) - window, int(line[3]) + window, "", ".", "", gene_string, ] gained_gff.append(gff_line) gained_window_gff.append(gff_window_line) gene_status = name2 gained_bed.append(bed_line) # for lost elif float(line[6]) < (-1 * cut_off): gff_line = [ line[1], line[0], "", line[2], line[3], "", ".", "", gene_string, ] gff_window_line = [ line[1], line[0], "", int(line[2]) - window, int(line[3]) + window, "", ".", "", gene_string, ] lost_gff.append(gff_line) lost_window_gff.append(gff_window_line) gene_status = name1 lost_bed.append(bed_line) # for conserved else: gene_status = "CONSERVED" conserved_bed.append(bed_line) # now fill in the gene Table for gene in gene_list: gene_table_line = [ gene, line[0], line[1], line[2], line[3], line[6], line[7], line[8], gene_status, ] gene_table.append(gene_table_line) # concat the bed full_bed = gained_bed + conserved_bed + lost_bed # start writing the output # there's the two gffs, the bed,the formatted table, the gene table # formatted table formatted_filename = os.path.join( output_folder, "{}_{}_MERGED_{}_RANK_TABLE.txt".format(genome, merge_name, enhancer_type), ) utils.unparse_table(formatted_rank_table, formatted_filename, "\t") # gffs gff_folder = utils.format_folder(output_folder + "gff/", True) gff_filename_gained = os.path.join( gff_folder, "{}_{}_{}_ONLY_{}_-0_+0.gff".format(genome, merge_name, name2.upper(), enhancer_type), ) gff_filename_window_gained = os.path.join( gff_folder, "{}_{}_{}_ONLY_{}_-{}KB_+{}KB.gff".format( genome, merge_name, name2.upper(), enhancer_type, str(window // 1000), str(window // 1000), ), ) gff_filename_lost = os.path.join( gff_folder, "{}_{}_{}_ONLY_{}_-0_+0.gff".format(genome, merge_name, name1.upper(), enhancer_type), ) gff_filename_window_lost = os.path.join( gff_folder, "{}_{}_{}_ONLY_{}_-{}KB_+{}KB.gff".format( genome, merge_name, name1.upper(), enhancer_type, str(window // 1000), str(window // 1000), ), ) utils.unparse_table(gained_gff, gff_filename_gained, "\t") utils.unparse_table(gained_window_gff, gff_filename_window_gained, "\t") utils.unparse_table(lost_gff, gff_filename_lost, "\t") utils.unparse_table(lost_window_gff, gff_filename_window_lost, "\t") # bed bed_filename = os.path.join( output_folder, "{}_{}_MERGED_{}.bed".format(genome, merge_name, enhancer_type)) utils.unparse_table(full_bed, bed_filename, "\t") # gene_table gene_filename = os.path.join( output_folder, "{}_{}_MERGED_{}_GENE_TABLE.txt".format(genome, merge_name, enhancer_type), ) utils.unparse_table(gene_table, gene_filename, "\t") # finally, move all of the plots to the output folder copyfile( glob.glob(os.path.join(merge_folder, "{}_ROSE".format(name1), "*.pdf"))[0], os.path.join( output_folder, "{}_{}_MERGED_{}_DELTA.pdf".format(genome, merge_name, enhancer_type), ), ) copyfile( glob.glob( os.path.join(merge_folder, "{}_ROSE".format(name1), "*RANK_PLOT.png"))[0], os.path.join( output_folder, "{}_{}_MERGED_{}_RANK_PLOT.png".format(genome, merge_name, enhancer_type), ), ) # now execute the bamPlot_turbo commands if plot_bam: bam1 = data_dict[name1]["bam"] bam2 = data_dict[name2]["bam"] bam_string = "{} {}".format(bam1, bam2) name_string = "{} {}".format(name1, name2) color_string = "0,0,0:100,100,100" if len(gained_gff) > 0: # gained command plot_title = "{}_ONLY_SE".format(name2) cmd = ( "bamPlot_turbo -g {} -b {} -i {} -o {} -n {} -c {} -t {} -r -y UNIFORM -p " "MULTIPLE".format( genome, bam_string, gff_filename_gained, output_folder, name_string, color_string, plot_title, )) os.system(cmd) # gained window command plot_title = "{}_ONLY_SE_{}KB_WINDOW".format( name2, str(window // 1000)) cmd = ( "bamPlot_turbo -g {} -b {} -i {} -o {} -n {} -c {} -t {} -r -y UNIFORM -p " "MULTIPLE".format( genome, bam_string, gff_filename_window_gained, output_folder, name_string, color_string, plot_title, )) os.system(cmd) if len(lost_gff) > 0: # lost command plot_title = "{}_ONLY_SE".format(name1) cmd = ( "bamPlot_turbo -g {} -b {} -i {} -o {} -n {} -c {} -t {} -r -y UNIFORM -p " "MULTIPLE".format( genome, bam_string, gff_filename_lost, output_folder, name_string, color_string, plot_title, )) os.system(cmd) # lost command plot_title = "{}_ONLY_SE_{}KB_WINDOW".format( name1, str(window // 1000)) cmd = ( "bamPlot_turbo -g {} -b {} -i {} -o {} -n {} -c {} -t {} -r -y UNIFORM -p " "MULTIPLE".format( genome, bam_string, gff_filename_window_lost, output_folder, name_string, color_string, plot_title, )) os.system(cmd) return
def map_gff_line_to_annot( gff_line, out_folder, n_bins, gene_dict, tx_collection, sense="both", header="" ): """For every line produces a file with all of the rectangles to draw.""" if not header: gff_string = "{}_{}_{}_{}".format( gff_line[0], gff_line[6], gff_line[3], gff_line[4] ) else: gff_string = header diagram_table = [[0, 0, 0, 0]] name_table = [["", 0, 0]] gff_locus = utils.Locus( gff_line[0], int(gff_line[3]), int(gff_line[4]), gff_line[6], gff_line[1], ) scale_factor = n_bins / gff_locus.len() # plotting buffer for diagrams plot_buffer = int(gff_locus.len() / n_bins * 20) overlap_loci = tx_collection.get_overlap(gff_locus, sense="both") gene_list = [locus.id for locus in overlap_loci] if gff_line[6] == "-": ref_point = int(gff_line[4]) else: ref_point = int(gff_line[3]) offset_collection = utils.LocusCollection([], 500) for gene_id in gene_list: gene = gene_dict[gene_id] print(gene.common_name()) if len(gene.common_name()) > 1: name = gene.common_name() else: name = gene_id offset = 4 * len(offset_collection.get_overlap(gene.tx_locus())) offset_collection.append( utils.make_search_locus(gene.tx_locus(), plot_buffer, plot_buffer,) ) # write the name of the gene down if gene.sense() == "+": gene_start = gene.tx_locus().start else: gene_start = gene.tx_locus().end gene_start = abs(gene_start - ref_point) * scale_factor name_table.append([name, gene_start, -2 - offset]) # draw a line across the entire txLocus [start, stop] = [ abs(x - ref_point) * scale_factor for x in gene.tx_locus().coords() ] diagram_table.append([start, -0.01 - offset, stop, 0.01 - offset]) # now draw thin boxes for all tx_exons if gene.tx_exons(): for tx_exon in gene.tx_exons(): [start, stop] = [ abs(x - ref_point) * scale_factor for x in tx_exon.coords() ] diagram_table.append([start, -0.5 - offset, stop, 0.5 - offset]) # now draw fatty boxes for the coding exons if any if gene.cd_exons(): for cd_exon in gene.cd_exons(): [start, stop] = [ abs(x - ref_point) * scale_factor for x in cd_exon.coords() ] diagram_table.append([start, -1 - offset, stop, 1 - offset]) utils.unparse_table( diagram_table, os.path.join(out_folder, "{}_diagramTemp.txt".format(gff_string)), "\t", ) utils.unparse_table( name_table, os.path.join(out_folder, "{}_nameTemp.txt".format(gff_string)), "\t", )
def map_collection( stitched_collection, reference_collection, bam_file_list, mapped_folder, output, ref_name, ): """Makes a table of factor density in a stitched locus. Rank table by number of loci stitched together. """ print("FORMATTING TABLE") loci = list(stitched_collection.get_loci()) locus_table = [[ "REGION_ID", "CHROM", "START", "STOP", "NUM_LOCI", "CONSTITUENT_SIZE" ]] loci_len_list = [] # strip out any that are in chrY for locus in loci: if locus.chr == "chrY": loci.remove(locus) for locus in loci: # numLociList.append(int(stitchLocus.id.split('_')[1])) loci_len_list.append(locus.len()) # numOrder = order(numLociList,decreasing=True) len_order = utils.order(loci_len_list, decreasing=True) ticker = 0 for i in len_order: ticker += 1 if ticker % 1000 == 0: print(ticker) locus = loci[i] # First get the size of the enriched regions within the stitched locus ref_enrich_size = 0 ref_overlapping_loci = reference_collection.get_overlap(locus, "both") for ref_locus in ref_overlapping_loci: ref_enrich_size += ref_locus.len() try: stitch_count = int(locus.id.split("_")[0]) except ValueError: stitch_count = 1 coords = [int(x) for x in locus.coords()] locus_table.append([ locus.id, locus.chr, min(coords), max(coords), stitch_count, ref_enrich_size, ]) print("GETTING MAPPED DATA") print("USING A bam_file LIST:") print(bam_file_list) for bam_file in bam_file_list: bam_file_name = os.path.basename(bam_file) print("GETTING MAPPING DATA FOR {}".format(bam_file)) # assumes standard convention for naming enriched region gffs # opening up the mapped GFF mapped_gff_file = os.path.join( mapped_folder, "{}_{}_MAPPED".format(ref_name, bam_file_name), "matrix.txt") print("OPENING {}".format(mapped_gff_file)) mapped_gff = utils.parse_table(mapped_gff_file, "\t") signal_dict = defaultdict(float) print("MAKING SIGNAL DICT FOR {}".format(bam_file)) mapped_loci = [] for line in mapped_gff[1:]: chrom = line[1].split("(")[0] start = int(line[1].split(":")[-1].split("-")[0]) end = int(line[1].split(":")[-1].split("-")[1]) mapped_loci.append(utils.Locus(chrom, start, end, ".", line[0])) try: signal_dict[line[0]] = float(line[2]) * (abs(end - start)) except ValueError: print("WARNING NO SIGNAL FOR LINE:") print(line) continue mapped_collection = utils.LocusCollection(mapped_loci, 500) locus_table[0].append(bam_file_name) for i in range(1, len(locus_table)): signal = 0.0 line = locus_table[i] line_locus = utils.Locus(line[1], line[2], line[3], ".") overlapping_regions = mapped_collection.get_overlap(line_locus, sense="both") for region in overlapping_regions: signal += signal_dict[region.id] locus_table[i].append(signal) utils.unparse_table(locus_table, output, "\t")
def optimize_stitching(locus_collection, name, out_folder, step_size=500): """ takes a locus collection and starts writing out stitching stats at step sized intervals """ max_stitch = 15000 # set a hard wired match stitching parameter stitch_table = [[ "STEP", "NUM_REGIONS", "TOTAL_CONSTIT", "TOTAL_REGION", "MEAN_CONSTIT", "MEDIAN_CONSTIT", "MEAN_REGION", "MEDIAN_REGION", "MEAN_STITCH_FRACTION", "MEDIAN_STITCH_FRACTION", ]] # first consolidate the collection locus_collection = locus_collection.stitch_collection(stitch_window=0) total_constit = sum([locus.len() for locus in locus_collection.get_loci()]) step = 0 while step <= max_stitch: print("Getting stitch stats for {} (bp)".format(step)) stitch_collection = locus_collection.stitch_collection( stitch_window=step) num_regions = len(stitch_collection) stitch_loci = stitch_collection.get_loci() region_lengths = [locus.len() for locus in stitch_loci] total_region = sum(region_lengths) constit_lengths = [] for locus in stitch_loci: constit_loci = locus_collection.get_overlap(locus) constit_lengths.append(sum([locus.len() for locus in constit_loci])) mean_constit = round(numpy.mean(constit_lengths), 2) median_constit = round(numpy.median(constit_lengths), 2) mean_region = round(numpy.mean(region_lengths), 2) median_region = round(numpy.median(region_lengths), 2) stitch_fractions = [ float(constit_lengths[i]) / float(region_lengths[i]) for i in range(len(region_lengths)) ] mean_stitch_fraction = round(numpy.mean(stitch_fractions), 2) median_stitch_fraction = round(numpy.median(stitch_fractions), 2) new_line = [ step, num_regions, total_constit, total_region, mean_constit, median_constit, mean_region, median_region, mean_stitch_fraction, median_stitch_fraction, ] stitch_table.append(new_line) step += step_size # write the stitch table to disk stitch_param_file = os.path.join(out_folder, "{}_stitch_params.tmp".format(name)) utils.unparse_table(stitch_table, stitch_param_file, "\t") # call the rscript r_cmd = "Rscript {} {} {} {}".format( os.path.join(ROOT_DIR, "scripts", "ROSE2_stitchOpt.R"), stitch_param_file, out_folder + "/", # TODO: fix R script so it does not require '/' name, ) print(r_cmd) # get back the stitch parameter r_output = subprocess.Popen(r_cmd, stdout=subprocess.PIPE, shell=True) r_output_test = r_output.communicate() print(r_output_test) stitch_param = r_output_test[0].decode("utf-8").split("\n")[2] try: stitch_param = int(stitch_param) except ValueError: print("INVALID STITCHING PARAMETER. STITCHING OPTIMIZATION FAILED") sys.exit() # delete? the table # os.system('rm -f %s' % (stitch_param_file)) return stitch_param
def main(): """Main run method for enhancer promoter contribution tool.""" parser = argparse.ArgumentParser() # required flags parser.add_argument( "-b", "--bam", dest="bam", nargs="*", help="Enter a space separated list of .bam files for the main factor", required=True, ) parser.add_argument( "-i", "--input", dest="input", type=str, help="Enter .gff or .bed file of regions to analyze", required=True, ) parser.add_argument( "-g", "--genome", dest="genome", type=str, help=( "specify a genome, HG18,HG19,HG38,MM8,MM9,MM10,RN6 are currently " "supported"), required=True, ) parser.add_argument( "-p", "--chrom-path", dest="chrom_path", type=str, help=("Provide path to a folder with a seperate fasta file for each " "chromosome"), required=True, ) # output flag parser.add_argument( "-o", "--output", dest="output", type=str, help="Enter the output folder.", required=True, ) # additional options flags and optional arguments parser.add_argument( "-a", "--activity", dest="activity", type=str, help=("specify a table where first column represents a list of active " "refseq genes"), required=False, ) parser.add_argument( "-c", "--control", dest="control", nargs="*", help=("Enter a space separated list of .bam files for background. If " "flagged, will perform background subtraction"), required=False, ) parser.add_argument( "-t", "--tss", dest="tss", type=int, help="Define the TSS area +/- the TSS. Default is 1kb", required=False, default=1000, ) parser.add_argument( "-d", "--distal", dest="distal", type=int, help="Enter a window to assign distal enhancer signal. Default is 50kb", required=False, default=50000, ) parser.add_argument( "--other-bams", dest="other", nargs="*", help="enter a space separated list of other bams to map to", required=False, ) parser.add_argument( "--name", dest="name", type=str, help= ("enter a root name for the analysis, otherwise will try to find the " "name from the input file"), required=False, ) parser.add_argument( "--top", dest="top", type=int, help= ("Run the analysis on the top N genes by total signal. Default is 5000" ), required=False, default=5000, ) parser.add_argument( "--tads", dest="tads", type=str, help= ("Include a .bed of tad regions to restrict enhancer/gene association" ), required=False, default=None, ) parser.add_argument( "--mask", dest="mask", default=None, help=( "Mask a set of regions from analysis. Provide a .bed or .gff of " "masking regions"), ) args = parser.parse_args() print(args) # ===================================================================================== # ===============================I. PARSING ARGUMENTS================================== # ===================================================================================== print( "\n\n#======================================\n#===========I. DATA SUMMARY============\n#=" "=====================================\n") # top analysis subset top = args.top # input genome genome = args.genome.upper() print("PERFORMING ANALYSIS ON {} GENOME BUILD".format(genome)) # set of bams bam_file_list = args.bam # bring in the input path input_path = args.input # try to get the input name or use the name argument if args.name: analysis_name = args.name else: analysis_name = os.path.basename(input_path).split(".")[0] print("USING {} AS ANALYSIS NAME".format(analysis_name)) # setting up the output folder parent_folder = utils.format_folder(args.output, True) output_folder = utils.format_folder( os.path.join(parent_folder, analysis_name), True) print("WRITING OUTPUT TO {}".format(output_folder)) if input_path.split(".")[-1] == "bed": # type is bed print("input in bed format, converting to gff") input_gff = utils.bed_to_gff(input_path) else: input_gff = utils.parse_table(input_path, "\t") # the tss window for proximal signal assignment tss_window = int(args.tss) # the distal window for assigning nearby enhancer signal distal_window = int(args.distal) # activity path if args.activity: activity_path = args.activity activity_table = utils.parse_table(activity_path, "\t") ref_col = 0 # try to find the column for refseq id for i in range(len( activity_table[2])): # use an internal row in case of header if str(activity_table[1][i]).count("NM_") or str( activity_table[1][i]).count("NR_"): ref_col = i # now check for header if not str(activity_table[0][i]).count("NM_") and not str( activity_table[0][i]).count("NR_"): print("REMOVING HEADER FROM GENE TABLE:") print(activity_table[0]) activity_table.pop(0) gene_list = [line[ref_col] for line in activity_table ] # this needs to be REFSEQ NM ID print("IDENTIFIED {} ACTIVE GENES".format(len(gene_list))) else: gene_list = [] # check if tads are being invoked if args.tads: print("LOADING TAD LOCATIONS FROM {}".format(args.tads)) tads_path = args.tads else: tads_path = "" print("LOADING ANNOTATION DATA FOR GENOME {}".format(genome)) genome_dir = args.chrom_path # making a chrom_dict that is a list of all chroms with sequence chrom_list = utils.uniquify( [name.split(".")[0] for name in os.listdir(genome_dir) if name]) # important here to define the window start_dict, tss_collection, mouse_convert_dict = load_annot_file( genome, tss_window, gene_list, ) print("FILTERING THE INPUT GFF FOR GOOD CHROMOSOMES") print(chrom_list) filtered_gff = [line for line in input_gff if chrom_list.count(line[0])] print("{} of INITIAL {} REGIONS ARE IN GOOD CHROMOSOMES".format( str(len(filtered_gff)), str(len(input_gff)), )) # ===================================================================================== # ================II. IDENTIFYING TSS PROXIMAL AND DISTAL ELEMENTS===================== # ===================================================================================== print( "\n\n#======================================\n#==II. MAPPING TO TSS/DISTAL REGIONS===\n#=" "=====================================\n") # now we need to split the input region print("SPLITTING THE INPUT GFF USING A WINDOW OF {}".format(tss_window)) split_gff = split_regions(filtered_gff, tss_collection, mask_file=args.mask) print(len(filtered_gff)) print(len(split_gff)) split_gff_path = os.path.join(output_folder, "{}_SPLIT.gff".format(analysis_name)) utils.unparse_table(split_gff, split_gff_path, "\t") print("WRITING TSS SPLIT GFF OUT TO {}".format(split_gff_path)) # now you have to map the bams to the gff print("MAPPING TO THE SPLIT GFF") mapped_folder = utils.format_folder( os.path.join(output_folder, "bam_mapping"), True) signal_table = map_bams(bam_file_list, split_gff_path, analysis_name, mapped_folder) signal_table_path = os.path.join( output_folder, "{}_signal_table.txt".format(analysis_name)) utils.unparse_table(signal_table, signal_table_path, "\t") if args.control: control_bam_file_list = args.control control_signal_table = map_bams( control_bam_file_list, split_gff_path, analysis_name, mapped_folder, ) control_signal_table_path = os.path.join( output_folder, "{}_control_signal_table.txt".format(analysis_name), ) utils.unparse_table(control_signal_table, control_signal_table_path, "\t") # now create the background subtracted summarized average table print("CREATING AN AVERAGE SIGNAL TABLE") average_table = make_average_table( output_folder, analysis_name, use_background=args.control # TODO: fix to True or False ) average_table_path = os.path.join( output_folder, "{}_average_table.txt".format(analysis_name)) utils.unparse_table(average_table, average_table_path, "\t") # now load up all of the cpg and other parameters to make the actual peak table # first check if this has already been done peak_table_path = os.path.join(output_folder, "{}_PEAK_TABLE.txt".format(analysis_name)) if utils.check_output(peak_table_path, 0.1, 0.1): print("PEAK TABLE OUTPUT ALREADY EXISTS") peak_table = utils.parse_table(peak_table_path, "\t") else: peak_table = make_peak_table( param_dict, split_gff_path, average_table_path, start_dict, gene_list, genome_dir, tss_window, distal_window, tads_path, ) utils.unparse_table(peak_table, peak_table_path, "\t") gene_table = make_gene_table(peak_table, analysis_name) gene_table_path = os.path.join(output_folder, "{}_GENE_TABLE.txt".format(analysis_name)) utils.unparse_table(gene_table, gene_table_path, "\t") # if mouse, need to convert genes over if genome.count("MM") == 1: print("CONVERTING MOUSE NAMES TO HUMAN HOMOLOGS FOR GSEA") converted_gene_table_path = os.path.join( output_folder, "{}_GENE_TABLE_CONVERTED.txt".format(analysis_name), ) converted_gene_table = [gene_table[0]] for line in gene_table[1:]: converted_name = mouse_convert_dict[line[0]] if converted_name: converted_gene_table.append([converted_name] + line[1:]) utils.unparse_table(converted_gene_table, converted_gene_table_path, "\t") gene_table_path = converted_gene_table_path gene_table = converted_gene_table # ===================================================================================== # ===================================III. PLOTTING ==================================== # ===================================================================================== print( "\n\n#======================================\n#===III. PLOTTING ENHANCER/PROMOTER===\n#==" "====================================\n") # if there are fewer genes in the gene table than the top genes, only run on all if len(gene_table) < int(top): print( "WARNING: ONLY {} GENES WITH SIGNAL AT EITHER PROMOTERS OR ENHANCERS. NOT ENOUGH TO" "RUN ANALYSIS ON TOP {}".format(str(len(gene_table) - 1), str(top))) top = 0 # now call the R code print("CALLING R PLOTTING SCRIPTS") call_r_waterfall(gene_table_path, output_folder, analysis_name, top)
def tf_edge_delta_out( crc_folder, bam_list, analysis_name, edge_table_path_1, edge_table_path_2, group1_list, group2_list, output="", ): """Calculates changes in group out degree at each predicted motif occurrence (by subpeaks).""" crc_folder = utils.format_folder(crc_folder, True) edge_path = merge_edge_tables( edge_table_path_1, edge_table_path_2, os.path.join(crc_folder, "{}_EDGE_TABLE.txt".format(analysis_name)), ) # make a gff of the edge table edge_table = utils.parse_table(edge_path, "\t") edge_gff = [] for line in edge_table[1:]: gff_line = [ line[2], "{}_{}".format(line[0], line[1]), "", line[3], line[4], "", ".", "", "{}_{}".format(line[0], line[1]), ] edge_gff.append(gff_line) edge_gff_path = os.path.join(crc_folder, "{}_EDGE_TABLE.gff".format(analysis_name)) utils.unparse_table(edge_gff, edge_gff_path, "\t") # direct the output to the crc folder signal_path = os.path.join( crc_folder, "{}_EDGE_TABLE_signal.txt".format(analysis_name)) all_group_list = group1_list + group2_list if not utils.check_output(signal_path, 0, 0): signal_table_list = pipeline_utils.map_regions( bam_list, [edge_gff_path], crc_folder, crc_folder, all_group_list, True, signal_path, extend_reads_to=100, ) print(signal_table_list) else: print("Found previous signal table at {}".format(signal_path)) # now bring in the signal table as a dictionary using the locus line as the id print("making log2 group1 vs group2 signal table at edges") signal_table = utils.parse_table(signal_path, "\t") # figure out columns for group1 and group2 group1_columns = [signal_table[0].index(name) for name in group1_list] group2_columns = [signal_table[0].index(name) for name in group2_list] group1_signal_vector = [] group2_signal_vector = [] for line in signal_table[1:]: group1_signal = numpy.mean( [float(line[col]) for col in group1_columns]) group2_signal = numpy.mean( [float(line[col]) for col in group2_columns]) group1_signal_vector.append(group1_signal) group2_signal_vector.append(group2_signal) group1_median = numpy.median(group1_signal_vector) group2_median = numpy.median(group2_signal_vector) print("group1 median signal") print(group1_median) print("group2 median signal") print(group2_median) # now that we have the median, we can take edges where at least 1 edge is above the median # and both are above zero and generate a new table w/ the fold change signal_filtered_path = signal_path.replace(".txt", "_filtered.txt") if utils.check_output(signal_filtered_path, 0, 0): print("Found filtered signal table for edges at {}".format( signal_filtered_path)) signal_table_filtered = utils.parse_table(signal_filtered_path, "\t") else: signal_table_filtered = [ signal_table[0] + ["GROUP1_MEAN", "GROUP2_MEAN", "GROUP1_vs_GROUP2_LOG2"] ] for line in signal_table[1:]: group1_signal = numpy.mean( [float(line[col]) for col in group1_columns]) group2_signal = numpy.mean( [float(line[col]) for col in group2_columns]) if (group1_signal > group1_median or group2_signal > group2_median ) and min(group1_signal, group2_signal) > 0: delta = numpy.log2(group1_signal / group2_signal) new_line = line + [group1_signal, group2_signal, delta] signal_table_filtered.append(new_line) utils.unparse_table(signal_table_filtered, signal_filtered_path, "\t") # now get a list of all TFs in the system tf_list = utils.uniquify( [line[0].split("_")[0] for line in signal_table_filtered[1:]]) tf_list.sort() print(tf_list) out_degree_table = [[ "TF_NAME", "EDGE_COUNT", "DELTA_MEAN", "DELTA_MEDIAN", "DELTA_STD", "DELTA_SEM", ]] for tf_name in tf_list: print(tf_name) edge_vector = [ float(line[-1]) for line in signal_table_filtered[1:] if line[0].split("_")[0] == tf_name ] edge_count = len(edge_vector) delta_mean = round(numpy.mean(edge_vector), 4) delta_median = round(numpy.median(edge_vector), 4) delta_std = round(numpy.std(edge_vector), 4) delta_sem = round(stats.sem(edge_vector), 4) tf_out_line = [ tf_name, edge_count, delta_mean, delta_median, delta_std, delta_sem, ] out_degree_table.append(tf_out_line) # set final output if not output: output_path = os.path.join( crc_folder, "{}_EDGE_DELTA_OUT.txt".format(analysis_name)) else: output_path = output utils.unparse_table(out_degree_table, output_path, "\t") print(output_path) return output_path
def main(): """Main run call.""" debug = False parser = argparse.ArgumentParser() # required flags parser.add_argument( "-i", "--i", dest="input", required=True, help= ("Enter a comma separated list of .gff or .bed file of binding sites used to make " "enhancers"), ) parser.add_argument( "-r", "--rankby", dest="rankby", required=True, help="Enter a comma separated list of bams to rank by", ) parser.add_argument("-o", "--out", dest="out", required=True, help="Enter an output folder") parser.add_argument( "-g", "--genome", dest="genome", required=True, help="Enter the genome build (MM9,MM8,HG18,HG19)", ) # optional flags parser.add_argument( "-n", "--name", dest="name", required=False, help="Provide a name for the analysis otherwise ROSE will guess", ) parser.add_argument( "-c", "--control", dest="control", required=False, help= ("Enter a comma separated list of control bams. Can either provide a single control " "bam for all rankby bams, or provide a control bam for each individual bam" ), ) parser.add_argument( "-s", "--stitch", dest="stitch", default="", help= ("Enter a max linking distance for stitching. Default will determine optimal stitching" " parameter"), ) parser.add_argument( "-t", "--tss", dest="tss", default=0, help="Enter a distance from TSS to exclude. 0 = no TSS exclusion", ) parser.add_argument( "--mask", dest="mask", required=False, help= "Mask a set of regions from analysis. Provide a .bed or .gff of masking regions", ) # RETRIEVING FLAGS args = parser.parse_args() # making the out folder if it doesn't exist out_folder = utils.format_folder(args.out, True) # figuring out folder schema gff_folder = utils.format_folder(os.path.join(out_folder, "gff"), True) mapped_folder = utils.format_folder(os.path.join(out_folder, "mappedGFF"), True) # GETTING INPUT FILE(s) input_list = [ input_file for input_file in args.input.split(",") if len(input_file) > 1 ] # converting all input files into GFFs and moving into the GFF folder input_gf_list = [] for input_file in input_list: # GETTING INPUT FILE if args.input.split(".")[-1] == "bed": # CONVERTING A BED TO GFF input_gff_name = os.path.basename(args.input)[0:-4] input_gff_file = os.path.join(gff_folder, "{}.gff".format(input_gff_name)) utils.bed_to_gff(args.input, input_gff_file) elif args.input.split(".")[-1] == "gff": # COPY THE INPUT GFF TO THE GFF FOLDER input_gff_file = args.input copyfile( input_gff_file, os.path.join(gff_folder, os.path.basename(input_gff_file)), ) else: print( "WARNING: INPUT FILE DOES NOT END IN .gff or .bed. ASSUMING .gff FILE FORMAT" ) # COPY THE INPUT GFF TO THE GFF FOLDER input_gff_file = args.input copyfile( input_gff_file, os.path.join(gff_folder, os.path.basename(input_gff_file)), ) input_gf_list.append(input_gff_file) # GETTING THE LIST OF bam_fileS TO PROCESS # either same number of bams for rankby and control # or only 1 control #or none! # bamlist should be all rankby bams followed by control bams bam_file_list = [] if args.control: control_bam_list = [ bam for bam in args.control.split(",") if len(bam) > 0 ] rankby_bam_list = [ bam for bam in args.rankby.split(",") if len(bam) > 0 ] if len(control_bam_list) == len(rankby_bam_list): # case where an equal number of backgrounds are given bam_file_list = rankby_bam_list + control_bam_list elif len(control_bam_list) == 1: # case where a universal background is applied bam_file_list = rankby_bam_list + control_bam_list * len( rankby_bam_list) else: print( "ERROR: EITHER PROVIDE A SINGLE CONTROL BAM FOR ALL SAMPLES, OR ONE CONTROL BAM" " FOR EACH SAMPLE") sys.exit() else: bam_file_list = [bam for bam in args.rankby.split(",") if len(bam) > 0] # Stitch parameter if args.stitch == "": stitch_window = "" else: stitch_window = int(args.stitch) # tss args tss_window = int(args.tss) if tss_window != 0: remove_tss = True else: remove_tss = False # GETTING THE GENOME genome = args.genome.upper() print("USING {} AS THE GENOME".format(genome)) # GETTING THE CORRECT ANNOT FILE try: annot_file = rose2_utils.genome_dict[genome] except KeyError: print("ERROR: UNSUPPORTED GENOMES TYPE {}".format(genome)) sys.exit() # FINDING THE ANALYSIS NAME if args.name: input_name = args.name else: input_name = os.path.basename(input_gf_list[0]).split(".")[0] print("USING {} AS THE ANALYSIS NAME".format(input_name)) print("FORMATTING INPUT REGIONS") # MAKING THE RAW INPUT FILE FROM THE INPUT GFFs # use a simpler unique region naming system if len(input_gf_list) == 1: input_gff = utils.parse_table(input_gf_list[0], "\t") else: input_loci = [] for gff_file in input_gf_list: print("\tprocessing {}".format(gff_file)) gff = utils.parse_table(gff_file, "\t") gff_collection = utils.gff_to_locus_collection(gff, 50) input_loci += gff_collection.get_loci() input_collection = utils.LocusCollection(input_loci, 50) input_collection = (input_collection.stitch_collection() ) # stitches to produce unique regions input_gff = utils.locus_collection_to_gff(input_collection) formatted_gff = [] # now number things appropriately for i, line in enumerate(input_gff): # use the coordinates to make a new id input_name_chr_sense_start_stop chrom = line[0] coords = [int(line[3]), int(line[4])] sense = line[6] line_id = "{}_{}".format(input_name, str(i + 1)) # 1 indexing new_line = [ chrom, line_id, line_id, min(coords), max(coords), "", sense, "", line_id, ] formatted_gff.append(new_line) # name of the master input gff file master_gff_file = os.path.join( gff_folder, "{}_{}_ALL_-0_+0.gff".format(genome, input_name)) utils.unparse_table(formatted_gff, master_gff_file, "\t") print("USING {} AS THE INPUT GFF".format(master_gff_file)) # GET CHROMS FOUND IN THE BAMS print("GETTING CHROMS IN bam_fileS") bam_chrom_list = rose2_utils.get_bam_chrom_list(bam_file_list) print("USING THE FOLLOWING CHROMS") print(bam_chrom_list) # LOADING IN THE GFF AND FILTERING BY CHROM print("LOADING AND FILTERING THE GFF") input_gff = rose2_utils.filter_gff(master_gff_file, bam_chrom_list) # LOADING IN THE BOUND REGION REFERENCE COLLECTION print("LOADING IN GFF REGIONS") reference_collection = utils.gff_to_locus_collection(input_gff) print("CHECKING REFERENCE COLLECTION:") rose2_utils.check_ref_collection(reference_collection) # MASKING REFERENCE COLLECTION # see if there's a mask if args.mask: mask_file = args.mask # if it's a bed file if mask_file.split(".")[-1].upper() == "BED": mask_gff = utils.bedToGFF(mask_file) elif mask_file.split(".")[-1].upper() == "GFF": mask_gff = utils.parse_table(mask_file, "\t") else: print("MASK MUST BE A .gff or .bed FILE") sys.exit() mask_collection = utils.gff_to_locus_collection(mask_gff) # now mask the reference loci reference_loci = reference_collection.get_loci() filtered_loci = [ locus for locus in reference_loci if len(mask_collection.get_overlap(locus, "both")) == 0 ] print("FILTERED OUT {} LOCI THAT WERE MASKED IN {}".format( len(reference_loci) - len(filtered_loci), mask_file)) reference_collection = utils.LocusCollection(filtered_loci, 50) # NOW STITCH REGIONS print("STITCHING REGIONS TOGETHER") stitched_collection, debug_output, stitch_window = rose2_utils.region_stitching( reference_collection, input_name, out_folder, stitch_window, tss_window, annot_file, remove_tss, ) # NOW MAKE A STITCHED COLLECTION GFF print("MAKING GFF FROM STITCHED COLLECTION") stitched_gff = utils.locus_collection_to_gff(stitched_collection) print(stitch_window) print(type(stitch_window)) if not remove_tss: stitched_gff_file = os.path.join( gff_folder, "{}_{}KB_STITCHED.gff".format(input_name, str(stitch_window // 1000)), ) stitched_gff_name = "{}_{}KB_STITCHED".format( input_name, str(stitch_window // 1000)) debug_out_file = os.path.join( gff_folder, "{}_{}KB_STITCHED.debug".format(input_name, str(stitch_window // 1000)), ) else: stitched_gff_file = os.path.join( gff_folder, "{}_{}KB_STITCHED_TSS_DISTAL.gff".format( input_name, str(stitch_window // 1000)), ) stitched_gff_name = "{}_{}KB_STITCHED_TSS_DISTAL".format( input_name, str(stitch_window // 1000)) debug_out_file = os.path.join( gff_folder, "{}_{}KB_STITCHED_TSS_DISTAL.debug".format( input_name, str(stitch_window // 1000)), ) # WRITING DEBUG OUTPUT TO DISK if debug: print("WRITING DEBUG OUTPUT TO DISK AS {}".format(debug_out_file)) utils.unparse_table(debug_output, debug_out_file, "\t") # WRITE THE GFF TO DISK print("WRITING STITCHED GFF TO DISK AS {}".format(stitched_gff_file)) utils.unparse_table(stitched_gff, stitched_gff_file, "\t") # SETTING UP THE OVERALL OUTPUT FILE output_file1 = os.path.join( out_folder, "{}_ENHANCER_REGION_MAP.txt".format(stitched_gff_name)) print("OUTPUT WILL BE WRITTEN TO {}".format(output_file1)) # MAPPING TO THE NON STITCHED (ORIGINAL GFF) # MAPPING TO THE STITCHED GFF bam_file_list_unique = list(bam_file_list) bam_file_list_unique = utils.uniquify(bam_file_list_unique) # prevent redundant mapping print("MAPPING TO THE FOLLOWING BAMS:") print(bam_file_list_unique) for bam_file in bam_file_list_unique: bam_file_name = os.path.basename(bam_file) # MAPPING TO THE STITCHED GFF mapped_out1_folder = os.path.join( mapped_folder, "{}_{}_MAPPED".format(stitched_gff_name, bam_file_name)) mapped_out1_file = os.path.join( mapped_folder, "{}_{}_MAPPED".format(stitched_gff_name, bam_file_name), "matrix.txt", ) if utils.check_output(mapped_out1_file, 0.2, 0.2): print("FOUND {} MAPPING DATA FOR BAM: {}".format( stitched_gff_file, mapped_out1_file)) else: cmd1 = "bamliquidator_batch --sense . -e 200 --match_bamToGFF -r {} -o {} {}".format( stitched_gff_file, mapped_out1_folder, bam_file, ) print(cmd1) os.system(cmd1) if utils.check_output(mapped_out1_file, 0.2, 5): print("SUCCESSFULLY MAPPED TO {} FROM BAM: {}".format( stitched_gff_file, bam_file_name)) else: print("ERROR: FAILED TO MAP {} FROM BAM: {}".format( stitched_gff_file, bam_file_name)) sys.exit() print("BAM MAPPING COMPLETED NOW MAPPING DATA TO REGIONS") # CALCULATE DENSITY BY REGION # NEED TO FIX THIS FUNCTION TO ACCOUNT FOR DIFFERENT OUTPUTS OF LIQUIDATOR rose2_utils.map_collection( stitched_collection, reference_collection, bam_file_list, mapped_folder, output_file1, ref_name=stitched_gff_name, ) print("FINDING AVERAGE SIGNAL AMONGST BAMS") meta_output_file = collapse_region_map(output_file1, input_name + "_MERGED_SIGNAL", control_bams=args.control) # now try the merging print("CALLING AND PLOTTING SUPER-ENHANCERS") control_name = "NONE" cmd = "Rscript {} {} {} {} {}".format( os.path.join(ROOT_DIR, "scripts", "ROSE2_callSuper.R"), out_folder + "/", # TODO: fix R script so it does not require '/' meta_output_file, input_name, control_name, ) print(cmd) os.system(cmd) # calling the gene mapper print("CALLING GENE MAPPING") super_table_file = "{}_SuperEnhancers.table.txt".format(input_name) # for now don't use ranking bam to call top genes cmd = "ROSE2_geneMapper -g {} -i {} -f".format( genome, os.path.join(out_folder, super_table_file)) print(cmd) os.system(cmd) stretch_table_file = "{}_StretchEnhancers.table.txt".format(input_name) cmd = "ROSE2_geneMapper -g {} -i {} -f".format( genome, os.path.join(out_folder, stretch_table_file)) print(cmd) os.system(cmd) superstretch_table_file = "{}_SuperStretchEnhancers.table.txt".format( input_name) cmd = "ROSE2_geneMapper.py -g {} -i {} -f".format(genome, out_folder, superstretch_table_file) os.system(cmd)
def main(): """Main run call.""" debug = False parser = argparse.ArgumentParser() # required flags parser.add_argument( "-i", "--i", dest="input", required=True, help="Enter a .gff or .bed file of binding sites used to make enhancers", ) parser.add_argument( "-r", "--rankby", dest="rankby", required=True, help="bam_file to rank enhancer by", ) parser.add_argument( "-o", "--out", dest="out", required=True, help="Enter an output folder" ) parser.add_argument( "-g", "--genome", dest="genome", required=True, help="Enter the genome build (MM9,MM8,HG18,HG19)", ) # optional flags parser.add_argument( "-b", "--bams", dest="bams", required=False, help="Enter a comma separated list of additional bam files to map to", ) parser.add_argument( "-c", "--control", dest="control", required=False, help="bam_file to rank enhancer by", ) parser.add_argument( "-s", "--stitch", dest="stitch", default="", help=( "Enter a max linking distance for stitching. Default will determine optimal stitching" " parameter" ), ) parser.add_argument( "-t", "--tss", dest="tss", default=0, help="Enter a distance from TSS to exclude. 0 = no TSS exclusion", ) parser.add_argument( "--mask", dest="mask", required=False, help="Mask a set of regions from analysis. Provide a .bed or .gff of masking regions", ) # RETRIEVING FLAGS args = parser.parse_args() # making the out folder if it doesn't exist out_folder = utils.format_folder(args.out, True) # figuring out folder schema gff_folder = utils.format_folder(os.path.join(out_folder, "gff"), True) mapped_folder = utils.format_folder(os.path.join(out_folder, "mapped_gff"), True) # GETTING INPUT FILE if args.input.split(".")[-1] == "bed": # CONVERTING A BED TO GFF input_gff_name = args.input.split("/")[-1][0:-4] input_gff_file = os.path.join(gff_folder, "{}.gff".format(input_gff_name)) utils.bed_to_gff(args.input, input_gff_file) elif args.input.split(".")[-1] == "gff": # COPY THE INPUT GFF TO THE GFF FOLDER input_gff_file = args.input copyfile( input_gff_file, os.path.join(gff_folder, os.path.basename(input_gff_file)) ) else: print( "WARNING: INPUT FILE DOES NOT END IN .gff or .bed. ASSUMING .gff FILE FORMAT" ) # COPY THE INPUT GFF TO THE GFF FOLDER input_gff_file = args.input copyfile( input_gff_file, os.path.join(gff_folder, os.path.basename(input_gff_file)) ) # GETTING THE LIST OF bam_fileS TO PROCESS if args.control: bam_file_list = [args.rankby, args.control] else: bam_file_list = [args.rankby] if args.bams: bam_file_list += args.bams.split(",") # bam_file_list = utils.uniquify(bam_file_list) # makes sad when you have the same control # bam over and over again # optional args # Stitch parameter if args.stitch == "": stitch_window = "" else: stitch_window = int(args.stitch) # tss args tss_window = int(args.tss) if tss_window != 0: remove_tss = True else: remove_tss = False # GETTING THE BOUND REGION FILE USED TO DEFINE ENHANCERS print("USING {} AS THE INPUT GFF".format(input_gff_file)) input_name = os.path.basename(input_gff_file).split(".")[0] # GETTING THE GENOME genome = args.genome print("USING {} AS THE GENOME".format(genome)) annot_file = rose2_utils.genome_dict[genome.upper()] # GET CHROMS FOUND IN THE BAMS print("GETTING CHROMS IN bam_fileS") bam_chrom_list = rose2_utils.get_bam_chrom_list(bam_file_list) print("USING THE FOLLOWING CHROMS") print(bam_chrom_list) # LOADING IN THE GFF AND FILTERING BY CHROM print("LOADING AND FILTERING THE GFF") input_gff = rose2_utils.filter_gff(input_gff_file, bam_chrom_list) # LOADING IN THE BOUND REGION REFERENCE COLLECTION print("LOADING IN GFF REGIONS") reference_collection = utils.gff_to_locus_collection(input_gff) print("STARTING WITH {} INPUT REGIONS".format(len(reference_collection))) print("CHECKING REFERENCE COLLECTION:") rose2_utils.check_ref_collection(reference_collection) # MASKING REFERENCE COLLECTION # see if there's a mask if args.mask: mask_file = args.mask print("USING MASK FILE {}".format(mask_file)) # if it's a bed file if mask_file.split(".")[-1].upper() == "BED": mask_gff = utils.bed_to_gff(mask_file) elif mask_file.split(".")[-1].upper() == "GFF": mask_gff = utils.parse_table(mask_file, "\t") else: print("MASK MUST BE A .gff or .bed FILE") mask_collection = utils.gff_to_locus_collection(mask_gff) print("LOADING {} MASK REGIONS".format(str(len(mask_collection)))) # now mask the reference loci reference_loci = reference_collection.get_loci() filtered_loci = [ locus for locus in reference_loci if len(mask_collection.get_overlap(locus, "both")) == 0 ] print( "FILTERED OUT {} LOCI THAT WERE MASKED IN {}".format( str(len(reference_loci) - len(filtered_loci)), mask_file ) ) reference_collection = utils.LocusCollection(filtered_loci, 50) # NOW STITCH REGIONS print("STITCHING REGIONS TOGETHER") stitched_collection, debug_output, stitch_window = rose2_utils.region_stitching( reference_collection, input_name, out_folder, stitch_window, tss_window, annot_file, remove_tss, ) # NOW MAKE A STITCHED COLLECTION GFF print("MAKING GFF FROM STITCHED COLLECTION") stitched_gff = utils.locus_collection_to_gff(stitched_collection) # making sure start/stop ordering are correct for i in range(len(stitched_gff)): line = stitched_gff[i] start = int(line[3]) stop = int(line[4]) if start > stop: line[3] = stop line[4] = start print(stitch_window) print(type(stitch_window)) if not remove_tss: stitched_gff_file = os.path.join( gff_folder, "{}_{}KB_STITCHED.gff".format(input_name, str(stitch_window // 1000)), ) stitched_gff_name = "{}_{}KB_STITCHED".format( input_name, str(stitch_window // 1000) ) debug_out_file = os.path.join( gff_folder, "{}_{}KB_STITCHED.debug".format(input_name, str(stitch_window // 1000)), ) else: stitched_gff_file = os.path.join( gff_folder, "{}_{}KB_STITCHED_TSS_DISTAL.gff".format( input_name, str(stitch_window // 1000) ), ) stitched_gff_name = "{}_{}KB_STITCHED_TSS_DISTAL".format( input_name, str(stitch_window // 1000) ) debug_out_file = os.path.join( gff_folder, "{}_{}KB_STITCHED_TSS_DISTAL.debug".format( input_name, str(stitch_window // 1000) ), ) # WRITING DEBUG OUTPUT TO DISK if debug: print("WRITING DEBUG OUTPUT TO DISK AS {}".format(debug_out_file)) utils.unparse_table(debug_output, debug_out_file, "\t") # WRITE THE GFF TO DISK print("WRITING STITCHED GFF TO DISK AS {}".format(stitched_gff_file)) utils.unparse_table(stitched_gff, stitched_gff_file, "\t") # SETTING UP THE OVERALL OUTPUT FILE output_file1 = os.path.join( out_folder, "{}_ENHANCER_REGION_MAP.txt".format(stitched_gff_name) ) print("OUTPUT WILL BE WRITTEN TO {}".format(output_file1)) # MAPPING TO THE NON STITCHED (ORIGINAL GFF) # MAPPING TO THE STITCHED GFF bam_file_list_unique = list(bam_file_list) bam_file_list_unique = utils.uniquify(bam_file_list_unique) # prevent redundant mapping print("MAPPING TO THE FOLLOWING BAMS:") print(bam_file_list_unique) for bam_file in bam_file_list_unique: bam_file_name = os.path.basename(bam_file) # MAPPING TO THE STITCHED GFF mapped_out1_folder = os.path.join( mapped_folder, "{}_{}_MAPPED".format(stitched_gff_name, bam_file_name) ) mapped_out1_file = os.path.join( mapped_folder, "{}_{}_MAPPED".format(stitched_gff_name, bam_file_name), "matrix.txt", ) if utils.check_output(mapped_out1_file, 0.2, 0.2): print( "FOUND {} MAPPING DATA FOR BAM: {}".format( stitched_gff_file, mapped_out1_file ) ) else: cmd1 = "bamliquidator_batch --sense . -e 200 --match_bamToGFF -r {} -o {} {}".format( stitched_gff_file, mapped_out1_folder, bam_file, ) print(cmd1) os.system(cmd1) if utils.check_output(mapped_out1_file, 0.2, 5): print( "SUCCESSFULLY MAPPED TO {} FROM BAM: {}".format( stitched_gff_file, bam_file_name ) ) else: print( "ERROR: FAILED TO MAP {} FROM BAM: {}".format( stitched_gff_file, bam_file_name ) ) sys.exit() print("BAM MAPPING COMPLETED NOW MAPPING DATA TO REGIONS") # CALCULATE DENSITY BY REGION # NEED TO FIX THIS FUNCTION TO ACCOUNT FOR DIFFERENT OUTPUTS OF LIQUIDATOR rose2_utils.map_collection( stitched_collection, reference_collection, bam_file_list, mapped_folder, output_file1, ref_name=stitched_gff_name, ) print("CALLING AND PLOTTING SUPER-ENHANCERS") if args.control: control_name = os.path.basename(args.control) else: control_name = "NONE" cmd = "Rscript {} {} {} {} {}".format( os.path.join(ROOT_DIR, "scripts", "ROSE2_callSuper.R"), out_folder + "/", # TODO: fix R script so it does not require '/' output_file1, input_name, control_name, ) print(cmd) os.system(cmd) # calling the gene mapper time.sleep(20) super_table_file = "{}_SuperEnhancers.table.txt".format(input_name) if args.control: cmd = "ROSE2_geneMapper -g {} -r {} -c {} -i {}".format( genome, args.rankby, args.control, os.path.join(out_folder, super_table_file), ) else: cmd = "ROSE2_geneMapper -g {} -r {} -i {}".format( genome, args.rankby, os.path.join(out_folder, super_table_file) ) os.system(cmd) stretch_table_file = "{}_StretchEnhancers.table.txt".format(input_name) if args.control: cmd = "ROSE2_geneMapper -g {} -r {} -c {} -i {}".format( genome, args.rankby, args.control, os.path.join(out_folder, stretch_table_file), ) else: cmd = "ROSE2_geneMapper -g {} -r {} -i {}".format( genome, args.rankby, os.path.join(out_folder, stretch_table_file) ) os.system(cmd) superstretch_table_file = "{}_SuperStretchEnhancers.table.txt".format(input_name) if args.control: cmd = "ROSE2_geneMapper -g {} -r {} -c {} -i {}".format( genome, args.rankby, args.control, os.path.join(out_folder, superstretch_table_file), ) else: cmd = "ROSE2_geneMapper -g {} -r {} -i {}".format( genome, args.rankby, os.path.join(out_folder, superstretch_table_file) ) os.system(cmd)
def make_bam_plot_tables( gff, genome, bam_file_list, color_list, n_bins, sense, extension, rpm, out_folder, names, title, bed_collection, scale=None, ): """Makes a plot table for each line of the gff mapped against all the bams in the bamList.""" # load in the gff if isinstance(gff, str): gff = utils.parse_table(gff, "\t") # load in the annotation print("loading in annotation for {}".format(genome)) gene_dict, tx_collection = load_annot_file(genome) # make an MMR dict so MMRs are only computed once print("Getting information about read depth in bams") mmr_dict = {} if scale: print("Applying scaling factors") scale_list = [float(x) for x in scale] else: scale_list = [1] * len(bam_file_list) # now iterate through the bam files for i, bam_file in enumerate(bam_file_list): # millionMappedReads idx_cmd = "samtools idxstats {}".format(bam_file) idx_pipe = subprocess.Popen( idx_cmd, stdin=subprocess.PIPE, stderr=subprocess.PIPE, stdout=subprocess.PIPE, shell=True, ) # TODO: this does not produce an error if samtools are not installed idx_stats = idx_pipe.communicate() idx_stats = idx_stats[0].decode("utf-8").split("\n") idx_stats = [line.split("\t") for line in idx_stats] raw_count = sum([int(line[2]) for line in idx_stats[:-1]]) # implement scaling read_scale_factor = scale_list[i] if rpm: mmr = round(raw_count / 1000000 / read_scale_factor, 4) else: mmr = round(1 / read_scale_factor, 4) mmr_dict[bam_file] = mmr ticker = 1 # go line by line in the gff summary_table = [ [ "DIAGRAM_TABLE", "NAME_TABLE", "BED_DIAGRAM_TABLE", "BED_NAME_TABLE", "PLOT_TABLE", "CHROM", "ID", "SENSE", "START", "END", ] ] for gff_line in gff: gff_string = "line_{}_{}_{}_{}_{}_{}".format( ticker, gff_line[0], gff_line[1], gff_line[6], gff_line[3], gff_line[4], ) ticker += 1 print("writing the gene diagram table for region {}".format(gff_line[1])) map_gff_line_to_annot( gff_line, out_folder, n_bins, gene_dict, tx_collection, sense="both", header=gff_string, ) map_gff_line_to_bed( gff_line, out_folder, n_bins, bed_collection, header=gff_string, ) out_table = [] out_table.append( ["BAM", "GENE_ID", "NAME", "LOCUSLINE", "COLOR1", "COLOR2", "COLOR3"] + ["bin_" + str(n) for n in range(1, int(n_bins) + 1, 1)] ) for i, bam_file in enumerate(bam_file_list): name = names[i] color = color_list[i] print( "getting data for location {} in dataset {}".format( gff_line[1], bam_file ) ) mmr = mmr_dict[bam_file] new_line = map_bam_to_gff_line( bam_file, mmr, name, gff_line, color, n_bins, sense, extension, ) out_table.append(new_line) # get the gene name if gff_line[1] in gene_dict: gene_name = gene_dict[gff_line[1]].common_name() else: gene_name = gff_line[1] utils.unparse_table( out_table, os.path.join(out_folder, "{}_plotTemp.txt".format(gff_string)), "\t", ) diagram_table = os.path.join( out_folder, "{}_diagramTemp.txt".format(gff_string) ) plot_table = os.path.join(out_folder, "{}_plotTemp.txt".format(gff_string)) name_table = os.path.join(out_folder, "{}_nameTemp.txt".format(gff_string)) bed_name_table = os.path.join( out_folder, "{}_bedNameTemp.txt".format(gff_string) ) bed_diagram_table = os.path.join( out_folder, "{}_bedDiagramTemp.txt".format(gff_string) ) summary_table.append( [ diagram_table, name_table, bed_diagram_table, bed_name_table, plot_table, gff_line[0], gene_name, gff_line[6], gff_line[3], gff_line[4], ] ) summary_table_file_name = os.path.join(out_folder, "{}_summary.txt".format(title)) utils.unparse_table(summary_table, summary_table_file_name, "\t") return summary_table_file_name