def main(args, outs): '''Find cut sites on a per chromosome basis and write out a bedgraph''' if args.fragments is None: outs.count_dict = None outs.cut_sites = None return ctg_mgr = ReferenceManager(args.reference_path) contig_len = ctg_mgr.get_contig_lengths() chrom_len = contig_len[args.contig] half_window = WINDOW_SIZE // 2 Cuts = np.zeros(chrom_len, dtype='int32') # find windowed cut sites for _, start, stop, _, _ in parsed_fragments_from_contig(contig=args.contig, filename=args.fragments, index=args.fragments_index): Cuts[max(0, start - half_window): min(start + half_window + 1, chrom_len)] += 1 Cuts[max(0, stop - half_window): min(stop + half_window + 1, chrom_len)] += 1 # get count dict count_dict = Counter(v for v in Cuts if v > 0) with open(outs.count_dict, 'w') as count_dict_out: pickle.dump(count_dict, count_dict_out) # write bedgraph of * windowed cutsites * if len(count_dict): write_chrom_bedgraph(args.contig, chrom_len, Cuts, outs.cut_sites) else: outs.cut_sites = None
def split(args): if args.fragments is None: return {'chunks': [], 'join': {}} ctg_mgr = ReferenceManager(args.reference_path) all_contigs = ctg_mgr.primary_contigs(allow_sex_chromosomes=True) contig_len = ctg_mgr.get_contig_lengths() BYTES_PER_INT32_WITH_SAFETY = 5 chunks = [] for contig in all_contigs: chunks.append({'contig': contig, '__mem_gb': int(np.ceil(BYTES_PER_INT32_WITH_SAFETY * contig_len[contig] / 1024 / 1024 / 1024))}) return {'chunks': chunks, 'join': {'__mem_gb': 5}}
def split(args): """split into a chunk for each library in aggr csv, and define a unique gem group""" aggr_df = pd.read_csv(args.aggr_csv, sep=',') nchunks = len(aggr_df) ctg_mgr = ReferenceManager(args.reference_path) max_contig_len = max(ctg_mgr.get_contig_lengths().values()) BYTES_PER_INT32_WITH_SAFETY = 5 mem_gb = 2 * int( np.ceil( BYTES_PER_INT32_WITH_SAFETY * max_contig_len / 1024 / 1024 / 1024)) return { 'chunks': [{ 'n': group, '__mem_gb': mem_gb, '__vmem_gb': mem_gb + 6 } for group in range(nchunks)], 'join': { '__mem_gb': 12 } }
def main(args, outs): ref = ReferenceManager(args.reference_path) contig_len = ref.get_contig_lengths()[args.contig] with open(args.peaks, "r") as infile: peak_regions = get_target_regions(infile) fragment_counts = Counter() targeted_counts = Counter() cumulative_fragment_length = {padding: Counter() for padding in PADDING_VALUES} covered_bases = {padding: {} for padding in PADDING_VALUES} for contig, start, stop, barcode, _ in parsed_fragments_from_contig( args.contig, args.fragments, args.fragments_index ): fragment_counts[barcode] += 1 if fragment_overlaps_target(contig, start, stop, peak_regions): targeted_counts[barcode] += 1 for padding in PADDING_VALUES: adj_start = max(0, start - padding) adj_stop = min(contig_len, stop + padding) cumulative_fragment_length[padding][barcode] += adj_stop - adj_start if barcode not in covered_bases[padding]: # Total # of covered bases, current start/stop of active region covered_bases[padding][barcode] = 0, None, None current_covered, active_start, active_stop = covered_bases[padding][barcode] if active_start is None: active_start = adj_start active_stop = adj_stop else: if adj_start < active_stop: active_stop = max(adj_stop, active_stop) else: current_covered += active_stop - active_start active_start = adj_start active_stop = adj_stop covered_bases[padding][barcode] = current_covered, active_start, active_stop final_covered = {padding: Counter() for padding in PADDING_VALUES} for padding in PADDING_VALUES: for barcode in covered_bases[padding]: current_covered, active_start, active_stop = covered_bases[padding][barcode] if active_start is None: final_covered[padding][barcode] = current_covered else: final_covered[padding][barcode] = current_covered + active_stop - active_start with open(outs.fragment_counts, "w") as outfile: pickle.dump(fragment_counts, outfile) with open(outs.targeted_counts, "w") as outfile: pickle.dump(targeted_counts, outfile) with open(outs.fragment_lengths, "w") as outfile: pickle.dump(cumulative_fragment_length, outfile) with open(outs.covered_bases, "w") as outfile: pickle.dump(final_covered, outfile) outs.peak_coverage = 0 if args.contig in peak_regions: contig_len = ref.contig_lengths[args.contig] outs.peak_coverage = count_covered_bases( peak_regions[args.contig].starts, peak_regions[args.contig].ends, contig_len, padding=DISTANCE, )
def join(args, outs, chunk_defs, chunk_outs): # Sample ID / pipestance name check_sample_id(args.sample_id) # force_cells check_force_cells(args.force_cells, ulimit=10000000) # allow arbitrarily large limit for reanalyzer # # Reference # ref directory structure and timestamps ok, msg = check_refdata(args.reference_path, max_contigs=None) if ok: martian.log_info(msg) else: martian.exit(msg) # formatting check_reference_format(args.reference_path) contig_manager = ReferenceManager(args.reference_path) # peaks format check and nonoverlapping if args.peaks is None: martian.exit("peaks file not provided") exists_and_readable(args.peaks, "peaks") bed_format_checker(args.peaks, contig_manager.fasta_index) contain_three_columns(args.peaks) if is_overlapping(args.peaks): martian.exit("{} contains overlapping peak regions".format(args.peaks)) # check parameters files if args.parameters is not None: if not os.path.exists(args.parameters): martian.exit("{} does not exist".format(args.parameters)) # fragments checks whitelist_barcodes = load_barcode_whitelist(args.barcode_whitelist) species_list = contig_manager.list_species() observed_gem_groups = set() observed_species = set() if args.fragments is None: martian.exit("fragments file not provided") exists_and_readable(args.fragments, "fragments") contig_lens = contig_manager.get_contig_lengths() # check bounds and matching contigs in reference and species for chrom, start, stop, bc, _ in open_fragment_file(args.fragments): spec = chrom.split("_") observed_species.add(spec[0] if spec[0] != chrom else "") barcode, gem_group = bc.split("-") observed_gem_groups.add(gem_group) if args.check_executables: # run this only non-locally if barcode not in whitelist_barcodes: martian.exit("{} is not a valid whitelist barcode".format(barcode)) if chrom not in contig_lens: martian.exit("contig {} not present in reference".format(chrom)) if stop > contig_lens[chrom]: martian.exit("fragment {}:{}-{} boundaries exceed contig size ({} bp)".format(chrom, start, stop, contig_lens[chrom])) # ensure fragments are on the correct reference for species in observed_species: if species not in species_list: martian.exit("{} contains fragments mapped to species not recognized in the reference".format(args.fragments)) if len(observed_gem_groups) > 1: martian.log_info("multiple gem groups present in {}, likely generated in a previous aggregation run".format(args.fragments)) # fragments index is synced with fragments if args.fragments_index is None: martian.exit("fragments index file not provided") if not os.path.exists(args.fragments_index): martian.exit("{} does not exist".format(args.fragments_index)) try: all_contigs = contig_manager.primary_contigs(allow_sex_chromosomes=True) for contig in all_contigs: en = 0 for chrom, start, end, bc, dups in parsed_fragments_from_contig(contig, args.fragments, index=args.fragments_index): if en >= FRAGMENTS_SCAN_SIZE: break en += 1 except: martian.exit("fragments index is not in sync with the fragments file") # aggr csv checks if args.aggregation_csv is not None: check_aggr_csv(args.aggregation_csv, args.reference_path, cursory=True) # cell barcode checks if args.cell_barcodes is not None: if not os.path.exists(args.cell_barcodes): martian.exit("{} does not exist".format(args.cell_barcodes)) check_singlecell_format(args.cell_barcodes, species_list, whitelist_barcodes) # Open file handles limit if args.check_executables: check_filehandle_limit() martian.log_info(tk_preflight.record_package_versions())
def main(args, outs): """Compute the depth and signal per library""" # read lib_id = args.n + 1 aggr_df = pd.read_csv(args.aggr_csv, sep=',') library_info = {lib_id: {}} for label in aggr_df.columns.values.tolist(): library_info[lib_id][label] = str(aggr_df.iloc[args.n][label]) # if no normalization, don't waste compute if args.normalization is None: with open(outs.library_info, 'w') as f: pickle.dump(library_info, f) return # set ref properties ctg_mgr = ReferenceManager(args.reference_path) contig_lens = ctg_mgr.get_contig_lengths() max_contig_len = max(contig_lens.values()) curr_chrom = None count_dict = Counter() chrom_len = 1 half_window = WINDOW_SIZE // 2 # traverse fragments file and count stats fragments_f = aggr_df.iloc[args.n]['fragments'] Cuts = None special_normalization = (args.normalization in ["signal_mean", "signal_noise_threshold"]) if special_normalization: Cuts = np.zeros(max_contig_len, dtype='int32') for chrom, start, stop, bc, dups in open_fragment_file( filename=fragments_f): if chrom != curr_chrom: curr_chrom = chrom if chrom not in contig_lens: martian.exit( "fragment {}:{}-{} in {} is mapped to a contig not in the reference" .format(chrom, start, stop, fragments_f)) if special_normalization: count_dict += Counter( Cuts[i] for i in xrange(chrom_len) if Cuts[i] > 0) # only traverse chrom len Cuts[:] = 0 # reset and reuse chrom_len = contig_lens[chrom] if special_normalization: Cuts[max(0, start - half_window):min(start + half_window + 1, chrom_len)] += 1 Cuts[max(0, stop - half_window):min(stop + half_window + 1, chrom_len)] += 1 if special_normalization: count_dict += Counter(Cuts[i] for i in xrange(chrom_len) if Cuts[i] > 0) # only traverse chrom len scdf = pd.read_csv(library_info[lib_id]['cells'], sep=',') cell_mask = np.full(len(scdf), False) for species in ctg_mgr.list_species(): cell_mask |= scdf['is_{}_cell_barcode'.format(species)] == 1 library_info[lib_id]['total_fragments_per_cell'] = np.median( scdf[cell_mask]['total'] if 'total' in scdf[cell_mask].columns else scdf[cell_mask]['passed_filters'] + scdf[cell_mask]['duplicate']) library_info[lib_id]['unique_fragments_per_cell'] = np.median( scdf[cell_mask]['passed_filters']) # do peak calling fit on the count dict and get signal fit if args.normalization in ["signal_mean", "signal_noise_threshold"]: threshold, params = estimate_final_threshold(count_dict, PEAK_ODDS_RATIO) library_info[lib_id]['original_threshold'] = threshold library_info[lib_id]['signal_mean'] = 1 / params.p_signal # dump library info with open(outs.library_info, 'w') as f: pickle.dump(library_info, f)
def check_aggr_csv(aggr_csv, reference_path, cursory=False): """Check aggr csv has correct columns, then progressively stronger checks on duplicates and formating of files. These stronger checks are enabled by default, unless you want to test the basic minimum, for example in reanalyzer""" contig_manager = ReferenceManager(reference_path) # aggr_csv checks exists_and_readable(aggr_csv, "aggr_csv") if cursory: nlibs, library_info, msg = parse_aggr_csv(aggr_csv, whitelist=["library_id"], blacklist=None) else: nlibs, library_info, msg = parse_aggr_csv(aggr_csv) if msg is not None: martian.exit(msg) # At least one library should be there if nlibs == 0: martian.exit("aggregation csv does not include any library. Provide at least two libraries.") if cursory: return # Enable aggr(count1) to run if nlibs == 1: martian.log_info("Aggregator should be run on more than one library") # avoid aggr of duplicate files (assessed by filename). species_list = contig_manager.list_species() for aggr_key in library_info[1]: # at least one library is present files = {} for lib_id in library_info: fname = library_info[lib_id][aggr_key] if fname in files: martian.exit("File {} already specified for a different library under {}".format(fname, aggr_key)) # singlecell.csv should contain 'barcode' and 'is_{}_cell_barcode' columns with the correct type if aggr_key == "cells": check_singlecell_format(fname, species_list, allow_multi_gem_groups=False) # peaks.bed need to be formatted correctly with right contigs if provided in aggr.csv # also check if peaks are non overlapping if aggr_key == "peaks": exists_and_readable(fname, "peaks") bed_format_checker(fname, contig_manager.fasta_index) contain_three_columns(fname) if is_overlapping(fname): martian.exit("{} contains overlapping peak regions".format(fname)) # checks on fragments contig_lens = contig_manager.get_contig_lengths() if aggr_key == "fragments": observed_gem_groups = set() observed_species = set() exists_and_readable(fname, "fragments") en = 0 for chrom, start, stop, bc, _ in open_fragment_file(fname): if en >= FRAGMENTS_SCAN_SIZE: break spec = chrom.split("_") observed_species.add(spec[0] if spec[0] != chrom else "") observed_gem_groups.add(bc.split("-")[1]) if chrom not in contig_lens: martian.exit("fragment {}:{}-{} in {} is mapped to a contig not in the reference".format(chrom, start, stop, fname)) if stop > contig_lens[chrom]: martian.exit("fragment {}:{}-{} boundaries exceed contig size ({} bp)".format(chrom, start, stop, contig_lens[chrom])) en += 1 for species in observed_species: if species not in species_list: martian.exit("{} contains fragments mapped to species not recognized in the reference".format(fname)) if len(observed_gem_groups) > 1: martian.exit("multiple gem groups present in {}, likely generated in a previous aggregation run".format(fname))