def readDescriptions(options): '''read descriptions from tab separated file.''' description_header, descriptions, description_width = [], {}, 0 if options.input_filename_descriptions: E.info("reading descriptions from %s" % options.input_filename_descriptions) with IOTools.openFile(options.input_filename_descriptions) as inf: first = True for line in inf: if line.startswith("#"): continue data = line[:-1].split("\t") if description_width: assert len(data) - 1 == description_width, \ "inconsistent number of descriptions in %s" %\ options.input_filename_descriptions else: description_width = len(data) - 1 if first: description_header = data[1:] first = False else: descriptions[data[0]] = data[1:] assert len(description_header) == description_width, \ "number of descriptions (%i) inconsistent with header (%s) in %s" % \ (description_width, len(description_header), options.input_filename_descriptions) return description_header, descriptions, description_width
def readSegmentList(label, filenames, enable_split_tracks=False, ignore_tracks=False): """read one or more segment files. Arguments --------- label : string Label to use for IntervalCollection. filenames : list List of filenames to load in :term:`bed` format. enable_split_tracks : bool If True, allow tracks to be split across multiple files. ignore_tracks : int If True, ignore track information. Returns ------- segments : IntervalCollection The segment collection. """ results = Engine.IntervalCollection(name=label) E.info("%s: reading tracks from %i files" % (label, len(filenames))) results.load(filenames, allow_multiple=enable_split_tracks, ignore_tracks=ignore_tracks) E.info("%s: read %i tracks from %i files" % (label, len(results), len(filenames))) return results
def readSegmentList(label, filenames, enable_split_tracks=False, ignore_tracks=False): """read one or more segment files. Arguments --------- label : string Label to use for IntervalCollection. filenames : list List of filenames to load in :term:`bed` format. enable_split_tracks : bool If True, allow tracks to be split across multiple files. ignore_tracks : int If True, ignore track information. Returns ------- segments : IntervalCollection The segment collection. """ results = Engine.IntervalCollection(name=label) E.info("%s: reading tracks from %i files" % (label, len(filenames))) results.load(filenames, allow_multiple=enable_split_tracks, ignore_tracks=ignore_tracks) E.info("%s: read %i tracks from %i files" % (label, len(results), len(filenames))) return results
def fromCounts(filename): '''build annotator results from a tab-separated table with counts.''' annotator_results = [] with IOTools.openFile(filename, "r") as infile: E.info("loading data") header = infile.readline() if not header == "track\tannotation\tobserved\tcounts\n": raise ValueError("%s not a counts file: got %s" % (infile, header)) for line in infile: track, annotation, observed, counts = line[:-1].split("\t") samples = numpy.array( list(map(float, counts.split(","))), dtype=numpy.float) observed = float(observed) annotator_results.append(Engine.AnnotatorResult( track=track, annotation=annotation, counter="na", observed=observed, samples=samples)) return annotator_results
def __init__(self, num_samples, samples, samples_outfile, sampler, workspace_generator, counters, outfiles, num_threads=1): self.num_samples = num_samples self.samples = samples self.samples_outfile = samples_outfile self.sampler = sampler self.workspace_generator = workspace_generator self.counters = counters self.outfile_sample_stats = outfiles.get("sample_stats", None) self.outfile_sample_metrics = outfiles.get("sample_metrics", None) if self.outfile_sample_stats: E.debug("sample stats go to %s" % self.outfile_sample_stats) self.outfile_sample_stats.write( "sample\tisochore\tnsegments\tnnucleotides\tmean\t" "std\tmin\tq1\tmedian\tq3\tmax\n") self.last_sample_id = None self.all_lengths = [] self.num_threads = num_threads
def readDescriptions(options): '''read descriptions from tab separated file.''' description_header, descriptions, description_width = [], {}, 0 if options.input_filename_descriptions: E.info("reading descriptions from %s" % options.input_filename_descriptions) with IOTools.openFile(options.input_filename_descriptions) as inf: first = True for line in inf: if line.startswith("#"): continue data = line[:-1].split("\t") if description_width: assert len( data) - 1 == description_width, "inconsistent number of descriptions in %s" % options.input_filename_descriptions else: description_width = len(data) - 1 if first: description_header = data[1:] first = False else: descriptions[data[0]] = data[1:] assert len(description_header) == description_width, "number of descriptions (%i) inconsistent with header (%s) in %s" % \ (description_width, len(description_header), options.input_filename_descriptions) return description_header, descriptions, description_width
def readSegmentList(label, filenames, options, enable_split_tracks=False): # read one or more segment files results = GatEngine.IntervalCollection(name=label) E.info("%s: reading tracks from %i files" % (label, len(filenames))) results.load(filenames, split_tracks=enable_split_tracks) E.info("%s: read %i tracks from %i files" % (label, len(results), len(filenames))) dumpStats(results, "stats_%s_raw" % label, options) results.normalize() dumpStats(results, "stats_%s_normed" % label, options) return results
def computeSamples(self, work, report_interval=100): '''compute samples according to work. returns a list of results. ''' n = len(work) E.debug('sampling will work on %i items' % n) results = [] if self.num_threads == 0: for i, w in enumerate(work): r = computeSample( (w, self.samples_outfile, self.outfile_sample_metrics, None)) if i % report_interval == 0: E.info("%i/%i done (%5.2f)" % (i, n, 100.0 * i / n)) results.append(r) else: E.info("generating processpool with %i threads for %i items" % (self.num_threads, len(work))) manager = multiprocessing.Manager() lock = manager.Lock() pool = multiprocessing.Pool(self.num_threads) # use file names - not files when multiprocessing samples_outfile, metrics_outfile = None, None if self.samples_outfile: samples_outfile = self.samples_outfile.name self.samples_outfile.flush() if self.outfile_sample_metrics: metrics_outfile = self.outfile_sample_metrics.name self.outfile_sample_metrics.flush() ww = [(w, samples_outfile, metrics_outfile, lock) for w in work] for i, r in enumerate(pool.imap_unordered(computeSample, ww)): if i % report_interval == 0: E.info("%i/%i done (%5.2f)" % (i, n, 100.0 * i / n)) results.append(r) pool.close() pool.join() return results
def run(segments, annotations, workspace, sampler, counters, workspace_generator, **kwargs): '''run an enrichment analysis. segments: an IntervalCollection workspace: an IntervalCollection annotations: an IntervalCollection kwargs recognized are: cache filename of cache num_samples number of samples to compute output_counts_pattern output counts to filename output_samples_pattern if given, output samles to these files, one per segment sample_files if given, read samples from these files. fdr method to compute qvalues outfiles dictionary of optional additional output files. pseudo_count pseudo_count to add to observed and expected values reference data with reference observed and expected values. ''' # get arguments num_samples = kwargs.get("num_samples", 10000) cache = kwargs.get("cache", None) output_counts_pattern = kwargs.get("output_counts_pattern", None) sample_files = kwargs.get("sample_files", []) pseudo_count = kwargs.get("pseudo_count", 1.0) reference = kwargs.get("reference", None) output_samples_pattern = kwargs.get("output_samples_pattern", None) outfiles = kwargs.get("outfiles", {}) num_threads = kwargs.get("num_threads", 0) ################################################## ################################################## ################################################## # computing summary metrics for segments if "segment_metrics" in outfiles: E.info("computing summary metrics for segments") outfile = outfiles["segment_metrics"] outfile.write("track\tsection\tmetric\t%s\n" % "\t".join(Stats.Summary().getHeaders())) for track in segments.tracks: IO.outputMetrics(outfile, segments[track], workspace, track, 'segments', ) E.info("wrote summary metrics for segments to %s" % str(outfile)) ################################################## ################################################## ################################################## # collect observed counts from segments E.info("collecting observed counts") observed_counts = [] for counter in counters: observed_counts.append(Engine.computeCounts( counter=counter, aggregator=sum, segments=segments, annotations=annotations, workspace=workspace, workspace_generator=workspace_generator)) ################################################## ################################################## ################################################## # sample and collect counts ################################################## E.info("starting sampling") if cache: E.info("samples are cached in %s" % cache) samples = Engine.SamplesCached(filename=cache) elif sample_files: if not output_samples_pattern: raise ValueError( "require output_samples_pattern if loading samples from files") # build regex regex = re.compile(re.sub("%s", "(\S+)", output_samples_pattern)) E.info("loading samples from %i files" % len(sample_files)) samples = Engine.SamplesFile( filenames=sample_files, regex=regex) else: samples = Engine.Samples() sampled_counts = {} counts = E.Counter() ntracks = len(segments.tracks) for ntrack, track in enumerate(segments.tracks): segs = segments[track] E.info("sampling: %s: %i/%i" % (track, ntrack + 1, ntracks)) if output_samples_pattern and not sample_files: filename = re.sub("%s", track, output_samples_pattern) E.debug("saving samples to %s" % filename) dirname = os.path.dirname(filename) if dirname and not os.path.exists(dirname): os.makedirs(dirname) if filename.endswith(".gz"): samples_outfile = gzip.open(filename, "w") else: samples_outfile = open(filename, "w") else: samples_outfile = None if workspace_generator.is_conditional: outer_sampler = ConditionalSampler(num_samples, samples, samples_outfile, sampler, workspace_generator, counters, outfiles, num_threads=num_threads) else: outer_sampler = UnconditionalSampler(num_samples, samples, samples_outfile, sampler, workspace_generator, counters, outfiles, num_threads=num_threads) counts_per_track = outer_sampler.sample( track, counts, counters, segs, annotations, workspace, outfiles) # skip empty tracks if counts_per_track is None: continue if samples_outfile: samples_outfile.close() sampled_counts[track] = counts_per_track # old code, refactor into loop to save samples if 0: E.info("sampling stats: %s" % str(counts)) if track not in samples: E.warn("no samples for track %s" % track) continue # clean up samples del samples[track] E.info("sampling finished") # build annotator results E.info("computing PValue statistics") annotator_results = list() counter_id = 0 for counter, observed_count in zip(counters, observed_counts): for track, r in observed_count.items(): for annotation, observed in r.items(): temp_segs, temp_annos, temp_workspace = workspace_generator( segments[track], annotations[annotation], workspace) # ignore empty results if temp_workspace.sum() == 0: continue # if reference is given, p-value will indicate difference # The test that track and annotation are present is done # elsewhere if reference: ref = reference[track][annotation] else: ref = None annotator_results.append(Engine.AnnotatorResultExtended( track=track, annotation=annotation, counter=counter.name, observed=observed, samples=sampled_counts[track][counter_id][annotation], track_segments=temp_segs, annotation_segments=temp_annos, workspace=temp_workspace, reference=ref, pseudo_count=pseudo_count)) counter_id += 1 # dump (large) table with counts if output_counts_pattern: for counter in counters: name = counter.name filename = re.sub("%s", name, output_counts_pattern) E.info("writing counts to %s" % filename) output = [x for x in annotator_results if x.counter == name] outfile = IOTools.openFile(filename, "w") outfile.write("track\tannotation\tobserved\tcounts\n") for o in output: outfile.write("%s\t%s\t%i\t%s\n" % (o.track, o.annotation, o.observed, ",".join(["%i" % x for x in o.samples]))) return annotator_results
def sample(self, track, counts, counters, segs, annotations, workspace, outfiles): '''sample and return counts. Return a list of counted results for each counter. ''' E.info("performing unconditional sampling") counts_per_track = [collections.defaultdict(list) for x in counters] # rebuild non-isochore annotations and workspace contig_annotations = annotations.clone() contig_annotations.fromIsochores() contig_annotations.setName("contig_" + annotations.getName()) contig_workspace = workspace.clone() contig_workspace.fromIsochores() E.info("workspace without conditioning: %i segments, %i nucleotides" % (workspace.counts(), workspace.sum())) temp_segs, _, temp_workspace = self.workspace_generator( segs, None, workspace) E.info("workspace after conditioning: %i segments, %i nucleotides" % (workspace.counts(), workspace.sum())) if workspace.sum() == 0: E.warn("empty workspace - no computation performed") return None work = [WorkData(track, x, self.sampler, temp_segs, annotations, contig_annotations, temp_workspace, contig_workspace, counters, ) for x in range(self.num_samples)] if self.num_threads > 0: E.info("setting up shared data for multi-processing") annotations.share() contig_annotations.share() contig_workspace.share("contig_workspace") temp_segs.share("generated_segments") temp_workspace.share("generated_workspace") E.info("sampling started") results = self.computeSamples(work) E.info("sampling completed") if self.num_threads > 0: E.info("retrieving private data") annotations.unshare() contig_annotations.unshare() contig_workspace.unshare() temp_segs.unshare() temp_workspace.unshare() # collate results for result in results: for counter_id, counter in enumerate(counters): for annotation in annotations.tracks: counts_per_track[counter_id][annotation].append( result[counter_id][annotation]) self.outputSampleStats(None, "", []) return counts_per_track
def main(argv): if not argv: argv = sys.argv # setup command line parser parser = optparse.OptionParser( version= "%prog version: $Id: script_template.py 2871 2010-03-03 10:20:44Z andreas $", usage=globals()["__doc__"]) parser.add_option("-a", "--annotation-file", "--annotations", dest="annotation_files", type="string", action="append", help="filename with annotations [default=%default].") parser.add_option( "-s", "--segment-file", "--segments", dest="segment_files", type="string", action="append", help= "filename with segments. Also accepts a glob in parentheses [default=%default]." ) parser.add_option( "-w", "--workspace-file", "--workspace", dest="workspace_files", type="string", action="append", help= "filename with workspace segments. Also accepts a glob in parentheses [default=%default]." ) parser.add_option( "-i", "--isochore-file", "--isochores", dest="isochore_files", type="string", action="append", help= "filename with isochore segments. Also accepts a glob in parentheses [default=%default]." ) parser.add_option( "-o", "--order", dest="output_order", type="choice", choices=("track", "annotation", "fold", "pvalue", "qvalue"), help="order results in output by fold, track, etc. [default=%default]." ) parser.add_option( "-q", "--qvalue-method", dest="qvalue_method", type="choice", choices=("storey", "BH", "bonferroni", "holm", "hommel", "hochberg", "BY", "none"), help= "method to perform multiple testing correction by controlling the fdr [default=%default]." ) parser.add_option("--qvalue-lambda", dest="qvalue_lambda", type="float", help="fdr computation: lambda [default=%default].") parser.add_option( "--qvalue-pi0-method", dest="qvalue_pi0_method", type="choice", choices=("smoother", "bootstrap"), help="fdr computation: method for estimating pi0 [default=%default].") parser.add_option( "--descriptions", dest="input_filename_descriptions", type="string", help="filename mapping annotation terms to descriptions. " " if given, the output table will contain additional columns " " [default=%default]") parser.add_option( "--ignore-segment-tracks", dest="ignore_segment_tracks", action="store_true", help= "ignore segment tracks - all segments belong to one track [default=%default]" ) parser.add_option( "--enable-split-tracks", dest="enable_split_tracks", action="store_true", help="permit the same track to be in multiple files [default=%default]" ) parser.add_option("--output-bed", dest="output_bed", type="choice", action="append", choices=("all", "annotations", "segments", "workspaces", "isochores", "overlap"), help="output bed files [default=%default].") parser.add_option("--output-stats", dest="output_stats", type="choice", action="append", choices=("all", "annotations", "segments", "workspaces", "isochores", "overlap"), help="output overlap summary stats [default=%default].") parser.add_option( "--restrict-workspace", dest="restrict_workspace", action="store_true", help="restrict workspace to those segments that contain both track" " and annotations [default=%default]") parser.add_option("--counter", dest="counters", type="choice", action="append", choices=("binom", "hyperg"), help="counter to use [default=%default].") parser.add_option( "--output-tables-pattern", dest="output_tables_pattern", type="string", help= "output pattern for result tables. Used if there are multiple counters used [default=%default]." ) parser.set_defaults(annotation_files=[], segment_files=[], workspace_files=[], sample_files=[], counters=[], output_stats=[], output_bed=[], output_tables_pattern="%s.tsv.gz", output_order="fold", input_filename_counts=None, input_filename_results=None, pvalue_method="empirical", output_plots_pattern=None, output_samples_pattern=None, qvalue_method="storey", qvalue_lambda=None, qvalue_pi0_method="smoother", ignore_segment_tracks=False, input_filename_descriptions=None, conditional="unconditional", conditional_extension=None, conditional_expansion=None, restrict_workspace=False, enable_split_tracks=False, shift_expansion=2.0, shift_extension=0, overlap_mode="midpoint", truncate_workspace_to_annotations=False, truncate_segments_to_workspace=False) # add common options (-h/--help, ...) and parse command line (options, args) = E.Start(parser, argv=argv, add_output_options=True) tstart = time.time() if len(options.counters) == 0: options.counters.append("binom") ############################################ segments, annotations, workspaces, isochores = IO.buildSegments(options) E.info("intervals loaded in %i seconds" % (time.time() - tstart)) # filter segments by workspace workspace = IO.applyIsochores(segments, annotations, workspaces, options, isochores) ############################################ description_header, descriptions, description_width = IO.readDescriptions( options) ############################################ ############################################ # compute per contig # compute bases covered by workspace workspace2basecoverage, isochores = {}, [] for contig, ww in workspace.iteritems(): workspace2basecoverage[contig] = ww.sum() isochores.append(contig) # compute percentage of bases covered by annotations in workspace # per isochore annotation2basecoverage = collections.defaultdict(dict) for annotation, aa in annotations.iteritems(): for isochore, a in aa.iteritems(): # need to truncate to workspace? annotation2basecoverage[annotation][isochore] = a.sum() results_per_contig = collections.defaultdict(list) E.info("computing counts per isochore") # results per isochore def emptyResult(segment, annotation, isochore, counter, nsegments_in_workspace, basecoverage_annotation, basecoverage_workspace): return GREAT_RESULT._make(( segment, annotation, isochore, counter, 0, # observed 0, # expected nsegments_in_workspace, 0, # nannotations_in_workspace 0, # nsegments_overlapping_annotation 0, # nannotations_overlapping_segments 0, # basecoverage_intersection 0, # basecoverage_segments basecoverage_annotation, basecoverage_workspace, 0.0, 1.0, 1.0, 1.0)) for isochore in isochores: basecoverage_workspace = workspace2basecoverage[isochore] # iterate over all isochores for segment, segmentdict in segments.iteritems(): try: ss = segmentdict[isochore] # select segments overlapping workspace segments_in_workspace = GatSegmentList.SegmentList(clone=ss) segments_in_workspace.intersect(workspace[isochore]) # number of segments in workspace nsegments_in_workspace = len(segments_in_workspace) except KeyError: ss = None basecoverage_segments = segments_in_workspace.sum() for annotation, annotationdict in annotations.iteritems(): # if annotation != "GO:0030957": continue try: aa = annotationdict[isochore] except KeyError: aa = None # p_A: proportion of bases covered by annotation try: basecoverage_annotation = annotation2basecoverage[ annotation][isochore] except KeyError: basecoverage_annotation = 0 if ss == None or aa == None: for counter in options.counters: results_per_contig[(counter, segment, annotation)].append( emptyResult( segment, annotation, isochore, counter, nsegments_in_workspace, basecoverage_annotation, basecoverage_workspace)) continue # select segments overlapping annotation segments_overlapping_annotation = GatSegmentList.SegmentList( clone=ss) segments_overlapping_annotation.intersect( annotations[annotation][isochore]) # number of segments in annotation nsegments_overlapping_annotation = ss.intersectionWithSegments( annotations[annotation][isochore], mode=options.overlap_mode) # number of nucleotides at the intersection of segments, # annotation and workspace basecoverage_intersection = segments_overlapping_annotation.sum( ) annotations_overlapping_segments = GatSegmentList.SegmentList( clone=aa) annotations_overlapping_segments.intersect(ss) nannotations_overlapping_segments = len( annotations_overlapping_segments) nannotations_in_workspace = len(aa) if nannotations_in_workspace == 0: for counter in options.counters: results_per_contig[(counter, segment, annotation)].append( emptyResult( segment, annotation, isochore, counter, nsegments_in_workspace, basecoverage_annotation, basecoverage_workspace)) continue fraction_coverage_annotation = basecoverage_annotation / \ float(basecoverage_workspace) fraction_hit_annotation = float( nannotations_overlapping_segments ) / nannotations_in_workspace for counter in options.counters: if counter.startswith("binom"): # GREAT binomial probability over "regions" # n = number of genomic regions = nannotations_in_workspace # ppi = fraction of genome annotated by annotation = fraction_coverage_annotation # kpi = genomic regions with annotation hit by segments = nannotations_in_segments # sf = survival functions = 1 -cdf # probability of observing >kpi in a sample of n where the probabily of succes is # ppi. pvalue = scipy.stats.binom.sf( nsegments_overlapping_annotation - 1, nsegments_in_workspace, fraction_coverage_annotation) expected = fraction_coverage_annotation * \ nsegments_in_workspace observed = nsegments_overlapping_annotation elif counter.startswith("hyperg"): # hypergeometric probability over nucleotides # Sampling without replacement # x,M,n,M # x = observed number of nucleotides in overlap of segments,annotations and workspace # M = number of nucleotides in workspace # n = number of nucleotides in annotations (and workspace) # N = number of nucleotides in segments (and workspace) # P-value of obtaining >x number of nucleotides # overlapping. rv = scipy.stats.hypergeom(basecoverage_workspace, basecoverage_annotation, basecoverage_segments) pvalue = rv.sf(basecoverage_intersection) expected = rv.mean() observed = basecoverage_intersection if expected != 0: fold = float(observed) / expected else: fold = 1.0 r = GREAT_RESULT._make( (segment, annotation, isochore, counter, observed, expected, nsegments_in_workspace, nannotations_in_workspace, nsegments_overlapping_annotation, nannotations_overlapping_segments, basecoverage_intersection, basecoverage_segments, basecoverage_annotation, basecoverage_workspace, fraction_coverage_annotation, fold, pvalue, 1.0)) # print "\t".join( map(str, r)) results_per_contig[(counter, segment, annotation)].append(r) E.info("merging counts per isochore") # compute sums results = [] for niteration, pair in enumerate(results_per_contig.iteritems()): counter, segment, annotation = pair[0] data = pair[1] nsegments_in_workspace = sum([x.nsegments_in_workspace for x in data]) nsegments_overlapping_annotation = sum([x.observed for x in data]) nannotations_in_workspace = sum( [x.nannotations_in_workspace for x in data]) nannotations_overlapping_segments = sum( [x.nannotations_overlapping_segments for x in data]) basecoverage_intersection = sum( [x.basecoverage_intersection for x in data]) basecoverage_segments = sum([x.basecoverage_segments for x in data]) basecoverage_annotation = sum( [x.basecoverage_annotation for x in data]) basecoverage_workspace = sum([x.basecoverage_workspace for x in data]) fraction_coverage_annotation = basecoverage_annotation / \ float(basecoverage_workspace) if counter.startswith("binom"): pvalue = scipy.stats.binom.sf(nsegments_overlapping_annotation - 1, nsegments_in_workspace, fraction_coverage_annotation) expected = fraction_coverage_annotation * nsegments_in_workspace observed = nsegments_overlapping_annotation elif counter.startswith("hyperg"): rv = scipy.stats.hypergeom(basecoverage_workspace, basecoverage_annotation, basecoverage_segments) pvalue = rv.sf(basecoverage_intersection) expected = rv.mean() observed = basecoverage_intersection if expected != 0: fold = float(observed) / expected else: fold = 1.0 r = GREAT_RESULT._make( (segment, annotation, "all", counter, observed, expected, nsegments_in_workspace, nannotations_in_workspace, nsegments_overlapping_annotation, nannotations_overlapping_segments, basecoverage_intersection, basecoverage_segments, basecoverage_annotation, basecoverage_workspace, fraction_coverage_annotation, fold, pvalue, 1.0)) results.append(r) IO.outputResults(results, options, GREAT_RESULT._fields, description_header, description_width, descriptions) E.Stop()
def buildSegments(options): '''load segments, annotations and workspace from parameters defined in *options*. The workspace will be split by isochores. returns segments, annotations and workspace. ''' options.segment_files = expandGlobs(options.segment_files) options.annotation_files = expandGlobs(options.annotation_files) options.workspace_files = expandGlobs(options.workspace_files) options.sample_files = expandGlobs(options.sample_files) ################################################## # arguments sanity check if not options.segment_files: raise ValueError("please specify at least one segment file") if not options.annotation_files: raise ValueError("please specify at least one annotation file") if not options.workspace_files: raise ValueError("please specify at least one workspace file") # read one or more segment files segments = readSegmentList("segments", options.segment_files, ignore_tracks=options.ignore_segment_tracks) segments.normalize() if segments.sum() == 0: E.critical("no segments in input file - run aborted") raise ValueError("segments file is empty - run aborted") if len(segments) > 1000: raise ValueError("too many (%i) segment files - use track definitions " "or --ignore-segment-tracks" % len(segments)) annotations = readSegmentList( "annotations", options.annotation_files, enable_split_tracks=options.enable_split_tracks, ignore_tracks=options.annotations_label is not None) if options.annotations_label is not None: annotations.setName(options.annotations_label) if options.annotations_to_points: annotations.toPositions(options.annotations_to_points) if options.overlapping_annotations: # only sort, do not merge annotations.sort() else: annotations.normalize() workspaces = readSegmentList("workspaces", options.workspace_files, options, options.enable_split_tracks) workspaces.normalize() # intersect workspaces to build a single workspace E.info("collapsing workspaces") dumpStats(workspaces, "stats_workspaces_input", options) workspaces.collapse() dumpStats(workspaces, "stats_workspaces_collapsed", options) # use merged workspace only, discard others workspaces.restrict("collapsed") # build isochores or intersect annotations/segments with workspace if options.isochore_files: # read one or more isochore files isochores = Engine.IntervalCollection(name="isochores") E.info("%s: reading isochores from %i files" % ("isochores", len(options.isochore_files))) isochores.load(options.isochore_files) dumpStats(isochores, "stats_isochores_raw", options) # merge isochores and check if consistent (fully normalized) isochores.sort() # check that there are no overlapping segments within isochores isochores.check() # TODO: flag is_normalized not properly set isochores.normalize() # check that there are no overlapping segments between isochores # truncate isochores to workspace # crucial if isochores are larger than workspace. isochores.intersect(workspaces["collapsed"]) else: isochores = None return segments, annotations, workspaces, isochores
def outputResults(results, options, header, description_header, description_width, descriptions, format_observed="%i"): '''compute FDR and output results.''' pvalues = [x.pvalue for x in results] ################################################## ################################################## ################################################## # compute global fdr ################################################## E.info("computing FDR statistics") qvalues = GatEngine.getQValues(pvalues, method=options.qvalue_method, vlambda=options.qvalue_lambda, pi0_method=options.qvalue_pi0_method) try: results = [x._replace(qvalue=qvalue) for x, qvalue in zip(results, qvalues)] is_tuple = True except AttributeError: # not a namedtuple for x, qvalue in zip(results, qvalues): x.qvalue = qvalue x.format_observed = format_observed is_tuple = False counters = set([x.counter for x in results]) for counter in counters: if len(counters) == 1: outfile = options.stdout output = results else: outfilename = re.sub("%s", counter, options.output_tables_pattern) E.info("output for counter %s goes to outfile %s" % (counter, outfilename)) outfile = IOTools.openFile(outfilename, "w") output = [x for x in results if x.counter == counter] outfile.write( "\t".join(list(header) + list(description_header)) + "\n") if options.output_order == "track": output.sort(key=lambda x: (x.track, x.annotation)) elif options.output_order == "observed": output.sort(key=lambda x: x.observed) elif options.output_order == "annotation": output.sort(key=lambda x: (x.annotation, x.track)) elif options.output_order == "fold": output.sort(key=lambda x: x.fold) elif options.output_order == "pvalue": output.sort(key=lambda x: x.pvalue) elif options.output_order == "qvalue": output.sort(key=lambda x: x.qvalue) else: raise ValueError("unknown sort order %s" % options.output_order) for result in output: if is_tuple: outfile.write("\t".join(map(str, result))) else: outfile.write(str(result)) if descriptions: try: outfile.write( "\t" + "\t".join(descriptions[result.annotation])) except KeyError: outfile.write("\t" + "\t".join([""] * description_width)) outfile.write("\n") if outfile != options.stdout: outfile.close()
def dumpBed(coll, section, options): if section in options.output_bed or \ "all" in options.output_bed or \ len([x for x in options.output_bed if re.search(x, section)]) > 0: coll.save(E.openOutputFile(section + ".bed"))
def dumpStats(coll, section, options): if section in options.output_stats or \ "all" in options.output_stats or \ len([x for x in options.output_stats if re.search(x, section)]) > 0: coll.outputStats(E.openOutputFile(section))
def applyIsochores(segments, annotations, workspaces, options, isochores=None, truncate_segments_to_workspace=False, truncate_workspace_to_annotations=False, restrict_workspace=False, ): '''apply isochores to segments and annotations. Segments and annotations are filtered in place to keep only those overlapping the workspace. If *isochores* are given, isochores are applied. If *truncate_segments_to_workspace*, truncate segments to workspace. If *restrict_workspace* is set, the workspace is confined to those parts that overlap both a segment and an annotation. If *truncate_workspace_to_annotations* is set, the workspace is truncated to keep only those parts that overlap annotations. returns a workspace divided into isochores. ''' if isochores: # intersect isochores and workspaces, segments and annotations # workspace and annotations are truncated # with segments it is optional. E.info("adding isochores to workspace") workspaces.toIsochores(isochores, truncate=True) annotations.toIsochores(isochores, truncate=True) segments.toIsochores( isochores, truncate=options.truncate_segments_to_workspace) if workspaces.sum() == 0: raise ValueError("isochores and workspaces do not overlap") if annotations.sum() == 0: raise ValueError("isochores and annotations do not overlap") if segments.sum() == 0: raise ValueError("isochores and segments do not overlap") dumpStats(workspaces, "stats_workspaces_isochores", options) dumpStats(annotations, "stats_annotations_isochores", options) dumpStats(segments, "stats_segments_isochores", options) dumpBed(workspaces, "workspaces_isochores", options) dumpBed(annotations, "annotations_isochores", options) dumpBed(segments, "segments_isochores", options) else: # intersect workspace and segments/annotations # annotations and segments are truncated by workspace if options.truncate_segments_to_workspace: segments.intersect(workspaces["collapsed"]) else: segments.filter(workspaces["collapsed"]) annotations.intersect(workspaces["collapsed"]) dumpStats(annotations, "stats_annotations_truncated", options) dumpStats(segments, "stats_segments_truncated", options) workspace = workspaces["collapsed"] if restrict_workspace: E.info("restricting workspace") # this is very cumbersome - refactor merge and collapse # to return an IntervalDictionary instead of adding it # to the list of tracks for x in (segments, annotations): if "merged" in segments: workspace.filter(segments["merged"]) else: segments.merge() workspace.filter(segments["merged"]) del segments["merged"] dumpStats(workspaces, "stats_workspaces_restricted", options) if truncate_workspace_to_annotations: E.info("truncating workspace to annotations") annotations.merge() workspace.intersect(annotations["merged"]) del annotations["merged"] dumpStats(workspaces, "stats_workspaces_truncated", options) # segments.dump( open("segments_dump.bed", "w" ) ) # workspaces.dump( open("workspaces_dump.bed", "w" ) ) # output overlap stats # output segment densities per workspace if "overlap" in options.output_stats or \ "all" in options.output_stats: for track in segments.tracks: workspaces.outputOverlapStats(E.openOutputFile("overlap_%s" % track), segments[track]) return workspace
def main(argv): if not argv: argv = sys.argv # setup command line parser parser = optparse.OptionParser(version="%prog version: $Id: script_template.py 2871 2010-03-03 10:20:44Z andreas $", usage=globals()["__doc__"]) parser.add_option("-a", "--gene-file", "--annotations", dest="annotation_files", type="string", action="append", help="filename with annotations - here, location of genes [default=%default].") parser.add_option("-s", "--segment-file", "--segments", dest="segment_files", type="string", action="append", help="filename with segments. Also accepts a glob in parentheses [default=%default].") parser.add_option("-w", "--workspace-file", "--workspace", dest="workspace_files", type="string", action="append", help="filename with workspace segments. Also accepts a glob in parentheses [default=%default].") parser.add_option("-g", "--number-of-genes", dest="number_of_genes", type="int", help="total number of genes [default=%default]") parser.add_option("-m", "--annotation-file", dest="annotation_file", type="string", help="filename mapping genes to annotations [default=%default]") parser.add_option("-o", "--order", dest="output_order", type="choice", choices=( "track", "annotation", "fold", "pvalue", "qvalue"), help="order results in output by fold, track, etc. [default=%default].") parser.add_option("-q", "--qvalue-method", dest="qvalue_method", type="choice", choices=( "storey", "BH", "bonferroni", "holm", "hommel", "hochberg", "BY", "none"), help="method to perform multiple testing correction by controlling the fdr [default=%default].") parser.add_option("--qvalue-lambda", dest="qvalue_lambda", type="float", help="fdr computation: lambda [default=%default].") parser.add_option("--qvalue-pi0-method", dest="qvalue_pi0_method", type="choice", choices=("smoother", "bootstrap"), help="fdr computation: method for estimating pi0 [default=%default].") parser.add_option("--descriptions", dest="input_filename_descriptions", type="string", help="filename mapping annotation terms to descriptions. " " if given, the output table will contain additional columns " " [default=%default]") parser.add_option("--ignore-segment-tracks", dest="ignore_segment_tracks", action="store_true", help="ignore segment tracks - all segments belong to one track [default=%default]") parser.add_option("--enable-split-tracks", dest="enable_split_tracks", action="store_true", help="permit the same track to be in multiple files [default=%default]") parser.add_option("--output-bed", dest="output_bed", type="choice", action="append", choices=("all", "annotations", "segments", "workspaces", "isochores", "overlap"), help="output bed files [default=%default].") parser.add_option("--output-stats", dest="output_stats", type="choice", action="append", choices=("all", "annotations", "segments", "workspaces", "isochores", "overlap"), help="output overlap summary stats [default=%default].") parser.set_defaults( annotation_files=[], segment_files=[], workspace_files=[], sample_files=[], annotation_file=None, num_samples=1000, nbuckets=100000, bucket_size=1, counter="nucleotide-overlap", output_stats=[], output_bed=[], output_filename_counts=None, output_order="fold", cache=None, input_filename_counts=None, input_filename_results=None, pvalue_method="empirical", output_plots_pattern=None, output_samples_pattern=None, qvalue_method="storey", qvalue_lambda=None, qvalue_pi0_method="smoother", sampler="annotator", ignore_segment_tracks=False, input_filename_descriptions=None, conditional="unconditional", conditional_extension=None, conditional_expansion=None, restrict_workspace=False, enable_split_tracks=False, shift_expansion=2.0, shift_extension=0, overlap_mode="midpoint", number_of_genes=None, ) # add common options (-h/--help, ...) and parse command line (options, args) = E.Start(parser, argv=argv, add_output_options=True) tstart = time.time() # load segments options.segment_files = IO.expandGlobs(options.segment_files) options.annotation_files = IO.expandGlobs(options.annotation_files) options.workspace_files = IO.expandGlobs(options.workspace_files) # read one or more segment files segments = IO.readSegmentList("segments", options.segment_files, options) if options.ignore_segment_tracks: segments.merge(delete=True) E.info("merged all segments into one track with %i segments" % len(segments)) if len(segments) > 1000: raise ValueError( "too many (%i) segment files - use track definitions or --ignore-segment-tracks" % len(segments)) # load workspace workspaces = IO.readSegmentList( "workspaces", options.workspace_files, options, options.enable_split_tracks) # intersect workspaces to build a single workspace E.info("collapsing workspaces") workspaces.collapse() # use merged workspace only, discard others workspaces.restrict("collapsed") workspace = workspaces["collapsed"] E.info("intervals loaded in %i seconds" % (time.time() - tstart)) ############################################ # load table mapping a gene id to annotations gene2annotations = IOTools.readMultiMap(IOTools.openFile(options.annotation_file), has_header=True) annotations = set([y for x in gene2annotations.values() for y in x]) E.info("loaded %i annotations for %i genes" % (len(gene2annotations), len(annotations))) ############################################ # load bed file with gene coordinates assert len(options.annotation_files) == 1 indexed_genes = collections.defaultdict(Intersecter) total_genes = 0 # number of genes per contig contig2ngenes = collections.defaultdict(int) # compute number of genes with a particular annotation # per contig annotation2ngenes = collections.defaultdict(int) for line in IOTools.openFile(options.annotation_files[0]): if line.startswith("#"): continue contig, start, end, gene_id = line[:-1].split("\t")[:4] indexed_genes[contig].add_interval( Interval(int(start), int(end), gene_id)) contig2ngenes[contig] += 1 total_genes += 1 try: for annotation in gene2annotations[gene_id]: annotation2ngenes[annotation] += 1 except KeyError: pass E.info("indexed locations for %i contigs" % len(indexed_genes)) ############################################ description_header, descriptions, description_width = IO.readDescriptions( options) ############################################ ############################################ # compute results E.info("computing counts") results = [] # iterate over segments for segment, segmentdict in segments.iteritems(): # genes hit by segments per annotation genes_hit_by_segments_with_annotations = collections.defaultdict(int) # genes hit by segments genes_hit_by_segments = 0 for contig, ss in segmentdict.iteritems(): for start, end in ss: overlapping_genes = list( indexed_genes[contig].find(start, end)) genes_hit_by_segments += len(overlapping_genes) for x in overlapping_genes: gene_id = x.value try: for annotation in gene2annotations[gene_id]: genes_hit_by_segments_with_annotations[ annotation] += 1 except KeyError: pass # N = number of genes in genome N = total_genes # n = number of genes selected by segments n = genes_hit_by_segments for annotation in annotations: # K = number of genes carrying annotation K = annotation2ngenes[annotation] # k = number of genes selected by segments and with annotation k = genes_hit_by_segments_with_annotations[annotation] if n == 0 or N == 0 or K == 0: expected = 0 fold = 1.0 pvalue = 1.0 else: expected = float(n * K) / N fold = k / expected pvalue = scipy.stats.hypergeom.sf(k - 1, N, K, n) r = GENESET_RESULT._make(( segment, annotation, N, K, n, k, expected, fold, pvalue, 1.0)) results.append(r) IO.outputResults(results, options, GENESET_RESULT._fields, description_header, description_width, descriptions) E.Stop()
def outputResults(results, options, header, description_header, description_width, descriptions, format_observed="%i"): '''compute FDR and output results.''' pvalues = [x.pvalue for x in results] ################################################## ################################################## ################################################## # compute global fdr ################################################## E.info("computing FDR statistics") qvalues = Engine.getQValues(pvalues, method=options.qvalue_method, vlambda=options.qvalue_lambda, pi0_method=options.qvalue_pi0_method) try: results = [ x._replace(qvalue=qvalue) for x, qvalue in zip(results, qvalues) ] is_tuple = True except AttributeError: # not a namedtuple for x, qvalue in zip(results, qvalues): x.qvalue = qvalue x.format_observed = format_observed is_tuple = False counters = set([x.counter for x in results]) for counter in counters: if len(counters) == 1: outfile = options.stdout output = results else: outfilename = re.sub("%s", counter, options.output_tables_pattern) E.info("output for counter %s goes to outfile %s" % (counter, outfilename)) outfile = IOTools.openFile(outfilename, "w") output = [x for x in results if x.counter == counter] outfile.write("\t".join(list(header) + list(description_header)) + "\n") if options.output_order == "track": output.sort(key=lambda x: (x.track, x.annotation)) elif options.output_order == "observed": output.sort(key=lambda x: x.observed) elif options.output_order == "annotation": output.sort(key=lambda x: (x.annotation, x.track)) elif options.output_order == "fold": output.sort(key=lambda x: x.fold) elif options.output_order == "pvalue": output.sort(key=lambda x: x.pvalue) elif options.output_order == "qvalue": output.sort(key=lambda x: x.qvalue) else: raise ValueError("unknown sort order %s" % options.output_order) for result in output: if is_tuple: outfile.write("\t".join(map(str, result))) else: outfile.write(str(result)) if descriptions: try: outfile.write("\t" + "\t".join(descriptions[result.annotation])) except KeyError: outfile.write("\t" + "\t".join([""] * description_width)) outfile.write("\n") if outfile != options.stdout: outfile.close()
def plotResults(results, options): '''plot annotator results.''' ################################################## # plot histograms if options.output_plots_pattern and HASPLOT: def buildPlotFilename(options, key): filename = re.sub("%s", key, options.output_plots_pattern) filename = re.sub("[^a-zA-Z0-9-_./]", "_", filename) dirname = os.path.dirname(filename) if dirname and not os.path.exists(dirname): os.makedirs(dirname) return filename E.info("plotting sample stats") for r in results: plt.figure() k = [] if r.track != "merged": k.append(r.track) k.append(r.annotation) if r.counter != "na": k.append(r.counter) key = "-".join(k) s = r.samples hist, bins = numpy.histogram(s, bins=100) # plot bars plt.hist(s, bins=100, normed=True, label=key) plt.axvline(r.observed, color='r', linewidth=2) # plot estimated sigma = r.stddev mu = r.expected plt.plot(bins, 1.0 / (sigma * numpy.sqrt(2 * numpy.pi)) * numpy.exp(-(bins - mu)**2 / (2 * sigma**2)), label="std distribution", linewidth=2, color='g') plt.legend() filename = buildPlotFilename(options, key) plt.savefig(filename) E.info("plotting P-value distribution") key = "pvalue" plt.figure() x, bins, y = plt.hist([r.pvalue for r in results], bins=numpy.arange(0, 1.05, 0.025), label="pvalue") plt.hist([r.qvalue for r in results], bins=numpy.arange(0, 1.05, 0.025), label="qvalue", alpha=0.5) plt.legend() # hist, bins = numpy.histogram( \ # [r.pvalue for r in Engine.iterator_results(annotator_results) ], # bins = 20 ) # plt.plot( bins[:-1], hist, label = key ) filename = buildPlotFilename(options, key) plt.savefig(filename)
def plotResults(results, options): '''plot annotator results.''' ################################################## # plot histograms if options.output_plots_pattern and HASPLOT: def buildPlotFilename(options, key): filename = re.sub("%s", key, options.output_plots_pattern) filename = re.sub("[^a-zA-Z0-9-_./]", "_", filename) dirname = os.path.dirname(filename) if dirname and not os.path.exists(dirname): os.makedirs(dirname) return filename E.info("plotting sample stats") for r in results: plt.figure() k = [] if r.track != "merged": k.append(r.track) k.append(r.annotation) if r.counter != "na": k.append(r.counter) key = "-".join(k) s = r.samples hist, bins = numpy.histogram(s, bins=100) # plot bars plt.hist(s, bins=100, normed=True, label=key) plt.axvline(r.observed, color='r', linewidth=2) # plot estimated sigma = r.stddev mu = r.expected plt.plot(bins, 1.0 / (sigma * numpy.sqrt(2 * numpy.pi)) * numpy.exp(- (bins - mu) ** 2 / (2 * sigma ** 2)), label="std distribution", linewidth=2, color='g') plt.legend() filename = buildPlotFilename(options, key) plt.savefig(filename) E.info("plotting P-value distribution") key = "pvalue" plt.figure() x, bins, y = plt.hist([r.pvalue for r in results], bins=numpy.arange(0, 1.05, 0.025), label="pvalue") plt.hist([r.qvalue for r in results], bins=numpy.arange(0, 1.05, 0.025), label="qvalue", alpha=0.5) plt.legend() # hist, bins = numpy.histogram( \ # [r.pvalue for r in GatEngine.iterator_results(annotator_results) ], # bins = 20 ) # plt.plot( bins[:-1], hist, label = key ) filename = buildPlotFilename(options, key) plt.savefig(filename)
def main(argv=None): """script main. parses command line options in sys.argv, unless *argv* is given. """ if not argv: argv = sys.argv # setup command line parser parser = optparse.OptionParser( version= "%prog version: $Id: script_template.py 2871 2010-03-03 10:20:44Z andreas $", usage=globals()["__doc__"]) parser.add_option( "-o", "--order", dest="output_order", type="choice", choices=("track", "annotation", "fold", "pvalue", "qvalue", "observed"), help="order results in output by fold, track, etc. [default=%default]." ) parser.add_option("-p", "--pvalue-method", dest="pvalue_method", type="choice", choices=( "empirical", "norm", ), help="type of pvalue reported [default=%default].") parser.add_option( "-q", "--qvalue-method", dest="qvalue_method", type="choice", choices=("storey", "BH", "bonferroni", "holm", "hommel", "hochberg", "BY", "none"), help= "method to perform multiple testing correction by controlling the fdr [default=%default]." ) parser.add_option("--qvalue-lambda", dest="qvalue_lambda", type="float", help="fdr computation: lambda [default=%default].") parser.add_option( "--qvalue-pi0-method", dest="qvalue_pi0_method", type="choice", choices=("smoother", "bootstrap"), help="fdr computation: method for estimating pi0 [default=%default].") parser.add_option( "--descriptions", dest="input_filename_descriptions", type="string", help="filename mapping annotation terms to descriptions. " " if given, the output table will contain additional columns " " [default=%default]") parser.add_option( "--pseudo-count", dest="pseudo_count", type="float", help= "pseudo count. The pseudo count is added to both the observed and expected overlap. " " Using a pseudo-count avoids gat reporting fold changes of 0 [default=%default]." ) parser.add_option("--output-plots-pattern", dest="output_plots_pattern", type="string", help="output pattern for plots [default=%default]") parser.set_defaults( pvalue_method="empirical", qvalue_method="BH", qvalue_lambda=None, qvalue_pi0_method="smoother", # pseudo count for fold change computation to avoid 0 fc pseudo_count=1.0, output_order="observed", ) # add common options (-h/--help, ...) and parse command line (options, args) = E.Start(parser, argv=argv, add_output_options=True) input_filenames_counts = args ################################################## E.info("received %i filenames with counts" % len(input_filenames_counts)) ################################################## description_header, descriptions, description_width = IO.readDescriptions( options) all_annotator_results = [] for input_filename_counts in input_filenames_counts: E.info("processing %s" % input_filename_counts) annotator_results = gat.fromCounts(input_filename_counts) ################################################## if options.pvalue_method != "empirical": E.info("updating pvalues to %s" % options.pvalue_method) GatEngine.updatePValues(annotator_results, options.pvalue_method) ################################################## ################################################## ################################################## # compute global fdr ################################################## E.info("computing FDR statistics") GatEngine.updateQValues(annotator_results, method=options.qvalue_method, vlambda=options.qvalue_lambda, pi0_method=options.qvalue_pi0_method) all_annotator_results.append(annotator_results) pseudo_count = options.pseudo_count results = [] if len(all_annotator_results) == 1: E.info("performing pairwise comparison within a single file") # collect all annotations annotations, segments = list(), set() for x in all_annotator_results[0]: segments.add(x.track) annotations.append(x) if len(segments) != 1: raise NotImplementedError("multiple segments of interest") for data1, data2 in itertools.combinations(annotations, 2): # note that fold changes can be very large if there are 0 samples # this is fine for getting the distributional params (mean, stddev) fold_changes1 = data1.observed / (data1.samples + pseudo_count) fold_changes2 = data2.observed / (data2.samples + pseudo_count) # add a separate fc pseudo-count to avoid 0 values fold_changes1 += 0.0001 fold_changes2 += 0.0001 # Test is if relative fold change rfc is different from 1 # note: rfc = fc1 / fc2 = obs1 / exp1 * obs2 / exp2 # = obs1 / obs2 * exp2 / exp1 # Thus, it is equivalent to test rfc = obs1/obs2 versus exp2 / exp1 # # Convert to log space for easier plotting # Move the observed fold ratio in order to get an idea of the magnitude # of the underlying fold change delta_fold = data2.fold - data1.fold sampled_delta_fold = numpy.log( fold_changes1 / fold_changes2) + delta_fold observed_delta_fold = 0.0 + delta_fold result = GatEngine.AnnotatorResult(data1.annotation, data2.annotation, "na", observed_delta_fold, sampled_delta_fold, reference=None, pseudo_count=0) results.append(result) else: E.info("performing pairwise comparison between multiple files") ################################################## # perform pairwise comparison for index1, index2 in itertools.combinations( range(len(input_filenames_counts)), 2): E.info("comparing %i and %i" % (index1, index2)) a, b = all_annotator_results[index1], all_annotator_results[index2] # index results in a and b aa = collections.defaultdict(dict) for x in a: aa[x.track][x.annotation] = x bb = collections.defaultdict(dict) for x in b: bb[x.track][x.annotation] = x tracks_a = set(aa.keys()) tracks_b = set(bb.keys()) shared_tracks = tracks_a.intersection(tracks_b) if len(shared_tracks) == 0: E.warn("no shared tracks between {} and {}".format( index1, index2)) for track in sorted(shared_tracks): E.debug("computing results for track {}".format(track)) # get shared annotations annotations1 = aa[track].keys() annotations2 = bb[track].keys() shared_annotations = list( set(annotations1).intersection(set(annotations2))) E.info("%i shared annotations" % len(shared_annotations)) for annotation in shared_annotations: # if not annotation.startswith("Ram:"): continue data1 = aa[track][annotation] data2 = bb[track][annotation] # note that fold changes can be very large if there are 0 samples # this is fine for getting the distributional params (mean, # stddev) fold_changes1 = data1.observed / (data1.samples + pseudo_count) fold_changes2 = data2.observed / (data2.samples + pseudo_count) # add a separate fc pseudo-count to avoid 0 values fold_changes1 += 0.0001 fold_changes2 += 0.0001 # Test is if relative fold change rfc is different from 1 # note: rfc = fc1 / fc2 = obs1 / exp1 * obs2 / exp2 # = obs1 / obs2 * exp2 / exp1 # Thus, it is equivalent to test rfc = obs1/obs2 versus exp2 / exp1 # # Convert to log space for easier plotting # Move the observed fold ratio in order to get an idea of the magnitude # of the underlying fold change delta_fold = data2.fold - data1.fold sampled_delta_fold = numpy.log( fold_changes1 / fold_changes2) + delta_fold observed_delta_fold = 0.0 + delta_fold result = GatEngine.AnnotatorResult(track, annotation, "na", observed_delta_fold, sampled_delta_fold, reference=None, pseudo_count=0) results.append(result) if len(results) == 0: E.critical("no results found") E.Stop() return IO.outputResults(results, options, GatEngine.AnnotatorResult.headers, description_header, description_width, descriptions, format_observed="%6.4f") IO.plotResults(results, options) # write footer and output benchmark information. E.Stop()
def buildSegments(options): '''load segments, annotations and workspace from parameters defined in *options*. The workspace will be split by isochores. returns segments, annotations and workspace. ''' options.segment_files = expandGlobs(options.segment_files) options.annotation_files = expandGlobs(options.annotation_files) options.workspace_files = expandGlobs(options.workspace_files) options.sample_files = expandGlobs(options.sample_files) ################################################## # arguments sanity check if not options.segment_files: raise ValueError("please specify at least one segment file") if not options.annotation_files: raise ValueError("please specify at least one annotation file") if not options.workspace_files: raise ValueError("please specify at least one workspace file") # read one or more segment files segments = readSegmentList("segments", options.segment_files, options) if options.ignore_segment_tracks: segments.merge(delete=True) E.info("merged all segments into one track with %i segments" % len(segments)) if segments.sum() == 0: E.critical("no segments in input file - run aborted") raise ValueError("segments file is empty - run aborted") if len(segments) > 1000: raise ValueError( "too many (%i) segment files - use track definitions or --ignore-segment-tracks" % len(segments)) annotations = readSegmentList( "annotations", options.annotation_files, options, options.enable_split_tracks) workspaces = readSegmentList( "workspaces", options.workspace_files, options, options.enable_split_tracks) # intersect workspaces to build a single workspace E.info("collapsing workspaces") dumpStats(workspaces, "stats_workspaces_input", options) workspaces.collapse() dumpStats(workspaces, "stats_workspaces_collapsed", options) # use merged workspace only, discard others workspaces.restrict("collapsed") # build isochores or intersect annotations/segments with workspace if options.isochore_files: # read one or more isochore files isochores = GatEngine.IntervalCollection(name="isochores") E.info("%s: reading isochores from %i files" % ("isochores", len(options.isochore_files))) isochores.load(options.isochore_files) dumpStats(isochores, "stats_isochores_raw", options) # merge isochores and check if consistent (fully normalized) isochores.sort() # check that there are no overlapping segments within isochores isochores.check() # TODO: flag is_normalized not properly set isochores.normalize() # check that there are no overlapping segments between isochores # truncate isochores to workspace # crucial if isochores are larger than workspace. isochores.intersect(workspaces["collapsed"]) else: isochores = None return segments, annotations, workspaces, isochores
def main(argv=None): """script main. parses command line options in sys.argv, unless *argv* is given. """ if not argv: argv = sys.argv # setup command line parser parser = optparse.OptionParser(version="%prog version: $Id: script_template.py 2871 2010-03-03 10:20:44Z andreas $", usage=globals()["__doc__"]) parser.add_option("-o", "--order", dest="output_order", type="choice", choices=( "track", "annotation", "fold", "pvalue", "qvalue", "observed"), help="order results in output by fold, track, etc. [default=%default].") parser.add_option("-p", "--pvalue-method", dest="pvalue_method", type="choice", choices=("empirical", "norm", ), help="type of pvalue reported [default=%default].") parser.add_option("-q", "--qvalue-method", dest="qvalue_method", type="choice", choices=( "storey", "BH", "bonferroni", "holm", "hommel", "hochberg", "BY", "none"), help="method to perform multiple testing correction by controlling the fdr [default=%default].") parser.add_option("--qvalue-lambda", dest="qvalue_lambda", type="float", help="fdr computation: lambda [default=%default].") parser.add_option("--qvalue-pi0-method", dest="qvalue_pi0_method", type="choice", choices=("smoother", "bootstrap"), help="fdr computation: method for estimating pi0 [default=%default].") parser.add_option("--descriptions", dest="input_filename_descriptions", type="string", help="filename mapping annotation terms to descriptions. " " if given, the output table will contain additional columns " " [default=%default]") parser.add_option("--pseudo-count", dest="pseudo_count", type="float", help="pseudo count. The pseudo count is added to both the observed and expected overlap. " " Using a pseudo-count avoids gat reporting fold changes of 0 [default=%default].") parser.add_option("--output-plots-pattern", dest="output_plots_pattern", type="string", help="output pattern for plots [default=%default]") parser.set_defaults( pvalue_method="empirical", qvalue_method="BH", qvalue_lambda=None, qvalue_pi0_method="smoother", # pseudo count for fold change computation to avoid 0 fc pseudo_count=1.0, output_order="observed", ) # add common options (-h/--help, ...) and parse command line (options, args) = E.Start(parser, argv=argv, add_output_options=True) input_filenames_counts = args ################################################## E.info("received %i filenames with counts" % len(input_filenames_counts)) ################################################## description_header, descriptions, description_width = IO.readDescriptions( options) all_annotator_results = [] for input_filename_counts in input_filenames_counts: E.info("processing %s" % input_filename_counts) annotator_results = gat.fromCounts(input_filename_counts) ################################################## if options.pvalue_method != "empirical": E.info("updating pvalues to %s" % options.pvalue_method) GatEngine.updatePValues(annotator_results, options.pvalue_method) ################################################## ################################################## ################################################## # compute global fdr ################################################## E.info("computing FDR statistics") GatEngine.updateQValues(annotator_results, method=options.qvalue_method, vlambda=options.qvalue_lambda, pi0_method=options.qvalue_pi0_method) all_annotator_results.append(annotator_results) pseudo_count = options.pseudo_count results = [] if len(all_annotator_results) == 1: E.info("performing pairwise comparison within a file") # collect all annotations annotations, segments = list(), set() for x in all_annotator_results[0]: segments.add(x.track) annotations.append(x) if len(segments) != 1: raise NotImplementedError("multiple segments of interest") for data1, data2 in itertools.combinations(annotations, 2): # note that fold changes can be very large if there are 0 samples # this is fine for getting the distributional params (mean, stddev) fold_changes1 = data1.observed / (data1.samples + pseudo_count) fold_changes2 = data2.observed / (data2.samples + pseudo_count) # add a separate fc pseudo-count to avoid 0 values fold_changes1 += 0.0001 fold_changes2 += 0.0001 # Test is if relative fold change rfc is different from 1 # note: rfc = fc1 / fc2 = obs1 / exp1 * obs2 / exp2 # = obs1 / obs2 * exp2 / exp1 # Thus, it is equivalent to test rfc = obs1/obs2 versus exp2 / exp1 # # Convert to log space for easier plotting # Move the observed fold ratio in order to get an idea of the magnitude # of the underlying fold change delta_fold = data2.fold - data1.fold sampled_delta_fold = numpy.log( fold_changes1 / fold_changes2) + delta_fold observed_delta_fold = 0.0 + delta_fold result = GatEngine.AnnotatorResult(data1.annotation, data2.annotation, "na", observed_delta_fold, sampled_delta_fold, reference=None, pseudo_count=0) results.append(result) else: E.info("performing pairwise comparison between files") ################################################## # perform pairwise comparison for index1, index2 in itertools.combinations(range(len(input_filenames_counts)), 2): E.info("comparing %i and %i" % (index1, index2)) a, b = all_annotator_results[index1], all_annotator_results[index2] # index results in a and b aa = collections.defaultdict(dict) for x in a: aa[x.track][x.annotation] = x bb = collections.defaultdict(dict) for x in b: bb[x.track][x.annotation] = x if len(aa.keys()) != 1 or len(bb.keys()) != 1: raise NotImplementedError("multiple segments of interest") track = "merged" # get shared annotations annotations1 = aa[track].keys() annotations2 = bb[track].keys() shared_annotations = list( set(annotations1).intersection(set(annotations2))) E.info("%i shared annotations" % len(shared_annotations)) for annotation in shared_annotations: # if not annotation.startswith("Ram:"): continue data1 = aa[track][annotation] data2 = bb[track][annotation] # note that fold changes can be very large if there are 0 samples # this is fine for getting the distributional params (mean, # stddev) fold_changes1 = data1.observed / (data1.samples + pseudo_count) fold_changes2 = data2.observed / (data2.samples + pseudo_count) # add a separate fc pseudo-count to avoid 0 values fold_changes1 += 0.0001 fold_changes2 += 0.0001 # Test is if relative fold change rfc is different from 1 # note: rfc = fc1 / fc2 = obs1 / exp1 * obs2 / exp2 # = obs1 / obs2 * exp2 / exp1 # Thus, it is equivalent to test rfc = obs1/obs2 versus exp2 / exp1 # # Convert to log space for easier plotting # Move the observed fold ratio in order to get an idea of the magnitude # of the underlying fold change delta_fold = data2.fold - data1.fold sampled_delta_fold = numpy.log( fold_changes1 / fold_changes2) + delta_fold observed_delta_fold = 0.0 + delta_fold result = GatEngine.AnnotatorResult(track, annotation, "na", observed_delta_fold, sampled_delta_fold, reference=None, pseudo_count=0) results.append(result) if len(results) == 0: E.critical("no results found") E.Stop() return IO.outputResults(results, options, GatEngine.AnnotatorResult.headers, description_header, description_width, descriptions, format_observed="%6.4f") IO.plotResults(results, options) # write footer and output benchmark information. E.Stop()
def applyIsochores( segments, annotations, workspaces, options, isochores=None, truncate_segments_to_workspace=False, truncate_workspace_to_annotations=False, restrict_workspace=False, ): '''apply isochores to segments and annotations. Segments and annotations are filtered in place to keep only those overlapping the workspace. If *isochores* are given, isochores are applied. If *truncate_segments_to_workspace*, truncate segments to workspace. If *restrict_workspace* is set, the workspace is confined to those parts that overlap both a segment and an annotation. If *truncate_workspace_to_annotations* is set, the workspace is truncated to keep only those parts that overlap annotations. returns a workspace divided into isochores. ''' if isochores: # intersect isochores and workspaces, segments and annotations # workspace and annotations are truncated # with segments it is optional. E.info("adding isochores to workspace") workspaces.toIsochores(isochores, truncate=True) annotations.toIsochores(isochores, truncate=True) segments.toIsochores(isochores, truncate=options.truncate_segments_to_workspace) if workspaces.sum() == 0: raise ValueError("isochores and workspaces do not overlap") if annotations.sum() == 0: raise ValueError("isochores and annotations do not overlap") if segments.sum() == 0: raise ValueError("isochores and segments do not overlap") dumpStats(workspaces, "stats_workspaces_isochores", options) dumpStats(annotations, "stats_annotations_isochores", options) dumpStats(segments, "stats_segments_isochores", options) dumpBed(workspaces, "workspaces_isochores", options) dumpBed(annotations, "annotations_isochores", options) dumpBed(segments, "segments_isochores", options) else: # intersect workspace and segments/annotations # annotations and segments are truncated by workspace if options.truncate_segments_to_workspace: segments.intersect(workspaces["collapsed"]) else: segments.filter(workspaces["collapsed"]) annotations.intersect(workspaces["collapsed"]) dumpStats(annotations, "stats_annotations_truncated", options) dumpStats(segments, "stats_segments_truncated", options) workspace = workspaces["collapsed"] if restrict_workspace: E.info("restricting workspace") # this is very cumbersome - refactor merge and collapse # to return an IntervalDictionary instead of adding it # to the list of tracks for x in (segments, annotations): if "merged" in segments: workspace.filter(segments["merged"]) else: segments.merge() workspace.filter(segments["merged"]) del segments["merged"] dumpStats(workspaces, "stats_workspaces_restricted", options) if truncate_workspace_to_annotations: E.info("truncating workspace to annotations") annotations.merge() annotations["merged"].normalize() workspace.intersect(annotations["merged"]) del annotations["merged"] dumpStats(workspaces, "stats_workspaces_truncated", options) # segments.dump( open("segments_dump.bed", "w" ) ) # workspaces.dump( open("workspaces_dump.bed", "w" ) ) # output overlap stats # output segment densities per workspace if "overlap" in options.output_stats or \ "all" in options.output_stats: for track in segments.tracks: workspaces.outputOverlapStats( E.openOutputFile("overlap_%s" % track), segments[track]) return workspace
def computeSample(args): '''compute a single sample. ''' workdata, samples_outfile, metrics_outfile, lock = args (track, sample_id, sampler, segs, annotations, contig_annotations, workspace, contig_workspace, counters) = workdata # E.debug("track=%s, sample=%s - started" % (track, str(sample_id))) counts = E.Counter() sample_id = str(sample_id) outf_samples = samples_outfile if samples_outfile: if lock: lock.acquire() outf_samples = IOTools.openFile(samples_outfile, "a") samples_outfile.write("track name=%s\n" % sample_id) if lock: outf_samples.close() lock.release() sample = Engine.IntervalDictionary() for isochore in list(segs.keys()): counts.pairs += 1 # skip empty isochores if workspace[isochore].isEmpty or segs[isochore].isEmpty: counts.skipped += 1 continue counts.sampled += 1 r = sampler.sample(segs[isochore], workspace[isochore]) # TODO : activate # self.outputSampleStats( sample_id, isochore, r ) sample.add(isochore, r) # save sample if samples_outfile: if lock: lock.acquire() outf_samples = IOTools.openFile(samples_outfile, "a") for start, end in r: outf_samples.write("%s\t%i\t%i\n" % (isochore, start, end)) if lock: outf_samples.close() lock.release() # re-combine isochores # adjacent intervals are merged. sample.fromIsochores() if metrics_outfile: if lock: lock.acquire() outf = IOTools.openFile(metrics_outfile, "a") else: outf = metrics_outfile IO.outputMetrics(outf, sample, workspace, track, sample_id) if lock: outf.close() lock.release() counts_per_track = [collections.defaultdict(float) for x in counters] # compute counts for each counter for counter_id, counter in enumerate(counters): # TODO: choose aggregator for annotation in annotations.tracks: counts_per_track[counter_id][annotation] = sum([ counter(sample[contig], contig_annotations[annotation][contig], contig_workspace[contig]) for contig in list(sample.keys())]) # E.debug("track=%s, sample=%s - completed" % (track,str(sample_id ))) return counts_per_track
def dumpStats(coll, section, options): if section in options.output_stats or \ "all" in options.output_stats or \ len([x for x in options.output_stats if re.search(x, section)]) > 0: coll.outputStats(E.openOutputFile(section))
def dumpBed(coll, section, options): if section in options.output_bed or \ "all" in options.output_bed or \ len([x for x in options.output_bed if re.search(x, section)]) > 0: coll.save(E.openOutputFile(section + ".bed"))
def main(argv=None): """script main. parses command line options in sys.argv, unless *argv* is given. """ if not argv: argv = sys.argv parser = gat.buildParser(usage=globals()["__doc__"]) # add common options (-h/--help, ...) and parse command line (options, args) = E.Start(parser, argv=argv, add_output_options=True) ################################################## description_header, descriptions, description_width = IO.readDescriptions( options) ################################################## size_pos, size_segment = GatSegmentList.getSegmentSize() E.debug("sizes: pos=%i segment=%i, max_coord=%i" % (size_pos, size_segment, 2 ** (8 * size_pos))) ################################################## # set default counter if not options.counters: options.counters.append("nucleotide-overlap") ################################################## if options.output_tables_pattern != None: if "%s" not in options.output_tables_pattern: raise ValueError( "output_tables_pattern should contain at least one '%s'") if options.output_samples_pattern != None: if "%s" not in options.output_samples_pattern: raise ValueError( "output_samples_pattern should contain at least one '%s'") if options.output_counts_pattern != None: if "%s" not in options.output_counts_pattern: raise ValueError( "output_counts_pattern should contain at least one '%s'") ################################################## # read fold changes that results should be compared with if options.null != "default": if not os.path.exists(options.null): raise OSError("file %s not found" % options.null) E.info("reading reference results from %s" % options.null) options.reference = IO.readAnnotatorResults(options.null) else: options.reference = None if options.input_filename_counts: # use pre-computed counts annotator_results = GatEngine.fromCounts(options.input_filename_counts) elif options.input_filename_results: # use previous results (re-computes fdr) E.info("reading gat results from %s" % options.input_filename_results) annotator_results = IO.readAnnotatorResults( options.input_filename_results) else: # do full gat analysis annotator_results = fromSegments(options, args) ################################################## if options.pvalue_method != "empirical": E.info("updating pvalues to %s" % options.pvalue_method) GatEngine.updatePValues(annotator_results, options.pvalue_method) ################################################## # output IO.outputResults(annotator_results, options, GatEngine.AnnotatorResultExtended.headers, description_header, description_width, descriptions) IO.plotResults(annotator_results, options) # write footer and output benchmark information. E.Stop()
def sample(self, track, counts, counters, segs, annotations, workspace, outfiles): '''conditional sampling - sample using only those segments that contain both a segment and an annotation. return dictionary with counts per track ''' E.info("performing conditional sampling") counts_per_track = [collections.defaultdict(list) for x in counters] # rebuild non-isochore annotations and workspace contig_annotations = annotations.clone() contig_annotations.fromIsochores() contig_annotations.setName("contig_" + annotations.getName()) contig_workspace = workspace.clone() contig_workspace.fromIsochores() E.info("setting up shared data for multi-processing") annotations.share() contig_annotations.share() contig_workspace.share("contig_workspace") E.info("workspace without conditioning: %i segments, %i nucleotides" % (workspace.counts(), workspace.sum())) if workspace.sum() == 0: E.warn("empty workspace - no computation performed") return None # compute samples conditionally - need to proceed by annotation for annoid, annotation in enumerate(annotations.tracks): annos = annotations[annotation] temp_segs, temp_annotations, temp_workspace = \ self.workspace_generator(segs, annos, workspace) # set up sharing temp_segs.share("generated_segments") temp_workspace.share("generated_workspace") E.info("workspace for annotation %s: %i segments, %i nucleotides" % (annotation, temp_workspace.counts(), temp_workspace.sum())) work = [WorkData('_'.join((track, annoid)), x, self.sampler, temp_segs, annotations, contig_annotations, temp_workspace, contig_workspace, counters, ) for x in range(self.num_samples)] E.info("sampling for annotation '%s' started" % annotation) results = self.computeSamples(work) E.info("sampling for annotation '%s' completed" % annotation) for result in results: for counter_id, counter in enumerate(counters): counts_per_track[counter_id][annotation].append( result[counter_id][annotation]) return counts_per_track
def fromSegments(options, args): '''run analysis from segment files. This is the most common use case. ''' tstart = time.time() # build segments segments, annotations, workspaces, isochores = IO.buildSegments(options) E.info("intervals loaded in %i seconds" % (time.time() - tstart)) # open various additional output files outfiles = {} for section in ( "sample", "segment_metrics", "sample_metrics", ): if section in options.output_stats or \ "all" in options.output_stats or \ len([x for x in options.output_stats if re.search(x, "section")]) > 0: outfiles[section] = E.openOutputFile(section) if 'sample_metrics' in outfiles: outfiles['sample_metrics'].write( "track\tsection\tmetric\t%s\n" % "\t".join(Stats.Summary().getHeaders())) # filter segments by workspace workspace = IO.applyIsochores( segments, annotations, workspaces, options, isochores, truncate_segments_to_workspace=options.truncate_segments_to_workspace, truncate_workspace_to_annotations=options. truncate_workspace_to_annotations, restrict_workspace=options.restrict_workspace) # check memory requirements # previous algorithm: memory requirements if all samples are stored # counts = segments.countsPerTrack() # max_counts = max(counts.values()) # memory = 8 * 2 * options.num_samples * max_counts * len(workspace) # initialize sampler if options.sampler == "annotator": sampler = Engine.SamplerAnnotator(bucket_size=options.bucket_size, nbuckets=options.nbuckets) elif options.sampler == "shift": sampler = Engine.SamplerShift(radius=options.shift_expansion, extension=options.shift_extension) elif options.sampler == "segments": sampler = Engine.SamplerSegments() elif options.sampler == "local-permutation": sampler = Engine.SamplerLocalPermutation() elif options.sampler == "global-permutation": sampler = Engine.SamplerGlobalPermutation() elif options.sampler == "brute-force": sampler = Engine.SamplerBruteForce() elif options.sampler == "uniform": sampler = Engine.SamplerUniform() # initialize counter counters = [] for counter in options.counters: if counter == "nucleotide-overlap": counters.append(Engine.CounterNucleotideOverlap()) elif counter == "nucleotide-density": counters.append(Engine.CounterNucleotideDensity()) elif counter == "segment-overlap": counters.append(Engine.CounterSegmentOverlap()) elif counter == "annotation-overlap": counters.append(Engine.CounterAnnotationOverlap()) elif counter == "segment-midoverlap": counters.append(Engine.CounterSegmentMidpointOverlap()) elif counter == "annotation-midoverlap": counters.append(Engine.CounterAnnotationMidpointOverlap()) else: raise ValueError("unknown counter '%s'" % counter) # initialize workspace generator if options.conditional == "unconditional": workspace_generator = Engine.UnconditionalWorkspace() elif options.conditional == "cooccurance": workspace_generator = Engine.ConditionalWorkspaceCooccurance() elif options.conditional == "annotation-centered": if options.conditional_expansion is None: raise ValueError( "please specify either --conditional-expansion or " "--conditional-extension") workspace_generator = Engine.ConditionalWorkspaceAnnotationCentered( options.conditional_extension, options.conditional_expansion) elif options.conditional == "segment-centered": if options.conditional_expansion is None: raise ValueError( "please specify either --conditional-expansion or " "--conditional-extension") workspace_generator = Engine.ConditionalWorkspaceSegmentCentered( options.conditional_extension, options.conditional_expansion) else: raise ValueError("unknown conditional workspace '%s'" % options.conditional) # check if reference is compplete if options.reference: for track in segments.tracks: if track not in options.reference: raise ValueError("missing track '%s' in reference" % track) r = options.reference[track] for annotation in annotations.tracks: if annotation not in r: raise ValueError( "missing annotation '%s' in annotations for " "track='%s'" % (annotation, track)) # compute annotator_results = gat.run( segments, annotations, workspace, sampler, counters, workspace_generator=workspace_generator, num_samples=options.num_samples, cache=options.cache, outfiles=outfiles, output_counts_pattern=options.output_counts_pattern, output_samples_pattern=options.output_samples_pattern, sample_files=options.sample_files, conditional=options.conditional, conditional_extension=options.conditional_extension, reference=options.reference, pseudo_count=options.pseudo_count, num_threads=options.num_threads) return annotator_results
def main(argv): if not argv: argv = sys.argv # setup command line parser parser = optparse.OptionParser(version="%prog version: $Id: script_template.py 2871 2010-03-03 10:20:44Z andreas $", usage=globals()["__doc__"]) parser.add_option("-a", "--annotation-file", "--annotations", dest="annotation_files", type="string", action="append", help="filename with annotations [default=%default].") parser.add_option("-s", "--segment-file", "--segments", dest="segment_files", type="string", action="append", help="filename with segments. Also accepts a glob in parentheses [default=%default].") parser.add_option("-w", "--workspace-file", "--workspace", dest="workspace_files", type="string", action="append", help="filename with workspace segments. Also accepts a glob in parentheses [default=%default].") parser.add_option("-i", "--isochore-file", "--isochores", dest="isochore_files", type="string", action="append", help="filename with isochore segments. Also accepts a glob in parentheses [default=%default].") parser.add_option("-o", "--order", dest="output_order", type="choice", choices=( "track", "annotation", "fold", "pvalue", "qvalue"), help="order results in output by fold, track, etc. [default=%default].") parser.add_option("-q", "--qvalue-method", dest="qvalue_method", type="choice", choices=( "storey", "BH", "bonferroni", "holm", "hommel", "hochberg", "BY", "none"), help="method to perform multiple testing correction by controlling the fdr [default=%default].") parser.add_option("--qvalue-lambda", dest="qvalue_lambda", type="float", help="fdr computation: lambda [default=%default].") parser.add_option("--qvalue-pi0-method", dest="qvalue_pi0_method", type="choice", choices=("smoother", "bootstrap"), help="fdr computation: method for estimating pi0 [default=%default].") parser.add_option("--descriptions", dest="input_filename_descriptions", type="string", help="filename mapping annotation terms to descriptions. " " if given, the output table will contain additional columns " " [default=%default]") parser.add_option("--ignore-segment-tracks", dest="ignore_segment_tracks", action="store_true", help="ignore segment tracks - all segments belong to one track [default=%default]") parser.add_option("--enable-split-tracks", dest="enable_split_tracks", action="store_true", help="permit the same track to be in multiple files [default=%default]") parser.add_option("--output-bed", dest="output_bed", type="choice", action="append", choices=("all", "annotations", "segments", "workspaces", "isochores", "overlap"), help="output bed files [default=%default].") parser.add_option("--output-stats", dest="output_stats", type="choice", action="append", choices=("all", "annotations", "segments", "workspaces", "isochores", "overlap"), help="output overlap summary stats [default=%default].") parser.add_option("--restrict-workspace", dest="restrict_workspace", action="store_true", help="restrict workspace to those segments that contain both track" " and annotations [default=%default]") parser.add_option("--counter", dest="counters", type="choice", action="append", choices=("binom", "hyperg"), help="counter to use [default=%default].") parser.add_option("--output-tables-pattern", dest="output_tables_pattern", type="string", help="output pattern for result tables. Used if there are multiple counters used [default=%default].") parser.set_defaults( annotation_files=[], segment_files=[], workspace_files=[], sample_files=[], counters=[], output_stats=[], output_bed=[], output_tables_pattern="%s.tsv.gz", output_order="fold", input_filename_counts=None, input_filename_results=None, pvalue_method="empirical", output_plots_pattern=None, output_samples_pattern=None, qvalue_method="storey", qvalue_lambda=None, qvalue_pi0_method="smoother", ignore_segment_tracks=False, input_filename_descriptions=None, conditional="unconditional", conditional_extension=None, conditional_expansion=None, restrict_workspace=False, enable_split_tracks=False, shift_expansion=2.0, shift_extension=0, overlap_mode="midpoint", truncate_workspace_to_annotations=False, truncate_segments_to_workspace=False ) # add common options (-h/--help, ...) and parse command line (options, args) = E.Start(parser, argv=argv, add_output_options=True) tstart = time.time() if len(options.counters) == 0: options.counters.append("binom") ############################################ segments, annotations, workspaces, isochores = IO.buildSegments(options) E.info("intervals loaded in %i seconds" % (time.time() - tstart)) # filter segments by workspace workspace = IO.applyIsochores( segments, annotations, workspaces, options, isochores) ############################################ description_header, descriptions, description_width = IO.readDescriptions( options) ############################################ ############################################ # compute per contig # compute bases covered by workspace workspace2basecoverage, isochores = {}, [] for contig, ww in workspace.iteritems(): workspace2basecoverage[contig] = ww.sum() isochores.append(contig) # compute percentage of bases covered by annotations in workspace # per isochore annotation2basecoverage = collections.defaultdict(dict) for annotation, aa in annotations.iteritems(): for isochore, a in aa.iteritems(): # need to truncate to workspace? annotation2basecoverage[annotation][isochore] = a.sum() results_per_contig = collections.defaultdict(list) E.info("computing counts per isochore") # results per isochore def emptyResult(segment, annotation, isochore, counter, nsegments_in_workspace, basecoverage_annotation, basecoverage_workspace): return GREAT_RESULT._make(( segment, annotation, isochore, counter, 0, # observed 0, # expected nsegments_in_workspace, 0, # nannotations_in_workspace 0, # nsegments_overlapping_annotation 0, # nannotations_overlapping_segments 0, # basecoverage_intersection 0, # basecoverage_segments basecoverage_annotation, basecoverage_workspace, 0.0, 1.0, 1.0, 1.0)) for isochore in isochores: basecoverage_workspace = workspace2basecoverage[isochore] # iterate over all isochores for segment, segmentdict in segments.iteritems(): try: ss = segmentdict[isochore] # select segments overlapping workspace segments_in_workspace = GatSegmentList.SegmentList(clone=ss) segments_in_workspace.intersect(workspace[isochore]) # number of segments in workspace nsegments_in_workspace = len(segments_in_workspace) except KeyError: ss = None basecoverage_segments = segments_in_workspace.sum() for annotation, annotationdict in annotations.iteritems(): # if annotation != "GO:0030957": continue try: aa = annotationdict[isochore] except KeyError: aa = None # p_A: proportion of bases covered by annotation try: basecoverage_annotation = annotation2basecoverage[ annotation][isochore] except KeyError: basecoverage_annotation = 0 if ss == None or aa == None: for counter in options.counters: results_per_contig[(counter, segment, annotation)].append(emptyResult(segment, annotation, isochore, counter, nsegments_in_workspace, basecoverage_annotation, basecoverage_workspace)) continue # select segments overlapping annotation segments_overlapping_annotation = GatSegmentList.SegmentList( clone=ss) segments_overlapping_annotation.intersect( annotations[annotation][isochore]) # number of segments in annotation nsegments_overlapping_annotation = ss.intersectionWithSegments(annotations[annotation][isochore], mode=options.overlap_mode) # number of nucleotides at the intersection of segments, # annotation and workspace basecoverage_intersection = segments_overlapping_annotation.sum() annotations_overlapping_segments = GatSegmentList.SegmentList( clone=aa) annotations_overlapping_segments.intersect(ss) nannotations_overlapping_segments = len( annotations_overlapping_segments) nannotations_in_workspace = len(aa) if nannotations_in_workspace == 0: for counter in options.counters: results_per_contig[(counter, segment, annotation)].append(emptyResult(segment, annotation, isochore, counter, nsegments_in_workspace, basecoverage_annotation, basecoverage_workspace)) continue fraction_coverage_annotation = basecoverage_annotation / \ float(basecoverage_workspace) fraction_hit_annotation = float( nannotations_overlapping_segments) / nannotations_in_workspace for counter in options.counters: if counter.startswith("binom"): # GREAT binomial probability over "regions" # n = number of genomic regions = nannotations_in_workspace # ppi = fraction of genome annotated by annotation = fraction_coverage_annotation # kpi = genomic regions with annotation hit by segments = nannotations_in_segments # sf = survival functions = 1 -cdf # probability of observing >kpi in a sample of n where the probabily of succes is # ppi. pvalue = scipy.stats.binom.sf(nsegments_overlapping_annotation - 1, nsegments_in_workspace, fraction_coverage_annotation) expected = fraction_coverage_annotation * \ nsegments_in_workspace observed = nsegments_overlapping_annotation elif counter.startswith("hyperg"): # hypergeometric probability over nucleotides # Sampling without replacement # x,M,n,M # x = observed number of nucleotides in overlap of segments,annotations and workspace # M = number of nucleotides in workspace # n = number of nucleotides in annotations (and workspace) # N = number of nucleotides in segments (and workspace) # P-value of obtaining >x number of nucleotides # overlapping. rv = scipy.stats.hypergeom(basecoverage_workspace, basecoverage_annotation, basecoverage_segments) pvalue = rv.sf(basecoverage_intersection) expected = rv.mean() observed = basecoverage_intersection if expected != 0: fold = float(observed) / expected else: fold = 1.0 r = GREAT_RESULT._make(( segment, annotation, isochore, counter, observed, expected, nsegments_in_workspace, nannotations_in_workspace, nsegments_overlapping_annotation, nannotations_overlapping_segments, basecoverage_intersection, basecoverage_segments, basecoverage_annotation, basecoverage_workspace, fraction_coverage_annotation, fold, pvalue, 1.0)) # print "\t".join( map(str, r)) results_per_contig[ (counter, segment, annotation)].append(r) E.info("merging counts per isochore") # compute sums results = [] for niteration, pair in enumerate(results_per_contig.iteritems()): counter, segment, annotation = pair[0] data = pair[1] nsegments_in_workspace = sum([x.nsegments_in_workspace for x in data]) nsegments_overlapping_annotation = sum([x.observed for x in data]) nannotations_in_workspace = sum( [x.nannotations_in_workspace for x in data]) nannotations_overlapping_segments = sum( [x.nannotations_overlapping_segments for x in data]) basecoverage_intersection = sum( [x.basecoverage_intersection for x in data]) basecoverage_segments = sum([x.basecoverage_segments for x in data]) basecoverage_annotation = sum( [x.basecoverage_annotation for x in data]) basecoverage_workspace = sum([x.basecoverage_workspace for x in data]) fraction_coverage_annotation = basecoverage_annotation / \ float(basecoverage_workspace) if counter.startswith("binom"): pvalue = scipy.stats.binom.sf(nsegments_overlapping_annotation - 1, nsegments_in_workspace, fraction_coverage_annotation) expected = fraction_coverage_annotation * nsegments_in_workspace observed = nsegments_overlapping_annotation elif counter.startswith("hyperg"): rv = scipy.stats.hypergeom(basecoverage_workspace, basecoverage_annotation, basecoverage_segments) pvalue = rv.sf(basecoverage_intersection) expected = rv.mean() observed = basecoverage_intersection if expected != 0: fold = float(observed) / expected else: fold = 1.0 r = GREAT_RESULT._make(( segment, annotation, "all", counter, observed, expected, nsegments_in_workspace, nannotations_in_workspace, nsegments_overlapping_annotation, nannotations_overlapping_segments, basecoverage_intersection, basecoverage_segments, basecoverage_annotation, basecoverage_workspace, fraction_coverage_annotation, fold, pvalue, 1.0)) results.append(r) IO.outputResults(results, options, GREAT_RESULT._fields, description_header, description_width, descriptions) E.Stop()
def main(argv=None): """script main. parses command line options in sys.argv, unless *argv* is given. """ if not argv: argv = sys.argv # setup command line parser parser = optparse.OptionParser(version="%prog version: $Id: script_template.py 2871 2010-03-03 10:20:44Z andreas $", usage=globals()["__doc__"]) parser.add_option("-l", "--sample-file", dest="sample_files", type="string", action="append", help="filename with sample files. Start processing from samples [default=%default].") parser.add_option("-o", "--order", dest="output_order", type="choice", choices=( "track", "annotation", "fold", "pvalue", "qvalue"), help="order results in output by fold, track, etc. [default=%default].") parser.add_option("-p", "--pvalue-method", dest="pvalue_method", type="choice", choices=("empirical", "norm", ), help="type of pvalue reported [default=%default].") parser.add_option("--results-file", dest="input_filename_results", type="string", help="start processing from results - no segments required [default=%default].") parser.add_option("--output-plots-pattern", dest="output_plots_pattern", type="string", help="output pattern for plots [default=%default]") parser.add_option("--output-samples-pattern", dest="output_samples_pattern", type="string", help="output pattern for samples. Samples are stored in bed format, one for " " each segment [default=%default]") parser.add_option("--plots", dest="plots", type="choice", choices=("all", "bars-per-track", "bars", ), help="plots to be created [default=%default].") parser.set_defaults( sample_files=[], num_samples=1000, output_stats=[], output_filename_counts=None, output_order="fold", input_filename_results=None, pvalue_method="empirical", output_plots_pattern=None, plots=[], ) # add common options (-h/--help, ...) and parse command line (options, args) = E.Start(parser, argv=argv, add_output_options=True) annotator_results = IO.readAnnotatorResults(options.input_filename_results) if "speparate-bars" in options.plots: plotBarplots(annotator_results, options) if "bars" in options.plots: plotBarplot(annotator_results, options) # write footer and output benchmark information. E.Stop()
def fromSegments(options, args): '''run analysis from segment files. This is the most common use case. ''' tstart = time.time() ################################################## ################################################## ################################################## # build segments segments, annotations, workspaces, isochores = IO.buildSegments(options) E.info("intervals loaded in %i seconds" % (time.time() - tstart)) ################################################## ################################################## ################################################## # open various additional output files ################################################## outfiles = {} for section in ("sample", "segment_metrics", "sample_metrics", ): if section in options.output_stats or \ "all" in options.output_stats or \ len([x for x in options.output_stats if re.search(x, "section")]) > 0: outfiles[section] = E.openOutputFile(section) if 'sample_metrics' in outfiles: outfiles['sample_metrics'].write( "track\tsection\tmetric\t%s\n" % "\t".join(Stats.Summary().getHeaders())) # filter segments by workspace workspace = IO.applyIsochores( segments, annotations, workspaces, options, isochores, truncate_segments_to_workspace=options.truncate_segments_to_workspace, truncate_workspace_to_annotations=options.truncate_workspace_to_annotations, restrict_workspace=options.restrict_workspace) ################################################## ################################################## ################################################## # check memory requirements counts = segments.countsPerTrack() max_counts = max(counts.values()) # previous algorithm: memory requirements if all samples are stored memory = 8 * 2 * options.num_samples * max_counts * len(workspace) ################################################## ################################################## ################################################## # initialize sampler if options.sampler == "annotator": sampler = GatEngine.SamplerAnnotator( bucket_size=options.bucket_size, nbuckets=options.nbuckets) elif options.sampler == "shift": sampler = GatEngine.SamplerShift( radius=options.shift_expansion, extension=options.shift_extension) elif options.sampler == "segments": sampler = GatEngine.SamplerSegments() elif options.sampler == "local-permutation": sampler = GatEngine.SamplerLocalPermutation() elif options.sampler == "global-permutation": sampler = GatEngine.SamplerGlobalPermutation() elif options.sampler == "brute-force": sampler = GatEngine.SamplerBruteForce() elif options.sampler == "uniform": sampler = GatEngine.SamplerUniform() ################################################## ################################################## ################################################## # initialize counter counters = [] for counter in options.counters: if counter == "nucleotide-overlap": counters.append(GatEngine.CounterNucleotideOverlap()) elif counter == "nucleotide-density": counters.append(GatEngine.CounterNucleotideDensity()) elif counter == "segment-overlap": counters.append(GatEngine.CounterSegmentOverlap()) elif counter == "annotations-overlap": counters.append(GatEngine.CounterAnnotationsOverlap()) elif counter == "segment-midoverlap": counters.append(GatEngine.CounterSegmentMidpointOverlap()) elif counter == "annotations-midoverlap": counters.append(GatEngine.CounterAnnotationsMidpointOverlap()) else: raise ValueError("unknown counter '%s'" % counter) ################################################## ################################################## ################################################## # initialize workspace generator if options.conditional == "unconditional": workspace_generator = GatEngine.UnconditionalWorkspace() elif options.conditional == "cooccurance": workspace_generator = GatEngine.ConditionalWorkspaceCooccurance() elif options.conditional == "annotation-centered": if options.conditional_extension == options.conditional_expansion is None: raise ValueError( "please specify either --conditional-expansion or " "--conditional-extension") workspace_generator = GatEngine.ConditionalWorkspaceAnnotationCentered( options.conditional_extension, options.conditional_expansion) elif options.conditional == "segment-centered": if options.conditional_extension == options.conditional_expansion is None: raise ValueError( "please specify either --conditional-expansion or " "--conditional-extension") workspace_generator = GatEngine.ConditionalWorkspaceSegmentCentered( options.conditional_extension, options.conditional_expansion) else: raise ValueError("unknown conditional workspace '%s'" % options.conditional) ################################################## ################################################## ################################################## # check if reference is compplete ################################################## if options.reference: for track in segments.tracks: if track not in options.reference: raise ValueError("missing track '%s' in reference" % track) r = options.reference[track] for annotation in annotations.tracks: if annotation not in r: raise ValueError( "missing annotation '%s' in annotations for " "track='%s'" % (annotation, track)) ################################################## ################################################## ################################################## # compute ################################################## annotator_results = gat.run( segments, annotations, workspace, sampler, counters, workspace_generator=workspace_generator, num_samples=options.num_samples, cache=options.cache, outfiles=outfiles, output_counts_pattern=options.output_counts_pattern, output_samples_pattern=options.output_samples_pattern, sample_files=options.sample_files, conditional=options.conditional, conditional_extension=options.conditional_extension, reference=options.reference, pseudo_count=options.pseudo_count, num_threads=options.num_threads) return annotator_results
def main(argv=None): """script main. parses command line options in sys.argv, unless *argv* is given. """ if not argv: argv = sys.argv parser = gat.buildParser(usage=globals()["__doc__"]) # add common options (-h/--help, ...) and parse command line (options, args) = E.Start(parser, argv=argv, add_output_options=True) ################################################## description_header, descriptions, description_width = IO.readDescriptions( options) ################################################## size_pos, size_segment = SegmentList.getSegmentSize() E.debug("sizes: pos=%i segment=%i, max_coord=%i" % (size_pos, size_segment, 2**(8 * size_pos))) ################################################## # set default counter if not options.counters: options.counters.append("nucleotide-overlap") ################################################## if options.output_tables_pattern is not None: if "%s" not in options.output_tables_pattern: raise ValueError( "output_tables_pattern should contain at least one '%s'") if options.output_samples_pattern is not None: if "%s" not in options.output_samples_pattern: raise ValueError( "output_samples_pattern should contain at least one '%s'") if options.output_counts_pattern is not None: if "%s" not in options.output_counts_pattern: raise ValueError( "output_counts_pattern should contain at least one '%s'") if options.random_seed is not None: # initialize python random number generator random.seed(options.random_seed) # initialize numpy random number generator numpy.random.seed(options.random_seed) ################################################## # read fold changes that results should be compared with if options.null != "default": if not os.path.exists(options.null): raise OSError("file %s not found" % options.null) E.info("reading reference results from %s" % options.null) options.reference = IO.readAnnotatorResults(options.null) else: options.reference = None if options.input_filename_counts: # use pre-computed counts annotator_results = Engine.fromCounts(options.input_filename_counts) elif options.input_filename_results: # use previous results (re-computes fdr) E.info("reading gat results from %s" % options.input_filename_results) annotator_results = IO.readAnnotatorResults( options.input_filename_results) else: # do full gat analysis annotator_results = fromSegments(options, args) ################################################## if options.pvalue_method != "empirical": E.info("updating pvalues to %s" % options.pvalue_method) Engine.updatePValues(annotator_results, options.pvalue_method) ################################################## # output IO.outputResults(annotator_results, options, Engine.AnnotatorResultExtended.headers, description_header, description_width, descriptions) IO.plotResults(annotator_results, options) # write footer and output benchmark information. E.Stop()