def test_group_regions_make_windows_finds_masks(): crossmap = GenomeHash(_MASKS) for flank_up, flank_down in _FLANKS: for test_name, test_group in _DO_GENERATE_MAX_WINDOW.items(): result_group = _DO_GENERATE_MAX_WINDOW_RESULTS_MASKED[ "%s_%s_%s" % (test_name, flank_up, flank_down)] yield check_maximal_window, test_name, crossmap, test_group, [ result_group ], flank_up, flank_down
def test_genomehash_create_from_list(self): # """Test creation `GenomeHash`es from lists without loss of information""" gh = GenomeHash(self.transcripts,do_copy=True) found = sorted(list(gh.feature_dict.values()),key=_name_sort) expected = sorted(list(self.tx_dict.values()),key=_name_sort) self.assertEquals(len(expected),len(found),"Features lost in creation of GenomeHash from list. Expected %s, found %s." % (len(expected),len(found))) self.assertEquals(expected, found, "Features lost in creation of GenomeHash from list:\nFirst:\n%s\nSecond:\n%s" % (expected,found))
def setUpClass(cls): """Set up test data for `TestGenomeHash`""" cls.binsize = 10000 cls.transcripts = list(BED_Reader(CommentReader(open(REF_FILES["100transcripts_bed"])),return_type=Transcript)) cls.coding_regions = list(BED_Reader(CommentReader(open(REF_FILES["100cds_bed"])),return_type=Transcript)) cls.coding_antisense = list(BED_Reader(CommentReader(open(REF_FILES["100cds_antisense_bed"])),return_type=Transcript)) cls.tx_dict = { X.get_name() : X for X in cls.transcripts } cls.cds_dict = { X.get_name() : X for X in cls.coding_regions } cls.as_cds_dict = { X.get_name() : X for X in cls.coding_antisense } cls.tx_hash = GenomeHash(cls.tx_dict,do_copy=False,binsize=cls.binsize) cls.cds_hash = GenomeHash(cls.cds_dict,do_copy=False,binsize=cls.binsize) cls.as_cds_hash = GenomeHash(cls.as_cds_dict,do_copy=False,binsize=cls.binsize) cls.shuffled_indices = list(range(len(cls.transcripts))) shuffle(cls.shuffled_indices)
def test_group_regions_make_windows_multiple_genes(): empty_hash = GenomeHash([]) flank_up = 50 flank_down = 100 for test_name, test_group in _DO_GENERATE_MULTI_GENE.items(): result_groups = _DO_GENERATE_MULTI_GENE_RESULTS["%s_%s_%s" % (test_name, flank_up, flank_down)] yield check_maximal_window, test_name, empty_hash, test_group, result_groups, flank_up, flank_down
def test_genomehash_create_from_dict(self): # """Test creation `GenomeHash`es from dictionaries without loss of information""" gh = GenomeHash(self.tx_dict,do_copy=True) found = sorted(list(gh.feature_dict.values()),key=_name_sort) expected = sorted(list(self.tx_dict.values()),key=_name_sort) self.assertEquals(len(expected),len(found),"Features lost in creation of GenomeHash from dict. Expected %s, found %s." % (len(expected),len(found))) self.assertEquals(sorted(list(gh.feature_dict.values()),key=_name_sort), sorted(list(self.tx_dict.values()),key=_name_sort), "Features lost in creation of GenomeHash from dict")
def test_genomehash_update_from_dict(self): # """Test addition to a `GenomeHash` from a dictionary without loss of features # # 1. Add features to an empty dictionary # # 2. Add features to a non-empty dictionary # """ tuple_sort = lambda x: _name_sort(x[1]) gh = GenomeHash({},do_copy=True) tx_items = sorted(list(self.tx_dict.items()),key=tuple_sort) tx_values = [X[1] for X in tx_items] dict1 = dict(tx_items[:50]) dict2 = dict(tx_items[50:]) self.assertEqual(len(dict1),50) self.assertEqual(len(dict2),50) # check values, as opposed to items, because keys of feature_dict # are unique numerical IDs as opposed to ames gh.update(dict1) self.assertEquals(sorted(list(gh.feature_dict.values()),key=_name_sort), tx_values[:50], "Features lost in first update of empty GenomeHash from dict") gh.update(dict2) self.assertEquals(sorted(list(gh.feature_dict.values()),key=_name_sort), tx_values, "Features lost in second update of non-empty GenomeHash from dict")
def test_group_regions_make_windows_finds_maximal_window(): # test case, 3 transcripts same start # test case, 3 transcripts, different starts # test case, 3 transcripts, 2 share start, 1 different # test case including ncRNAs # for all, with and without changed upstream, downstream flanks empty_hash = GenomeHash([]) for flank_up, flank_down in _FLANKS: for test_name, test_group in _DO_GENERATE_MAX_WINDOW.items(): result_group = _DO_GENERATE_MAX_WINDOW_RESULTS[ "%s_%s_%s" % (test_name, flank_up, flank_down)] yield check_maximal_window, test_name, empty_hash, test_group, [ result_group ], flank_up, flank_down
def test_genomehash_update_from_list(self): # """Test addition to a `GenomeHash` from a list without loss of features # # 1. Add features to an empty dictionary # # 2. Add features to a non-empty dictionary # """ gh = GenomeHash({},do_copy=True) tx_list = sorted(list(self.tx_dict.values()),key=_name_sort) self.assertGreater(len(tx_list),0) gh.update(tx_list[:50]) self.assertEquals(sorted(list(gh.feature_dict.values()),key=_name_sort), tx_list[:50], "Features lost in update of empty GenomeHash from list") gh.update(tx_list[50:]) self.assertEquals(sorted(list(gh.feature_dict.values()),key=_name_sort), tx_list, "Features lost in update of non-empty GenomeHash from list")
'YMR194C_B_mRNA_0': ['YMR194C_B_mRNA_0:0-324^396-729(-)', 'YMR194C_B_mRNA_0:0-328^400-729(-)'], 'YPL249C_A_mRNA_0': ['YPL249C_A_mRNA_0:0-410^648-697(-)', 'YPL249C_A_mRNA_0:0-417^655-697(-)'] } unmatched_query_juncs = { K: [SegmentChain.from_str(X) for X in V] for K, V in unmatched_query_juncs.items() } """Query junctions with no known matches""" unmatched_noncan_query_juncs = [ "YNL130C:0-23^145-180(-)", "YNL130C:0-53^165-180(-)", "YNL130C:0-70^141-180(-)", "YNL130C:0-49^121-180(-)", ] unmatched_noncan_query_juncs = [ SegmentChain.from_str(X) for X in unmatched_noncan_query_juncs ] """Query junctions without canonical splice junctions in the match range""" repetitive_regions = [ "YBR215W_mRNA_0:190-193(+)", # threeprime splice site plus "YHL001W_mRNA_0:144-149(+)", # fiveprime splice site plus "YIL133C_mRNA_0:935-940(-)", # threeprime splice site minus "YMR194C_B_mRNA_0:325-328(-)", # fiveprime splice site minus ] cross_hash = GenomeHash( [SegmentChain(GenomicSegment.from_str(X)) for X in repetitive_regions]) cross_hash_seqs = {X.chrom for X in cross_hash.feature_dict.values()}
def setUpClass(cls): cls.cols = [3, 4, 5, 6, 8, 9, 12] cls.bedfiles = {} cls.bbfiles = {} for col in cls.cols: cls.bedfiles[col] = resource_filename( "plastid", "test/data/annotations/100transcripts_bed%s.bed" % col) cls.bbfiles[col] = resource_filename( "plastid", "test/data/annotations/100transcripts_bed%s.bb" % col) cls.chrom_sizes = {} for line in open( resource_filename("plastid", "test/data/annotations/sacCer3.sizes")): chrom, size = line.strip().split("\t") cls.chrom_sizes[chrom] = int(size) cls.bbs = { K: BigBedReader(cls.bbfiles[K], return_type=Transcript) for K in cls.cols } # comparisons against genome hash cls.binsize = 10000 transcripts = list( BED_Reader(open(cls.bedfiles[12]), return_type=Transcript)) cls.tx_dict = {} cls.cds_dict = {} cls.as_cds_dict = {} for tx in transcripts: txid = tx.get_name() cls.tx_dict[txid] = tx cds_ivc = tx.get_cds() cds_ivc.attr["ID"] = txid if cds_ivc.length > 0: cls.cds_dict[txid] = tx.get_cds() cls.as_cds_dict[txid] = tx.get_cds().get_antisense() cls.as_cds_dict[txid].attr["ID"] = txid cls.tx_hash = GenomeHash(cls.tx_dict, do_copy=False, binsize=cls.binsize) cls.cds_hash = GenomeHash(cls.cds_dict, do_copy=False, binsize=cls.binsize) cls.as_cds_hash = GenomeHash(cls.as_cds_dict, do_copy=False, binsize=cls.binsize) cls.shuffled_indices = list(range(len(transcripts))) shuffle(cls.shuffled_indices) cls.flybbfile = resource_filename( "plastid", "test/data/annotations/dmel-all-no-analysis-r5.54.bb") cls.flybedfile = resource_filename( "plastid", "test/data/annotations/dmel-all-no-analysis-r5.54.bed") # BigBed files with and without extra columns, with and without autoSql descriptions cls.bb_bonuscols = { "bb4as": resource_filename( "plastid", "test/data/annotations/100transcripts_bed4plus_bonus_as.bb"), "bb12as": resource_filename( "plastid", "test/data/annotations/100transcripts_bed12plus_bonus_as.bb"), "bb4no_as": resource_filename( "plastid", "test/data/annotations/100transcripts_bed4plus_bonus_no_as.bb" ), "bb12no_as": resource_filename( "plastid", "test/data/annotations/100transcripts_bed12plus_bonus_no_as.bb" ), } cls.bonus_col_file = resource_filename( "plastid", "test/data/annotations/bonus_bed_columns.txt") # BigBed file with indexes cls.bb_indexed = resource_filename( "plastid", "test/data/annotations/dmel-bonus-cols.bb")
def process_partial_group(transcripts, mask_hash, printer): """Correct boundaries of merged genes, as described in :func:`do_generate` Parameters ---------- transcripts : dict Dictionary mapping unique transcript IDs to |Transcripts|. This set should be complete in the sense that it should contain all transcripts that have any chance of mutually overlapping each other (e.g. all on same chromosome and strand). mask_hash : |GenomeHash| |GenomeHash| of regions to exclude from analysis Returns ------- :class:`pandas.DataFrame` Table of merged gene positions :class:`pandas.DataFrame` Table of adjusted transcript positions :class:`dict` Dictionary mapping raw gene names to merged gene names """ gene_table = { "region": [], "transcript_ids": [], "exon_unmasked": [], "exon": [], "masked": [], "utr5": [], "cds": [], "utr3": [], "exon_bed": [], "utr5_bed": [], "cds_bed": [], "utr3_bed": [], "masked_bed": [], } # data table for transcripts transcript_table = { "region": [], "exon": [], "utr5": [], "cds": [], "utr3": [], "masked": [], "exon_unmasked": [], "transcript_ids": [], "exon_bed": [], "utr5_bed": [], "cds_bed": [], "utr3_bed": [], "masked_bed": [], } keycombos = list(itertools.permutations(("utr5", "cds", "utr3"), 2)) # merge genes that share exons & write output printer.write("Collapsing genes that share exons ...") merged_genes = merge_genes(transcripts) # remap transcripts to merged genes # and vice-versa merged_gene_tx = {} tx_merged_gene = {} printer.write("Mapping transcripts to merged genes...") for txid in transcripts: my_tx = transcripts[txid] my_gene = my_tx.get_gene() my_merged = merged_genes[my_gene] tx_merged_gene[txid] = my_merged try: merged_gene_tx[my_merged].append(txid) except KeyError: merged_gene_tx[my_merged] = [txid] # flatten merged genes printer.write( "Flattening merged genes, masking positions, and labeling subfeatures ..." ) for n, (gene_id, my_txids) in enumerate(merged_gene_tx.items()): if n % 1000 == 0 and n > 0: printer.write(" %s genes ..." % n) my_gene_positions = [] chroms = [] strands = [] for my_txid in my_txids: my_segmentchain = transcripts[my_txid] chroms.append(my_segmentchain.chrom) strands.append(my_segmentchain.strand) my_gene_positions.extend(my_segmentchain.get_position_list()) try: assert len(set(chroms)) == 1 except AssertionError: printer.write( "Skipping gene %s which contains multiple chromosomes: %s" % (gene_id, ",".join(chroms))) try: assert len(set(strands)) == 1 except AssertionError: printer.write( "Skipping gene %s which contains multiple strands: %s" % (gene_id, ",".join(strands))) my_gene_positions = set(my_gene_positions) gene_ivc_raw = SegmentChain( *positions_to_segments(chroms[0], strands[0], my_gene_positions)) gene_table["region"].append(gene_id) gene_table["transcript_ids"].append(",".join(sorted(my_txids))) gene_table["exon_unmasked"].append(gene_ivc_raw) printer.write(" %s genes total." % (n + 1)) # mask genes printer.write("Masking positions and labeling subfeature positions ...") gene_hash = GenomeHash(gene_table["exon_unmasked"], do_copy=False) for n, (gene_id, gene_ivc_raw) in enumerate( zip(gene_table["region"], gene_table["exon_unmasked"])): if n % 2000 == 0: printer.write(" %s genes ..." % n) my_chrom = gene_ivc_raw.spanning_segment.chrom my_strand = gene_ivc_raw.spanning_segment.strand masked_positions = [] nearby_genes = gene_hash[gene_ivc_raw] # don't mask out positions from identical gene gene_ivc_raw_positions = gene_ivc_raw.get_position_set() nearby_genes = [ X for X in nearby_genes if X.get_position_set() != gene_ivc_raw_positions ] for gene in nearby_genes: masked_positions.extend(gene.get_position_list()) nearby_masks = mask_hash[gene_ivc_raw] for mask in nearby_masks: masked_positions.extend(mask.get_position_list()) masked_positions = set(masked_positions) gene_positions_raw = gene_ivc_raw.get_position_set() mask_ivc_positions = gene_positions_raw & masked_positions total_mask_ivc = SegmentChain(*positions_to_segments( my_chrom, my_strand, mask_ivc_positions), ID=gene_id) gene_table["masked"].append(total_mask_ivc) gene_table["masked_bed"].append(total_mask_ivc.as_bed()) gene_post_mask = gene_positions_raw - masked_positions gene_post_mask_ivc = SegmentChain(*positions_to_segments( my_chrom, my_strand, gene_post_mask), ID=gene_id) gene_table["exon"].append(gene_post_mask_ivc) gene_table["exon_bed"].append(gene_post_mask_ivc.as_bed()) masked_positions = total_mask_ivc.get_position_set() tmp_positions = { "utr5": set(), "cds": set(), "utr3": set(), } txids = sorted(merged_gene_tx[gene_id]) chrom = gene_post_mask_ivc.chrom strand = gene_post_mask_ivc.strand # pool transcript positions for txid in txids: transcript = transcripts[txid] utr5pos = transcript.get_utr5().get_position_set() cdspos = transcript.get_cds().get_position_set() utr3pos = transcript.get_utr3().get_position_set() tmp_positions["utr5"] |= utr5pos tmp_positions["cds"] |= cdspos tmp_positions["utr3"] |= utr3pos # eliminate positions in which CDS & UTRs overlap from each transcript for txid in txids: transcript = transcripts[txid] transcript_positions = { "utr5": transcript.get_utr5().get_position_set(), "cds": transcript.get_cds().get_position_set(), "utr3": transcript.get_utr3().get_position_set(), } for key1, key2 in keycombos: transcript_positions[key1] -= tmp_positions[key2] transcript_positions[key1] -= masked_positions transcript_table["region"].append(txid) # all unmasked positions my_chain = SegmentChain(*positions_to_segments( chrom, strand, transcript.get_position_set() - masked_positions), ID=txid) transcript_table["exon"].append(str(my_chain)) transcript_table["exon_bed"].append(my_chain.as_bed()) # all uniquely-labeled unmasked positions for k, v in transcript_positions.items(): my_chain = SegmentChain(*positions_to_segments( chrom, strand, v), ID=txid) transcript_table[k].append(str(my_chain)) transcript_table["%s_bed" % k].append(my_chain.as_bed()) total_mask_ivc.attr["ID"] = txid transcript_table["masked"].append(str(total_mask_ivc)) transcript_table["masked_bed"].append(total_mask_ivc.as_bed()) transcript_table["exon_unmasked"].append(str(transcript)) transcript_table["transcript_ids"].append(txid) tmp_positions2 = copy.deepcopy(tmp_positions) for k1, k2 in keycombos: tmp_positions[k1] -= tmp_positions2[k2] tmp_positions[k1] -= masked_positions for k in (tmp_positions.keys()): my_chain = SegmentChain(*positions_to_segments( chrom, strand, tmp_positions[k]), ID=gene_id) gene_table[k].append(str(my_chain)) gene_table["%s_bed" % k].append(my_chain.as_bed()) printer.write(" %s genes total." % (n + 1)) # cast SegmentChains/Transcripts to strings to keep numpy from unpacking them conversion_keys = [ "exon", "utr5", "cds", "utr3", "masked", "exon_unmasked" ] for k in conversion_keys: gene_table[k] = [str(X) for X in gene_table[k]] transcript_table[k] = [str(X) for X in transcript_table[k]] gene_df = pd.DataFrame(gene_table) gene_df.sort_values(["region"], inplace=True) transcript_df = pd.DataFrame(transcript_table) transcript_df.sort_values(["region"], inplace=True) return gene_df, transcript_df, merged_genes
def main(argv=sys.argv[1:]): """Command-line program Parameters ---------- argv : list, optional A list of command-line arguments, which will be processed as if the script were called from the command line if :py:func:`main` is called directly. Default: `sys.argv[1:]`. The command-line arguments, if the script is invoked from the command line """ sp = SequenceParser() mp = MaskParser() bp = BaseParser() parser = argparse.ArgumentParser(description=format_module_docstring(__doc__), formatter_class=argparse.RawDescriptionHelpFormatter, parents=[bp.get_parser(),sp.get_parser(),mp.get_parser()], ) parser.add_argument("--maxslide",type=int,default=10, help="Maximum number of nt to search 5\' and 3\' of intron"+ " boundaries (Default: 10)") parser.add_argument("--ref",type=str,metavar="ref.bed",default=None, help="Reference file describing known splice junctions") parser.add_argument("--slide_canonical",action="store_true",default=False, help="Slide junctions to canonical junctions if present within equal support region") parser.add_argument("infile",type=str,metavar="input.bed", help="BED file describing discovered junctions") parser.add_argument("outbase",type=str, help="Basename for output files") args = parser.parse_args(argv) bp.get_base_ops_from_args(args) printer.write("Opening genome from %s..." % args.sequence_file) genome = sp.get_seqdict_from_args(args) # load crossmap cross_hash = mp.get_genome_hash_from_args(args) # load ref junctions if args.ref is not None: printer.write("Loading reference junctions from %s" % args.ref) known_hash = GenomeHash(list(BED_Reader(open(args.ref))),do_copy=False) else: known_hash = GenomeHash() # set up variables canonicals_plus = [("GT","AG"), ("GC","AG") ] canonicals_minus = [("CT","AC"), ("CT","GC") ] known_in_range = 0 canonical_in_range = 0 repetitive = 0 untouched = 0 c = 0 seen_already = [] outfiles = { "repetitive" : "%s_repetitive.bed" % args.outbase, "known" : "%s_shifted_known.bed" % args.outbase, "canonical" : "%s_shifted_canonical.bed" % args.outbase, "untouched" : "%s_untouched.bed" % args.outbase, } outfiles = { K : argsopener(V,args,"w") for K,V in outfiles.items() } # process data printer.write("Opening junctions from %s..." % args.infile) for ivc in BED_Reader(CommentReader(opener(args.infile))): processed = False tup = None if c % 1000 == 0 and c > 0: printer.write("Processed: %s\tknown: %s\tshifted to canonical: %s\trepetitive: %s\tuntouched: %s" % \ (c, known_in_range, canonical_in_range, repetitive, untouched)) assert len(ivc) == 2 strand = ivc.strand minus_range, plus_range = find_match_range(ivc,genome,args.maxslide) # see if either end of splice junction +- match_range lands in repetitive areas of genome if covered_by_repetitive(ivc,minus_range,plus_range,cross_hash): repetitive += 1 outfiles["repetitive"].write(ivc.as_bed()) processed = True # see if one or more known junctions in range if processed == False and args.ref is not None: # find_known_in_range(query_ivc,minus_range,plus_range,knownjunctions) known_juncs = find_known_in_range(ivc,minus_range,plus_range,known_hash.get_nearby_features(ivc)) if len(known_juncs) > 0: known_in_range += 1 for my_known in known_juncs: tup = get_junction_tuple(my_known) if tup not in seen_already: outfiles["known"].write(my_known.as_bed()) seen_already.append(tup) processed = True # see if one or more canonical junctions in range if processed == False and args.slide_canonical == True: canonicals = canonicals_plus if strand == "+" else canonicals_minus #find_canonicals_in_range(query_ivc,minus_range,plus_range,genome,canonicals) canonical_juncs = find_canonicals_in_range(ivc,minus_range,plus_range,genome,canonicals) if len(canonical_juncs) > 0: canonical_in_range += 1 for can in canonical_juncs: tup = get_junction_tuple(can) if tup not in seen_already: outfiles["canonical"].write(can.as_bed()) seen_already.append(tup) processed = True if processed == False: outfiles["untouched"].write(ivc.as_bed()) untouched += 1 c += 1 # save output printer.write("Totals: %s\tknown: %s\tshifted to canonical: %s\trepetitive: %s\tuntouched: %s" % \ (c, known_in_range, canonical_in_range, repetitive, untouched)) for v in outfiles.values(): v.close() printer.write("Done.")