def crossmap(source_compressed_gtf): requests.packages.urllib3.util.ssl_.DEFAULT_CIPHERS += 'HIGH:!DH:!aNULL' # Needed for UCSC # only download if necessary if not os.path.exists(os.path.join('data', 'hg38ToHg19.over.chain.gz')): sys.stdout.write('Downloading UCSC database... ') sys.stdout.flush() url = 'https://hgdownload.soe.ucsc.edu/goldenPath/hg38/liftOver/hg38ToHg19.over.chain.gz' try: p = requests.get(url, verify=False) with open(os.path.join('data', 'hg38ToHg19.over.chain.gz'), 'wb') as o: o.write(p.content) except Exception as e: print( '\n\nCannot connect to UCSC FTP site. No internet connection?\n' ) print(f'Exception: {e}') quit() sys.stdout.write('\nMaking a hg19-conveterted GTF file\n') mapTree, targetChromSizes, sourceChromSizes = read_chain_file( os.path.join('data', 'hg38ToHg19.over.chain.gz')) converted_gtf = source_compressed_gtf.replace('.gtf.gz', '.hg19_converted.gtf') crossmap_gff_file(mapTree, source_compressed_gtf, converted_gtf) # Note this file is not sorted! a = pybedtools.BedTool(converted_gtf) a.sort().remove_invalid().saveas('tmp.txt') os.rename('tmp.txt', converted_gtf)
infile=in_file, outfile_prefix=out_file, chrom_size=targetChromSizes, IS_size=args.insert_size, IS_std=args.insert_size_stdev, fold=args.insert_size_fold, addtag=args.add_tags, cstyle=args.cstyle) elif command == 'gff': chain_file = args.chain in_file = args.in_gff out_file = args.out_gff (mapTree, targetChromSizes, sourceChromSizes) = read_chain_file(chain_file) crossmap_gff_file(mapTree, in_file, out_file, cstyle=args.cstyle) elif command == 'wig': chain_file = args.chain in_file = args.in_wig out_file = args.out_wig (mapTree, targetChromSizes, sourceChromSizes) = read_chain_file(chain_file) crossmap_wig_file(mapTree, in_file, out_file, targetChromSizes, in_format='wiggle', cstyle=args.cstyle) elif command == 'bigwig':
def process_data(options, genome_build): # Dictionary of Gene objects genesdata = dict() # Load custom transcript IDs transIDs = None if options.input is not None: transIDs = readTranscriptIDs(options.input) print('\nOnly ' + str(len(transIDs)) + ' transcripts read from ' + options.input + ' are considered\n') else: print('\nAll transcripts from the Ensembl release are considered\n') # Load candidate and CCDS data for Ensembl <75 candidates = dict() if int(options.ensembl) < 75: datadir = os.path.dirname(os.path.realpath(__file__)) + '/data' for line in open(datadir + '/info' + options.ensembl + '.txt'): line = line.strip() if line == '': continue cols = line.split('\t') if cols[0] not in list(candidates.keys()): candidates[cols[0]] = dict() candidates[cols[0]][cols[1]] = int(cols[2]) ###################################################################### # Download Ensembl data if necessary source_compressed_gtf = 'Homo_sapiens.' + genome_build + '.' + options.ensembl + '.gtf.gz' source_compressed_gtf = os.path.join('data', source_compressed_gtf) if not os.path.exists(source_compressed_gtf): sys.stdout.write('Downloading Ensembl database... ') sys.stdout.flush() url = 'ftp://ftp.ensembl.org/pub/release-' + options.ensembl + '/gtf/homo_sapiens/Homo_sapiens.' + genome_build + '.' + options.ensembl + '.gtf.gz' try: wget.download(url) os.rename( 'Homo_sapiens.' + genome_build + '.' + options.ensembl + '.gtf.gz', source_compressed_gtf) except Exception as e: print( '\n\nCannot connect to Ensembl FTP site. No internet connection?\n' ) print(f'{e}\n{url}') quit() ################################################################ # Use crossmap to get hg19 if desired ################################################################# if options.no_hg19 is not False: requests.packages.urllib3.util.ssl_.DEFAULT_CIPHERS += 'HIGH:!DH:!aNULL' # Needed for UCSC # only download if necessary if not os.path.exists(os.path.join('data', 'hg38ToHg19.over.chain.gz')): sys.stdout.write('Downloading UCSC database... ') sys.stdout.flush() url = 'https://hgdownload.soe.ucsc.edu/goldenPath/hg38/liftOver/hg38ToHg19.over.chain.gz' try: p = requests.get(url, verify=False) with open(os.path.join('data', 'hg38ToHg19.over.chain.gz'), 'wb') as o: o.write(p.content) except Exception as e: print( '\n\nCannot connect to UCSC FTP site. No internet connection?\n' ) print(f'Exception: {e}') quit() sys.stdout.write('\nMaking a hg19-conveterted GTF file\n') mapTree, targetChromSizes, sourceChromSizes = read_chain_file( os.path.join('data', 'hg38ToHg19.over.chain.gz')) converted_gtf = os.path.join( 'data', 'Homo_sapiens.hg19_converted' + options.ensembl + '.gtf') crossmap_gff_file(mapTree, source_compressed_gtf, converted_gtf) # Note this file is not sorted! a = pybedtools.BedTool(converted_gtf) a.sort().remove_invalid().saveas('tmp.txt') os.rename('tmp.txt', converted_gtf) ################################################################ # ################################################################# # Iterate through the lines in the ensembl data file sys.stdout.write('Extracting transcript data from Ensembl...') transcript, prevenst, first, genesdata = parse_GTF( filename=source_compressed_gtf, options=options, genesdata=genesdata, transIDs=transIDs) sys.stdout.write('Done\n') sys.stdout.flush() # Finalize last transcript and add to Gene object if candidate if transcript is not None: transcript.finalize() if transcript.isCandidate(): if transcript.ENSG not in list(genesdata.keys()): genesdata[transcript.ENSG] = Gene(transcript.GENE, transcript.ENSG) genesdata[transcript.ENSG].TRANSCRIPTS[ transcript.ENST] = transcript # If no transcript ID from the input file was found in the Ensembl release if len(genesdata) == 0: print('\n\nNo transcripts from ' + options.input + ' found in Ensembl release.') print('\nNo transcript database created.') print( "-----------------------------------------------------------------\n" ) quit() write_temp(os.path.join(options.output_dir, options.output + '.txt'), options, transIDs, genesdata) enst_records = sort_tmpfile('temp.txt') assert (len(enst_records) > 0) writeToFile(enst_records, os.path.join(options.output_dir, options.output)) failed_conversions['GENE'] = set() failed_conversions['GENETYPE'] = set() failed_conversions['TRANSTYPE'] = set() failed_conversions['ENST'] = set() # ################################################################ # Begin converted GTF conversion # ################################################################ hg19_records = [] if options.no_hg19 is not False: sys.stdout.write('Extracting transcript data for hg19 version...') sys.stdout.flush() transcript, prevenst, first, genesdata = parse_GTF( filename=converted_gtf, options=options, genesdata=genesdata, transIDs=transIDs) # Finalize last transcript and add to Gene object if candidate if transcript is not None: try: transcript.finalize() except: warn(transcript) if transcript.isCandidate(): if transcript.ENSG not in list(genesdata.keys()): genesdata[transcript.ENSG] = Gene(transcript.GENE, transcript.ENSG) genesdata[transcript.ENSG].TRANSCRIPTS[ transcript.ENST] = transcript # If no transcript ID from the input file was found in the Ensembl release if len(genesdata) == 0: print('\n\nNo transcripts from ' + options.input + ' found in Ensembl release.') print('\nNo transcript database created.') print( "-----------------------------------------------------------------\n" ) quit() write_temp( os.path.join(options.output_dir, options.output + '.hg19_converted.txt'), options, transIDs, genesdata) sortedRecords = sort_tmpfile('temp.txt') writeToFile( sortedRecords, os.path.join(options.output_dir, options.output + '.hg19_converted')) sys.stdout.write('Completed hg19 version...') sys.stdout.flush() pickle.dump( failed_conversions, open( os.path.join(options.output_dir, options.output + '_failed_conversions.pkl'), 'wb')) hg19_records = sortedRecords # ################################################################ # END converted GTF conversion # ################################################################ # Remove temporary files sys.stdout.write('OK\n') sys.stdout.write('Removing temporary files... ') sys.stdout.flush() os.remove('temp.txt') # os.remove(source_compressed_gtf) print( f"Failed {failed_conversions['GENE'].__len__()} Genes and {failed_conversions['ENST'].__len__()} transcripts" ) # Return sorted records return len(enst_records), len(hg19_records)
chain_file = args[1] in_file = args[2] out_file = args[3] (mapTree, targetChromSizes, sourceChromSizes) = read_chain_file(chain_file) crossmap_region_file(mapTree, in_file, out_file, min_ratio = options.min_map_ratio) else: region_help() parser.print_usage() sys.exit(0) elif sys.argv[1].lower() == 'gff': if len(sys.argv) == 4: chain_file = sys.argv[2] in_file = sys.argv[3] (mapTree, targetChromSizes, sourceChromSizes) = read_chain_file(chain_file) crossmap_gff_file(mapTree, in_file, None) elif len(sys.argv) == 5: chain_file = sys.argv[2] in_file = sys.argv[3] out_file = sys.argv[4] (mapTree, targetChromSizes, sourceChromSizes) = read_chain_file(chain_file) crossmap_gff_file(mapTree, in_file, out_file) else: gff_help() sys.exit(0) elif sys.argv[1].lower() == 'wig': if len(sys.argv) == 5: chain_file = sys.argv[2] in_file = sys.argv[3] out_file = sys.argv[4] (mapTree, targetChromSizes, sourceChromSizes) = read_chain_file(chain_file)
def process_data(options): # Dictionary of Gene objects genesdata = dict() # Load custom transcript IDs transIDs = None if options.input is not None: transIDs = readTranscriptIDs(options.input) print('\nOnly ' + str(len(transIDs)) + ' transcripts read from ' + options.input + ' are considered\n') else: nm = 'All transcripts from the release are considered' if options.nm_only: nm = 'All NM transcripts from the release are considered' print(f'\n{nm}\n') # Load candidate and CCDS data for Ensembl <75 dict() ###################################################################### # Download RefSeq data if necessary source_compressed_gtf = options.refseq + '_genomic.gtf.gz' # https://ftp.ncbi.nlm.nih.gov/genomes/refseq/vertebrate_mammalian/Homo_sapiens/reference/GCF_000001405.39_GRCh38.p13/GCF_000001405.39_GRCh38.p13_genomic.gtf.gz source_compressed_gtf = os.path.join('data', source_compressed_gtf) if not os.path.exists(source_compressed_gtf): sys.stdout.write('Downloading RefSeq database... ') sys.stdout.flush() url = 'https://ftp.ncbi.nlm.nih.gov/genomes/refseq/vertebrate_mammalian/Homo_sapiens/reference/' + options.refseq + '/' + options.refseq + '_genomic.gtf.gz' try: wget.download(url) sys.stdout.flush() # Convert chromosome names #Note we will lose unmapped transcripts here! print(f'\nUnzipping {options.refseq + "_genomic.gtf.gz"}') cmd = 'bgzip -d ' + options.refseq + '_genomic.gtf.gz' os.system(cmd) out = open('temp.txt', 'w') print(f'Parsing {options.refseq + "_genomic.gtf"}') with open(options.refseq + '_genomic.gtf', 'r') as g: for line in g: if line.startswith('#'): continue try: new_line = replace_chrom_names(line) except: print(f'Failed: {line}') exit() if new_line: out.write(new_line) out.close() print(f'Compressing the GTF into: {source_compressed_gtf}') cmd = 'bgzip -c temp.txt > ' + source_compressed_gtf os.system(cmd) os.remove('temp.txt') except Exception as e: print('\n\nCannot connect to RefSeq FTP site. No internet connection?\n') print(f'{e}\n{url}') quit() ################################################################ # Use crossmap to get hg19 if desired ################################################################# if options.no_hg19 is not False: requests.packages.urllib3.util.ssl_.DEFAULT_CIPHERS += 'HIGH:!DH:!aNULL' # Needed for UCSC # only download if necessary if not os.path.exists(os.path.join('data', 'hg38ToHg19.over.chain.gz')): sys.stdout.write('Downloading UCSC database... ') sys.stdout.flush() url = 'https://hgdownload.soe.ucsc.edu/goldenPath/hg38/liftOver/hg38ToHg19.over.chain.gz' try: p = requests.get(url, verify=False) with open(os.path.join('data', 'hg38ToHg19.over.chain.gz'), 'wb') as o: o.write(p.content) except Exception as e: print('\n\nCannot connect to UCSC FTP site. No internet connection?\n') print(f'Exception: {e}') quit() converted_gtf = os.path.join('data', 'Homo_sapiens.RefSeq.hg19_converted.' + options.refseq + '.gtf') if not os.path.exists(converted_gtf): sys.stdout.write('\nMaking a hg19-conveterted GTF file\n') mapTree, targetChromSizes, sourceChromSizes = read_chain_file( os.path.join('data', 'hg38ToHg19.over.chain.gz')) crossmap_gff_file(mapTree, source_compressed_gtf, converted_gtf) # Note this file is not sorted! a = pybedtools.BedTool(converted_gtf) a.sort().remove_invalid().saveas('tmp.txt') os.rename('tmp.txt', converted_gtf) ################################################################ # ################################################################# # Iterate through the lines in the refseq data file sys.stdout.write('Extracting transcript data from RefSeq...') transcript, prevenst, first, genesdata = parse_GTF(filename=source_compressed_gtf, options=options, genesdata=genesdata, transIDs=transIDs) sys.stdout.write('Done\n') sys.stdout.flush() # Finalize last transcript and add to Gene object if candidate if transcript is not None: transcript.finalize() if transcript.isCandidate(): if transcript.ENSG not in list(genesdata.keys()): genesdata[transcript.ENSG] = Gene(transcript.GENE, transcript.ENSG) genesdata[transcript.ENSG].TRANSCRIPTS[transcript.ENST] = transcript # If no transcript ID from the input file was found in the Ensembl release if len(genesdata) == 0: print('\n\nNo transcripts found in this release.') print('\nNo transcript database created.') print("-----------------------------------------------------------------\n") quit() write_temp(os.path.join(options.output_dir, options.output + '.txt'), options, transIDs, genesdata) enst_records = sort_tmpfile('temp.txt') assert (len(enst_records) > 0) writeToFile(enst_records, os.path.join(options.output_dir, options.output)) failed_conversions['GENE'] = set() failed_conversions['GENETYPE'] = set() failed_conversions['TRANSTYPE'] = set() failed_conversions['ENST'] = set() # ################################################################ # Begin converted GTF conversion # ################################################################ hg19_records = [] if options.no_hg19 is not False: sys.stdout.write('Extracting transcript data for hg19 version...') sys.stdout.flush() transcript, prevenst, first, genesdata = parse_GTF(filename=converted_gtf, options=options, genesdata=genesdata, transIDs=transIDs) # Finalize last transcript and add to Gene object if candidate if transcript is not None: try: transcript.finalize() except: warn(transcript) if transcript.isCandidate(): if transcript.ENSG not in list(genesdata.keys()): genesdata[transcript.ENSG] = Gene(transcript.GENE, transcript.ENSG) genesdata[transcript.ENSG].TRANSCRIPTS[transcript.ENST] = transcript # If no transcript ID from the input file was found in the release if len(genesdata) == 0: print('\n\nNo transcripts from ' + options.input + ' found in the release.') print('\nNo transcript database created.') print("-----------------------------------------------------------------\n") quit() write_temp(os.path.join(options.output_dir, options.output + '.hg19_converted.txt'), options, transIDs, genesdata) sortedRecords = sort_tmpfile('temp.txt') writeToFile(sortedRecords, os.path.join(options.output_dir, options.output + '.hg19_converted')) sys.stdout.write('Completed hg19 version...') sys.stdout.flush() pickle.dump(failed_conversions, open(os.path.join(options.output_dir, options.output + '_failed_conversions.pkl'), 'wb')) hg19_records = sortedRecords # ################################################################ # END converted GTF conversion # ################################################################ # Remove temporary files sys.stdout.write('OK\n') sys.stdout.write('Removing temporary files... ') sys.stdout.flush() os.remove('temp.txt') os.remove(source_compressed_gtf) print(f"Failed {failed_conversions['GENE'].__len__()} Genes and {failed_conversions['ENST'].__len__()} transcripts") # Return sorted records return len(enst_records), len(hg19_records)