def load_broadinstitute_exac(): print('DATA_FOLDER: ' + DATA_FOLDER) t0 = time.time() exacs = load_broadinstitute_exac_all() for k, v in load_broadinstitute_exac_nontcga().items(): try: exacs[k]["exac"]["nontcga"] = v["exac"]["nontcga"] except KeyError: exacs[k] = v for k, v in load_broadinstitute_exac_nonpsych().items(): try: exacs[k]["exac"]["nonpsych"] = v["exac"]["nonpsych"] except KeyError: exacs[k] = v logging.info("Convert transcript ID to EntrezID") import dataload.sources.ensembl.ensembl_base as ensembl_base ensembl_parser = ensembl_base.EnsemblParser() ensembl_parser._load_ensembl2entrez_li() ensembl2entrez = list2dict(ensembl_parser.ensembl2entrez_li, 0, alwayslist=True) ensembl_dir = get_data_folder("ensembl") for line in tabfile_feeder( os.path.join(ensembl_dir, "gene_ensembl__translation__main.txt")): _, ensid, transid, _ = line if transid in exacs: data = exacs.pop( transid) # pop so no-match means no data in the end for entrezid in ensembl2entrez.get(ensid, [ensid]): exacs[entrezid] = data load_done('[%d, %s]' % (len(exacs), timesofar(t0))) return exacs
def load_broadinstitute_exac(): print('DATA_FOLDER: ' + DATA_FOLDER) t0 = time.time() exacs = load_broadinstitute_exac_all() for k,v in load_broadinstitute_exac_nontcga().items(): try: exacs[k]["exac"]["nontcga"] = v["exac"]["nontcga"] except KeyError: exacs[k] = v for k,v in load_broadinstitute_exac_nonpsych().items(): try: exacs[k]["exac"]["nonpsych"] = v["exac"]["nonpsych"] except KeyError: exacs[k] = v logging.info("Convert transcript ID to EntrezID") import dataload.sources.ensembl.ensembl_base as ensembl_base ensembl_parser = ensembl_base.EnsemblParser() ensembl_parser._load_ensembl2entrez_li() ensembl2entrez = list2dict(ensembl_parser.ensembl2entrez_li, 0, alwayslist=True) ensembl_dir = get_data_folder("ensembl") for line in tabfile_feeder(os.path.join(ensembl_dir,"gene_ensembl__translation__main.txt")): _,ensid,transid,_ = line if transid in exacs: data = exacs.pop(transid) # pop so no-match means no data in the end for entrezid in ensembl2entrez.get(ensid,[ensid]): exacs[entrezid] = data load_done('[%d, %s]' % (len(exacs), timesofar(t0))) return exacs
from __future__ import print_function import os.path from collections import defaultdict from dataload import get_data_folder from utils.dataload import anyfile, tabfile_feeder from biothings.utils.common import safewfile ENSEMBL_DATA_FOLDER = get_data_folder('ensembl') print('Ensembl DATA_FOLDER: ' + ENSEMBL_DATA_FOLDER) Entrez_DATA_FOLDER = get_data_folder('entrez') print('Ensembl DATA_FOLDER: ' + Entrez_DATA_FOLDER) gene_ensembl_1_xref_dm_file = os.path.join(ENSEMBL_DATA_FOLDER, "gene_ensembl__xref_entrezgene__dm.txt") gene_ensembl_2_main_file = os.path.join(ENSEMBL_DATA_FOLDER, "gene_ensembl__gene__main.txt") gene2ensembl_file = os.path.join(Entrez_DATA_FOLDER, "gene/gene2ensembl.gz") gene_main_file = os.path.join(Entrez_DATA_FOLDER, "gene/gene_info.gz") outfile = os.path.join(ENSEMBL_DATA_FOLDER, "gene_ensembl__gene__extra.txt") def find_multiple_mappings_from_entrezgene_file(gene_ensembl_entrezgene_dm_file): """Input gene_ensembl_entrezgene_dm_file, and identify how many NCBI gene IDs there are for each ensembl gene ID. Lines in input file are: 'gene_ensembl__xref_entrezgene__dm.txt' (useful columns in input_file): col1: Ensembl gene ID col2: NCBI gene ID
import os.path import copy #from config import DATA_ARCHIVE_ROOT from dataload import get_data_folder from utils.common import SubStr from utils.dataload import (load_start, load_done, tab2dict, tab2list, value_convert, normalized_value, list2dict, dict_nodup, dict_attrmerge ) #DATA_FOLDER = os.path.join(DATA_ARCHIVE_ROOT, 'by_resources/ensembl/69') DATA_FOLDER = get_data_folder('ensembl') print('DATA_FOLDER: ' + DATA_FOLDER) #fn to skip lines with LRG records.''' _not_LRG = lambda ld: not ld[1].startswith("LRG_") class EnsemblParser: def __init__(self): self.ensembl2entrez_li = None self.ensembl_main = None def _load_ensembl_2taxid(self): """ensembl2taxid""" DATAFILE = os.path.join(DATA_FOLDER, 'gene_ensembl__translation__main.txt') load_start(DATAFILE) ensembl2taxid = dict_nodup(tab2dict(DATAFILE, (0, 1), 1, includefn=_not_LRG)) # need to convert taxid to integer here ensembl2taxid = value_convert(ensembl2taxid, lambda x: int(x))
import os.path import time from biothings.utils.common import timesofar from utils.dataload import (load_start, load_done, tab2dict, tabfile_feeder, list2dict) from dataload import get_data_folder import logging # DATA_FOLDER = os.path.join(DATA_ARCHIVE_ROOT, 'by_resources/uniprot') DATA_FOLDER = get_data_folder('exac') def load_broadinstitute_exac_any(one_file,key): print("Loading file %s (%s)" % (one_file,key)) data = tab2dict(os.path.join(DATA_FOLDER, one_file), (0,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21), 0) exacs = {} for transcript in data: tupleexac = data[transcript] # remove version in key so we can search the dict easily later exacs[transcript.split(".")[0]] = {"exac" : { "transcript" : transcript, # but keep version here "n_exons" : int(tupleexac[0]), "cds_start" : int(tupleexac[1]), "cds_end" : int(tupleexac[2]), "bp" : int(tupleexac[3]), key : { "mu_syn" : float(tupleexac[4]), "mu_mis" : float(tupleexac[5]), "mu_lof" : float(tupleexac[6]),
import os.path import copy #from config import DATA_ARCHIVE_ROOT from dataload import get_data_folder from utils.common import SubStr from utils.dataload import (load_start, load_done, tab2dict, tab2list, value_convert, normalized_value, list2dict, dict_nodup, dict_attrmerge ) #DATA_FOLDER = os.path.join(DATA_ARCHIVE_ROOT, 'by_resources/ensembl/69') DATA_FOLDER = get_data_folder('ensembl') print('DATA_FOLDER: ' + DATA_FOLDER) #fn to skip lines with LRG records.''' def _not_LRG(ld): return not ld[1].startswith("LRG_") class EnsemblParser: def __init__(self): self.ensembl2entrez_li = None self.ensembl_main = None def _load_ensembl_2taxid(self): """ensembl2taxid""" DATAFILE = os.path.join(DATA_FOLDER, 'gene_ensembl__translation__main.txt') load_start(DATAFILE) ensembl2taxid = dict_nodup(tab2dict(DATAFILE, (0, 1), 1, includefn=_not_LRG)) # need to convert taxid to integer here
from __future__ import print_function import os.path from collections import defaultdict from dataload import get_data_folder from utils.dataload import anyfile, tabfile_feeder from biothings.utils.common import safewfile ENSEMBL_DATA_FOLDER = get_data_folder('ensembl') print('Ensembl DATA_FOLDER: ' + ENSEMBL_DATA_FOLDER) Entrez_DATA_FOLDER = get_data_folder('entrez') print('Ensembl DATA_FOLDER: ' + Entrez_DATA_FOLDER) gene_ensembl_1_xref_dm_file = os.path.join( ENSEMBL_DATA_FOLDER, "gene_ensembl__xref_entrezgene__dm.txt") gene_ensembl_2_main_file = os.path.join(ENSEMBL_DATA_FOLDER, "gene_ensembl__gene__main.txt") gene2ensembl_file = os.path.join(Entrez_DATA_FOLDER, "gene/gene2ensembl.gz") gene_main_file = os.path.join(Entrez_DATA_FOLDER, "gene/gene_info.gz") outfile = os.path.join(ENSEMBL_DATA_FOLDER, "gene_ensembl__gene__extra.txt") def find_multiple_mappings_from_entrezgene_file( gene_ensembl_entrezgene_dm_file): """Input gene_ensembl_entrezgene_dm_file, and identify how many NCBI gene IDs there are for each ensembl gene ID. Lines in input file are: 'gene_ensembl__xref_entrezgene__dm.txt' (useful columns in input_file): col1: Ensembl gene ID
import os.path import types import time from utils.common import timesofar from utils.dataload import (load_start, load_done, listitems, dupline_seperator, tabfile_feeder, list2dict, list_nondup, value_convert) from config import DATA_ARCHIVE_ROOT from dataload import get_data_folder #DATA_FOLDER = os.path.join(DATA_ARCHIVE_ROOT, 'by_resources/uniprot') DATA_FOLDER = get_data_folder('uniprot') #REF: #ftp://ftp.uniprot.org/pub/databases/uniprot/current_release/knowledgebase/idmapping/README def get_uniprot_section(uniprotkb_id): '''return either "Swiss-Prot" or "TrEMBL", two sections of UniProtKB, based on input uniprotkb_id, or entry name. The rule is (http://www.uniprot.org/manual/entry_name): Swiss-Prot entries have a maximum of 5 characters before the "_", TrEMBL entries have 6 characters before the "_" (the accession some examples: TrEMBL: O61847_CAEEL, F0YED1_AURAN Swiss-Prot: CDK2_HUMAN, CDK2_MOUSE ''' v = uniprotkb_id.split('_') if len(v) != 2: raise ValueError('Invalid UniprotKB ID') return 'TrEMBL' if len(v[0])==6 else "Swiss-Prot"
import os.path import time from biothings.utils.common import get_timestamp, timesofar from utils.dataload import (load_start, load_done, tabfile_feeder, list2dict, value_convert, dict_convert) from dataload import get_data_folder DATA_FOLDER = get_data_folder('cpdb') def _download(__metadata__): from utils.dataload import download as _download output_folder = os.path.join(os.path.split(DATA_FOLDER)[0], get_timestamp()) for species in ['human', 'mouse', 'yeast']: url = __metadata__['__url_{}__'.format(species)] output_file = 'CPDB_pathways_genes_{}.tab'.format(species) _download(url, output_folder, output_file) def load_cpdb(__metadata__): # only import pathways from these sources PATHWAY_SOURCES_INCLUDED = __metadata__['pathway_sources_included'] VALID_COLUMN_NO = 4 t0 = time.time() print('DATA_FOLDER: ' + DATA_FOLDER) DATA_FILES = [] DATA_FILES.append(os.path.join(DATA_FOLDER, 'CPDB_pathways_genes_mouse.tab')) DATA_FILES.append(os.path.join(DATA_FOLDER, 'CPDB_pathways_genes_yeast.tab'))
import os.path from utils.dataload import (load_start, load_done, tab2dict, value_convert) from dataload import get_data_folder #from config import DATA_ARCHIVE_ROOT #timestamp = '20121005' #DATA_FOLDER = os.path.join(DATA_ARCHIVE_ROOT, 'by_resources/pharmgkb', timestamp) DATA_FOLDER = get_data_folder('pharmgkb') def load_pharmgkb(): print('DATA_FOLDER: ' + DATA_FOLDER) DATAFILE = os.path.join(DATA_FOLDER, 'genes.zip') load_start(DATAFILE) gene2pharmgkb = tab2dict((DATAFILE, 'genes.tsv'), (0, 1), 1, header=1,includefn=lambda ld:ld[1]!='') fn = lambda value: {'pharmgkb': value} gene2pharmgkb = value_convert(gene2pharmgkb, fn, traverse_list=False) load_done('[%d]' % len(gene2pharmgkb)) return gene2pharmgkb
import os.path import time from biothings.utils.common import timesofar from utils.dataload import (load_start, load_done, listitems, dupline_seperator, tabfile_feeder, list2dict, list_nondup, value_convert) #from config import DATA_ARCHIVE_ROOT from dataload import get_data_folder #DATA_FOLDER = os.path.join(DATA_ARCHIVE_ROOT, 'by_resources/uniprot') DATA_FOLDER = get_data_folder('uniprot') #REF: #ftp://ftp.uniprot.org/pub/databases/uniprot/current_release/knowledgebase/idmapping/README VALID_COLUMN_NO = 22 def get_uniprot_section(uniprotkb_id): '''return either "Swiss-Prot" or "TrEMBL", two sections of UniProtKB, based on input uniprotkb_id, or entry name. The rule is (http://www.uniprot.org/manual/entry_name): Swiss-Prot entries have a maximum of 5 characters before the "_", TrEMBL entries have 6 or 10 characters before the "_" (the accession some examples: TrEMBL: O61847_CAEEL, F0YED1_AURAN, A0A024RB10_HUMAN Swiss-Prot: CDK2_HUMAN, CDK2_MOUSE ''' v = uniprotkb_id.split('_') if len(v) != 2: raise ValueError('Invalid UniprotKB ID') #return 'TrEMBL' if len(v[0])==6 else "Swiss-Prot"
import os.path from utils.dataload import (load_start, load_done, tab2dict, value_convert) from dataload import get_data_folder #from config import DATA_ARCHIVE_ROOT #timestamp = '20121005' #DATA_FOLDER = os.path.join(DATA_ARCHIVE_ROOT, 'by_resources/pharmgkb', timestamp) DATA_FOLDER = get_data_folder('pharmgkb') def load_pharmgkb(): print('DATA_FOLDER: ' + DATA_FOLDER) DATAFILE = os.path.join(DATA_FOLDER, 'genes.zip') load_start(DATAFILE) gene2pharmgkb = tab2dict((DATAFILE, 'genes.tsv'), (0, 1), 1, header=1, includefn=lambda ld: ld[1] != '') fn = lambda value: {'pharmgkb': value} gene2pharmgkb = value_convert(gene2pharmgkb, fn, traverse_list=False) load_done('[%d]' % len(gene2pharmgkb)) return gene2pharmgkb
import os.path import time from biothings.utils.common import timesofar from utils.dataload import (load_start, load_done, tab2dict, tabfile_feeder, list2dict) from dataload import get_data_folder import logging # DATA_FOLDER = os.path.join(DATA_ARCHIVE_ROOT, 'by_resources/uniprot') DATA_FOLDER = get_data_folder('exac') def load_broadinstitute_exac_any(one_file, key): print("Loading file %s (%s)" % (one_file, key)) data = tab2dict(os.path.join(DATA_FOLDER, one_file), (0, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21), 0) exacs = {} for transcript in data: tupleexac = data[transcript] # remove version in key so we can search the dict easily later exacs[transcript.split(".")[0]] = { "exac": { "transcript": transcript, # but keep version here "n_exons": int(tupleexac[0]), "cds_start": int(tupleexac[1]), "cds_end": int(tupleexac[2]), "bp": int(tupleexac[3]), key: { "mu_syn": float(tupleexac[4]),
''' Populates MICROBE gene entries with genomic position data Currently updates the 120 microbial taxids that are NCBI Reference Sequences run get_ref_microbe_taxids function to get an updated file for TAXIDS_FILE when it's necessary. ''' import os.path from biothings.utils.common import (dump, loadobj, get_timestamp) from utils.dataload import (tab2list, load_start, load_done) from dataload import get_data_folder DATA_FOLDER = get_data_folder('entrez') print('DATA_FOLDER: ' + DATA_FOLDER) __metadata__ = { '__collection__': 'entrez_genomic_pos', } TAXIDS_FILE = os.path.join(DATA_FOLDER, "../ref_microbe_taxids.pyobj") DATAFILE = os.path.join(DATA_FOLDER, 'gene/gene2refseq.gz') def load_genedoc(self): """ Loads gene data from NCBI's refseq2gene.gz file. Parses it based on genomic position data and refseq status provided by the list of taxids from get_ref_microbe_taxids() as lookup table :return: """
import os.path import time from biothings.utils.common import get_timestamp, timesofar from utils.dataload import (load_start, load_done, tabfile_feeder, list2dict, value_convert, dict_convert) from dataload import get_data_folder DATA_FOLDER = get_data_folder('cpdb') def _download(__metadata__): from utils.dataload import download as _download output_folder = os.path.join(os.path.split(DATA_FOLDER)[0], get_timestamp()) for species in ['human', 'mouse', 'yeast']: url = __metadata__['__url_{}__'.format(species)] output_file = 'CPDB_pathways_genes_{}.tab'.format(species) _download(url, output_folder, output_file) def load_cpdb(__metadata__): # only import pathways from these sources PATHWAY_SOURCES_INCLUDED = __metadata__['pathway_sources_included'] VALID_COLUMN_NO = 4 t0 = time.time() print('DATA_FOLDER: ' + DATA_FOLDER) DATA_FILES = [] DATA_FILES.append(os.path.join(DATA_FOLDER, 'CPDB_pathways_genes_mouse.tab')) DATA_FILES.append(os.path.join(DATA_FOLDER, 'CPDB_pathways_genes_yeast.tab'))
import os.path import time from utils.common import timesofar from utils.dataload import (load_start, load_done, tab2dict, tabfile_feeder, list2dict) from dataload import get_data_folder # DATA_FOLDER = os.path.join(DATA_ARCHIVE_ROOT, 'by_resources/uniprot') DATA_FOLDER = os.path.join(get_data_folder('ucsc'), 'goldenPath/currentGenomes') def load_exons_for_species(species, exons_key='exons'): refflat_file = os.path.join(DATA_FOLDER, species, 'database/refFlat.txt.gz') reflink_file = os.path.join(DATA_FOLDER, species, 'database/refLink.txt.gz') load_start(refflat_file) t0 = time.time() refseq2gene = tab2dict(reflink_file, (2, 6), 0, alwayslist=False) ref2exons = [] for ld in tabfile_feeder(refflat_file, header=0): refseq = ld[1] chr = ld[2] if chr.startswith('chr'): chr = chr[3:] exons = zip([int(x) for x in ld[9].split(',') if x], [int(x) for x in ld[10].split(',') if x]) assert len(exons) == int(ld[8]), (len(exons), int(ld[8])) ref2exons.append((refseq, { 'chr': chr,
import os.path import time from biothings.utils.common import timesofar from utils.dataload import (load_start, load_done, tab2dict, tabfile_feeder, list2dict) from dataload import get_data_folder # DATA_FOLDER = os.path.join(DATA_ARCHIVE_ROOT, 'by_resources/uniprot') DATA_FOLDER = os.path.join(get_data_folder('ucsc'), 'goldenPath/currentGenomes') REFLINK_FILE = os.path.join(get_data_folder('ucsc'), 'goldenPath/hgFixed/database/refLink.txt.gz') refseq2gene = tab2dict(REFLINK_FILE, (2, 6), 0, alwayslist=False) def load_exons_for_species(species, exons_key='exons'): refflat_file = os.path.join(DATA_FOLDER, species, 'database/refFlat.txt.gz') load_start(refflat_file) t0 = time.time() ref2exons = {} for ld in tabfile_feeder(refflat_file, header=0): refseq = ld[1] chr = ld[2] if chr.startswith('chr'): chr = chr[3:] exons = list(zip([int(x) for x in ld[9].split(',') if x], [int(x) for x in ld[10].split(',') if x])) assert len(exons) == int(ld[8]), (len(exons), int(ld[8])) ref2exons.setdefault(refseq,[]).append({ 'transcript' : refseq,
''' Populates MICROBE gene entries with genomic position data Currently updates the 120 microbial taxids that are NCBI Reference Sequences run get_ref_microbe_taxids function to get an updated file for TAXIDS_FILE when it's necessary. ''' import os.path from biothings.utils.common import (dump, loadobj, get_timestamp) from utils.dataload import (tab2list, load_start, load_done) from dataload import get_data_folder DATA_FOLDER = get_data_folder('entrez') print('DATA_FOLDER: ' + DATA_FOLDER) __metadata__ = { '__collection__': 'entrez_genomic_pos', } TAXIDS_FILE = os.path.join(DATA_FOLDER, "../ref_microbe_taxids_20151014.pyobj") DATAFILE = os.path.join(DATA_FOLDER, 'gene/gene2refseq.gz') def load_genedoc(self): """ Loads gene data from NCBI's refseq2gene.gz file. Parses it based on genomic position data and refseq status provided by the list of taxids from get_ref_microbe_taxids() as lookup table :return: """ taxids = loadobj(TAXIDS_FILE)
import os.path import time from biothings.utils.common import timesofar from utils.dataload import (load_start, load_done, tab2dict, tabfile_feeder, list2dict) from dataload import get_data_folder # DATA_FOLDER = os.path.join(DATA_ARCHIVE_ROOT, 'by_resources/uniprot') DATA_FOLDER = os.path.join(get_data_folder('ucsc'), 'goldenPath/currentGenomes') REFLINK_FILE = os.path.join(get_data_folder('ucsc'), 'goldenPath/hgFixed/database/refLink.txt.gz') refseq2gene = tab2dict(REFLINK_FILE, (2, 6), 0, alwayslist=False) def load_exons_for_species(species, exons_key='exons'): refflat_file = os.path.join(DATA_FOLDER, species, 'database/refFlat.txt.gz') load_start(refflat_file) t0 = time.time() ref2exons = {} for ld in tabfile_feeder(refflat_file, header=0): refseq = ld[1] chr = ld[2] if chr.startswith('chr'): chr = chr[3:] exons = list( zip([int(x) for x in ld[9].split(',') if x],