def get_deprecated_genes(taxids=None):
if taxids is None:
taxids = set(get_all_taxa()) | {'36329'}
taxid_str = '{' + " ".join(['"' + x + '"' for x in taxids]) + '}'
# get all genes that DONT Have any sitelinks and dont have any item links to them
s = """SELECT DISTINCT ?entrez ?item ?prot WHERE
{
values ?taxids {taxid}
?taxon wdt:P685 ?taxids .
?item wdt:P351 ?entrez .
?item wdt:P703 ?taxon .
FILTER NOT EXISTS {?article schema:about ?item}
OPTIONAL {?item wdt:P688 ?prot}
FILTER NOT EXISTS {?something ?prop ?item }
}""".replace("{taxid}", taxid_str)
bindings = wdi_core.WDItemEngine.execute_sparql_query(
s)['results']['bindings']
entrez_qid = {
x['entrez']['value']: x['item']['value'].rsplit("/")[-1]
for x in bindings
}
gene_protein = {
x['item']['value'].rsplit("/")[-1]: x['prot']['value'].rsplit("/")[-1]
for x in bindings if 'prot' in x
}
print("{} wikidata".format(len(entrez_qid)))
wd = set(entrez_qid.keys())
mgd = MyGeneDownloader(fields="entrezgene")
docs, total = mgd.get_mg_cursor(",".join(taxids))
mygene = set([
str(x['entrezgene']) for x in tqdm(docs, total=total)
if "entrezgene" in x
])
print("{} mygene".format(len(mygene)))
missing = wd - mygene
print("{} deprecated".format(len(missing)))
qids = {entrez_qid[x] for x in missing}
# dont delete the protein items because often there is a new gene (that replaced this deprecated gene,
# that now encodes this protein. We should just check them, there are currently only 9 out of
# a thousand something deprecated genes
protein_qids = {gene_protein[x] for x in qids if x in gene_protein}
print("Check these protein items: {}".format(protein_qids))
# qids.update(protein_qids)
return qids
def main(taxid, metadata, log_dir="./logs", run_id=None, fast_run=True, write=True, entrez=None):
"""
Main function for creating/updating genes
:param taxid: taxon to use (ncbi tax id)
:type taxid: str
:param metadata: looks like: {"ensembl" : 84, "cpdb" : 31, "netaffy" : "na35", "ucsc" : "20160620", .. }
:type metadata: dict
:param log_dir: dir to store logs
:type log_dir: str
:param fast_run: use fast run mode
:type fast_run: bool
:param write: actually perform write
:type write: bool
:param entrez: Only run this one gene
:type entrez: int
:return: None
"""
# make sure the organism is found in wikidata
taxid = int(taxid)
organism_wdid = wdi_helpers.prop2qid("P685", str(taxid))
if not organism_wdid:
print("organism {} not found in wikidata".format(taxid))
return None
# login
login = wdi_login.WDLogin(user=WDUSER, pwd=WDPASS)
if wdi_core.WDItemEngine.logger is not None:
wdi_core.WDItemEngine.logger.handles = []
wdi_core.WDItemEngine.logger.handlers = []
run_id = run_id if run_id is not None else datetime.now().strftime('%Y%m%d_%H:%M')
log_name = '{}-{}.log'.format(__metadata__['name'], run_id)
__metadata__['taxid'] = taxid
wdi_core.WDItemEngine.setup_logging(log_dir=log_dir, logger_name='WD_logger', log_name=log_name,
header=json.dumps(__metadata__))
# get organism metadata (name, organism type, wdid)
# TODO: this can be pulled from wd
if taxid in organisms_info and organisms_info[taxid]['type'] != "microbial":
validate_type = 'eukaryotic'
organism_info = organisms_info[taxid]
# make sure all chromosome items are found in wikidata
cb = ChromosomeBot()
chr_num_wdid = cb.get_or_create(organism_info, login=login)
chr_num_wdid = {k.upper(): v for k, v in chr_num_wdid.items()}
if int(organism_info['taxid']) == 9606:
bot = HumanGeneBot(organism_info, chr_num_wdid, login)
else:
bot = ChromosomalGeneBot(organism_info, chr_num_wdid, login)
else:
# check if its one of the reference microbial genomes
# raises valueerror if not...
organism_info = mcb.get_organism_info(taxid)
refseq_qid_chrom = mcb.get_or_create_chromosomes(taxid, login)
print(organism_info)
bot = MicrobeGeneBot(organism_info, refseq_qid_chrom, login)
validate_type = "microbial"
# Get handle to mygene records
mgd = MyGeneDownloader()
if entrez:
doc, total = mgd.get_mg_gene(entrez)
docs = iter([doc])
else:
doc_filter = lambda x: (x.get("type_of_gene") != "biological-region") and ("entrezgene" in x)
docs, total = mgd.get_mg_cursor(taxid, doc_filter)
print("total number of records: {}".format(total))
# the scroll_id/cursor times out from mygene if we iterate. So.... get the whole thing now
docs = list(docs)
docs = HelperBot.validate_docs(docs, validate_type, PROPS['Entrez Gene ID'])
records = HelperBot.tag_mygene_docs(docs, metadata)
bot.run(records, total=total, fast_run=fast_run, write=write)
for frc in wdi_core.WDItemEngine.fast_run_store:
frc.clear()
print("done updating, waiting 10 min")
time.sleep(10 * 60)
releases = dict()
releases_to_remove = set()
last_updated = dict()
metadata = {k: v for k, v in metadata.items() if k in {'uniprot', 'ensembl', 'entrez'}}
for k, v in parse_mygene_src_version(metadata).items():
if "release" in v:
if k not in releases:
releases[k] = wdi_helpers.id_mapper('P393', (('P629', source_items[k]),))
to_remove = set(releases[k].values())
to_remove.discard(releases[k][v['release']])
releases_to_remove.update(to_remove)
print(
"{}: Removing releases: {}, keeping release: {}".format(k, ", ".join(set(releases[k]) - {v['release']}),
v['release']))
else:
last_updated[source_items[k]] = datetime.strptime(v["timestamp"], "%Y%m%d")
print(last_updated)
bot.cleanup(releases_to_remove, last_updated)
help="only run using this taxon (ncbi tax id). or 'microbe' for all microbes. comma separated",
type=str, required=True)
parser.add_argument('--fastrun', dest='fastrun', action='store_true')
parser.add_argument('--no-fastrun', dest='fastrun', action='store_false')
parser.add_argument('--entrez', help="Run only this one gene")
parser.set_defaults(fastrun=True)
args = parser.parse_args()
log_dir = args.log_dir if args.log_dir else "./logs"
run_id = datetime.now().strftime('%Y%m%d_%H:%M')
__metadata__['run_id'] = run_id
taxon = args.taxon
fast_run = args.fastrun
mcb = MicrobialChromosomeBot()
# get metadata about sources
mgd = MyGeneDownloader()
metadata = mgd.get_metadata()['src_version']
if args.entrez:
main(taxon, metadata, run_id=run_id, log_dir=log_dir, fast_run=fast_run,
write=not args.dummy, entrez=args.entrez)
sys.exit(0)
if "microbe" in taxon:
microbe_taxa = mcb.get_all_taxids()
taxon = taxon.replace("microbe", ','.join(map(str, microbe_taxa)))
for taxon1 in taxon.split(","):
try:
main(taxon1, metadata, run_id=run_id, log_dir=log_dir, fast_run=fast_run, write=not args.dummy)
except Exception as e:
def main(taxid, metadata, log_dir="./logs", run_id=None, fast_run=True, write=True, entrez=None):
"""
Main function for creating/updating proteins
:param taxid: taxon to use (ncbi tax id)
:type taxid: str
:param metadata: looks like: {"ensembl" : 84, "cpdb" : 31, "netaffy" : "na35", "ucsc" : "20160620", .. }
:type metadata: dict
:param log_dir: dir to store logs
:type log_dir: str
:param fast_run: use fast run mode
:type fast_run: bool
:param write: actually perform write
:type write: bool
:param entrez: Only run this one protein (given by entrezgene id)
:type entrez: int
:return: None
"""
# make sure the organism is found in wikidata
taxid = int(taxid)
organism_wdid = wdi_helpers.prop2qid("P685", str(taxid))
if not organism_wdid:
print("organism {} not found in wikidata".format(taxid))
return None
# login
login = wdi_login.WDLogin(user=WDUSER, pwd=WDPASS)
if wdi_core.WDItemEngine.logger is not None:
wdi_core.WDItemEngine.logger.handles = []
wdi_core.WDItemEngine.logger.handlers = []
run_id = run_id if run_id is not None else datetime.now().strftime('%Y%m%d_%H:%M')
log_name = '{}-{}.log'.format(__metadata__['name'], run_id)
__metadata__['taxid'] = taxid
wdi_core.WDItemEngine.setup_logging(log_dir=log_dir, log_name=log_name, header=json.dumps(__metadata__))
# get organism metadata (name, organism type, wdid)
if taxid in organisms_info:
validate_type = 'eukaryotic'
organism_info = organisms_info[taxid]
else:
# check if its one of the microbe refs
# raises valueerror if not...
organism_info = get_organism_info(taxid)
validate_type = 'microbial'
print(organism_info)
# get all entrez gene id -> wdid mappings, where found in taxon is this strain
gene_wdid_mapping = id_mapper("P351", (("P703", organism_info['wdid']),))
bot = ProteinBot(organism_info, gene_wdid_mapping, login)
# Get handle to mygene records
mgd = MyGeneDownloader()
if entrez:
doc, total = mgd.get_mg_gene(entrez)
docs = iter([doc])
else:
doc_filter = lambda x: (x.get("type_of_gene") == "protein-coding") and ("uniprot" in x) and ("entrezgene" in x)
docs, total = mgd.get_mg_cursor(taxid, doc_filter)
print("total number of records: {}".format(total))
# the scroll_id/cursor times out from mygene if we iterate. So.... get the whole thing now
docs = list(docs)
docs = HelperBot.validate_docs(docs, validate_type, PROPS['Entrez Gene ID'])
records = HelperBot.tag_mygene_docs(docs, metadata)
bot.run(records, total=total, fast_run=fast_run, write=write)
for frc in wdi_core.WDItemEngine.fast_run_store:
frc.clear()
time.sleep(10 * 60)
releases = dict()
releases_to_remove = set()
last_updated = dict()
metadata = {k: v for k, v in metadata.items() if k in {'uniprot', 'ensembl', 'entrez'}}
for k, v in parse_mygene_src_version(metadata).items():
if "release" in v:
if k not in releases:
releases[k] = wdi_helpers.id_mapper('P393', (('P629', source_items[k]),))
to_remove = set(releases[k].values())
to_remove.discard(releases[k][v['release']])
releases_to_remove.update(to_remove)
print(
"{}: Removing releases: {}, keeping release: {}".format(k, ", ".join(set(releases[k]) - {v['release']}),
v['release']))
else:
last_updated[source_items[k]] = datetime.strptime(v["timestamp"], "%Y%m%d")
print(last_updated)
bot.cleanup(releases_to_remove, last_updated)
# after the run is done, disconnect the logging handler
# so that if we start another, it doesn't write twice
if wdi_core.WDItemEngine.logger is not None:
wdi_core.WDItemEngine.logger.handles = []
def main(metadata, log_dir="./logs", fast_run=True, write=True):
"""
Main function for creating/updating genes
:param metadata: looks like: {"ensembl" : 84, "cpdb" : 31, "netaffy" : "na35", "ucsc" : "20160620", .. }
:type metadata: dict
:param log_dir: dir to store logs
:type log_dir: str
:param fast_run: use fast run mode
:type fast_run: bool
:param write: actually perform write
:type write: bool
:return: None
"""
# login
login = wdi_login.WDLogin(user=WDUSER, pwd=WDPASS)
wdi_core.WDItemEngine.setup_logging(log_dir=log_dir,
logger_name='WD_logger',
log_name=log_name,
header=json.dumps(__metadata__))
# get all ids mappings
entrez_wdid = wdi_helpers.id_mapper(PROPS['Entrez Gene ID'])
wdid_entrez = {v: k for k, v in entrez_wdid.items()}
homo_wdid = wdi_helpers.id_mapper(PROPS['HomoloGene ID'],
return_as_set=True)
wdid_homo = dict()
for h**o, wdids in homo_wdid.items():
for wdid in wdids:
wdid_homo[wdid] = h**o
entrez_homo = {
wdid_entrez[wdid]: h**o
for wdid, h**o in wdid_homo.items() if wdid in wdid_entrez
}
taxon_wdid = wdi_helpers.id_mapper(PROPS['NCBI Taxonomy ID'])
# only do certain records
mgd = MyGeneDownloader(
q="_exists_:homologene AND type_of_gene:protein-coding",
fields=','.join(['taxid', 'homologene', 'entrezgene']))
docs, total = mgd.query()
docs = list(tqdm(docs, total=total))
records = HelperBot.tag_mygene_docs(docs, metadata)
# group together all orthologs
# d[taxid][entrezgene] = { set of entrezgene ids for orthologs }
d = defaultdict(lambda: defaultdict(set))
entrez_taxon = dict() # keep this for the qualifier on the statements
for doc in records:
this_taxid = doc['taxid']['@value']
this_entrez = doc['entrezgene']['@value']
entrez_taxon[str(this_entrez)] = str(this_taxid)
if str(this_entrez) not in entrez_wdid:
continue
for taxid, entrez in doc['homologene']['@value']['genes']:
if taxid == 4932 and this_taxid == 559292:
# ridiculous workaround because entrez has the taxid for the strain and homologene has it for the species
# TODO: This needs to be fixed if you want to use other things that may have species/strains .. ?`
continue
if taxid != this_taxid and str(entrez) in entrez_wdid:
d[str(this_taxid)][str(this_entrez)].add(str(entrez))
print("taxid: # of genes : {}".format({k: len(v) for k, v in d.items()}))
homogene_ver = metadata['homologene']
release = wdi_helpers.Release(
"HomoloGene build{}".format(homogene_ver),
"Version of HomoloGene",
homogene_ver,
edition_of_wdid='Q468215',
archive_url='ftp://ftp.ncbi.nih.gov/pub/HomoloGene/build{}/'.format(
homogene_ver)).get_or_create(login)
reference = lambda homogeneid: [
wdi_core.WDItemID(release, PROPS['stated in'], is_reference=True),
wdi_core.WDExternalID(
homogeneid, PROPS['HomoloGene ID'], is_reference=True)
]
ec = 0
for taxid, subd in tqdm(d.items()):
for entrezgene, orthologs in tqdm(subd.items(), leave=False):
try:
do_item(entrezgene, orthologs, reference, entrez_homo,
entrez_taxon, taxon_wdid, entrez_wdid, login, write)
except Exception as e:
wdi_helpers.format_msg(entrezgene, PROPS['Entrez Gene ID'],
None, str(e), type(e))
ec += 1
# clear the fast run store once we move on to the next taxon
wdi_core.WDItemEngine.fast_run_store = []
wdi_core.WDItemEngine.fast_run_container = None
print("Completed succesfully with {} exceptions".format(ec))
parser = argparse.ArgumentParser(description='run wikidata gene bot')
parser.add_argument('--log-dir', help='directory to store logs', type=str)
parser.add_argument('--dummy',
help='do not actually do write',
action='store_true')
parser.add_argument('--fastrun', dest='fastrun', action='store_true')
parser.add_argument('--no-fastrun', dest='fastrun', action='store_false')
parser.set_defaults(fastrun=True)
args = parser.parse_args()
log_dir = args.log_dir if args.log_dir else "./logs"
run_id = datetime.now().strftime('%Y%m%d_%H:%M')
__metadata__['run_id'] = run_id
fast_run = args.fastrun
# get metadata about sources
mgd = MyGeneDownloader()
metadata = dict()
src = mgd.get_metadata()['src']
for source in src.keys():
metadata[source] = src[source]["version"]
log_name = '{}-{}.log'.format(__metadata__['name'], run_id)
if wdi_core.WDItemEngine.logger is not None:
wdi_core.WDItemEngine.logger.handles = []
wdi_core.WDItemEngine.setup_logging(log_dir=log_dir,
log_name=log_name,
header=json.dumps(__metadata__),
logger_name='orthologs')
main(metadata, log_dir=log_dir, fast_run=fast_run, write=not args.dummy)