def rebuild_cache(self,
build_name=None,
sources=None,
target=None,
force_build=False):
"""Rebuild cache files for all sources involved in build_name, as well as
the latest merged collection found for that build"""
if build_name:
sources = mongo.get_source_fullnames(
self.managers["build_manager"].list_sources(build_name))
target = mongo.get_latest_build(build_name)
elif sources:
sources = mongo.get_source_fullnames(sources)
if not sources and not target:
raise Exception("No valid sources found")
def rebuild(col):
cur = mongo.id_feeder(col,
batch_size=10000,
logger=self.logger,
force_build=force_build)
[i for i in cur] # just iterate
@asyncio.coroutine
def do(srcs, tgt):
pinfo = {
"category": "cache",
"source": None,
"step": "rebuild",
"description": ""
}
self.logger.info("Rebuild cache for sources: %s, target: %s" %
(srcs, tgt))
for src in srcs:
# src can be a full name (eg. clinvar.clinvar_hg38) but id_feeder knows only name (clinvar_hg38)
if "." in src:
src = src.split(".")[1]
self.logger.info("Rebuilding cache for source '%s'" % src)
col = mongo.get_src_db()[src]
pinfo["source"] = src
job = yield from self.managers["job_manager"].defer_to_thread(
pinfo, partial(rebuild, col))
yield from job
self.logger.info("Done rebuilding cache for source '%s'" % src)
if tgt:
self.logger.info("Rebuilding cache for target '%s'" % tgt)
col = mongo.get_target_db()[tgt]
pinfo["source"] = tgt
job = self.managers["job_manager"].defer_to_thread(
pinfo, partial(rebuild, col))
yield from job
task = asyncio.ensure_future(do(sources, target))
return task
def rebuild_cache(build_name=None, sources=None, target=None):
"""Rebuild cache files for all sources involved in build_name, as well as
the latest merged collection found for that build"""
if build_name:
sources = mongo.get_source_fullnames(
build_manager.list_sources(build_name))
target = mongo.get_latest_build(build_name)
elif sources:
sources = mongo.get_source_fullnames(sources)
def rebuild(col):
cur = mongo.id_feeder(col,
batch_size=10000,
logger=config.logger,
force_build=True)
[i for i in cur] # just iterate
@asyncio.coroutine
def do(srcs, tgt):
pinfo = {
"category": "cache",
"source": None,
"step": "rebuild",
"description": ""
}
config.logger.info("Rebuild cache for sources: %s, target: %s" %
(srcs, tgt))
for src in srcs:
config.logger.info("Rebuilding cache for source '%s'" % src)
col = mongo.get_src_db()[src]
pinfo["source"] = src
job = yield from job_manager.defer_to_thread(
pinfo, partial(rebuild, col))
yield from job
config.logger.info("Done rebuilding cache for source '%s'" % src)
if tgt:
config.logger.info("Rebuilding cache for target '%s'" % tgt)
col = mongo.get_target_db()[tgt]
pinfo["source"] = tgt
job = job_manager.defer_to_thread(pinfo, partial(rebuild, col))
yield from job
task = asyncio.ensure_future(do(sources, target))
return task
def rebuild_cache(build_name=None,sources=None,target=None,force_build=False):
"""Rebuild cache files for all sources involved in build_name, as well as
the latest merged collection found for that build"""
if build_name:
sources = mongo.get_source_fullnames(build_manager.list_sources(build_name))
target = mongo.get_latest_build(build_name)
elif sources:
sources = mongo.get_source_fullnames(sources)
if not sources and not target:
raise Exception("No valid sources found")
def rebuild(col):
cur = mongo.id_feeder(col,batch_size=10000,logger=config.logger,force_build=force_build)
[i for i in cur] # just iterate
@asyncio.coroutine
def do(srcs,tgt):
pinfo = {"category" : "cache",
"source" : None,
"step" : "rebuild",
"description" : ""}
config.logger.info("Rebuild cache for sources: %s, target: %s" % (srcs,tgt))
for src in srcs:
# src can be a full name (eg. clinvar.clinvar_hg38) but id_feeder knows only name (clinvar_hg38)
if "." in src:
src = src.split(".")[1]
config.logger.info("Rebuilding cache for source '%s'" % src)
col = mongo.get_src_db()[src]
pinfo["source"] = src
job = yield from job_manager.defer_to_thread(pinfo, partial(rebuild,col))
yield from job
config.logger.info("Done rebuilding cache for source '%s'" % src)
if tgt:
config.logger.info("Rebuilding cache for target '%s'" % tgt)
col = mongo.get_target_db()[tgt]
pinfo["source"] = tgt
job = job_manager.defer_to_thread(pinfo, partial(rebuild,col))
yield from job
task = asyncio.ensure_future(do(sources,target))
return task