def configure_job_manager(self):
import asyncio
loop = asyncio.get_event_loop()
from biothings.utils.manager import JobManager
args = self.mixargs("job",{"num_workers":config.HUB_MAX_WORKERS,"max_memory_usage":config.HUB_MAX_MEM_USAGE})
job_manager = JobManager(loop,**args)
self.managers["job_manager"] = job_manager
def test():
from biothings.utils.manager import JobManager
from biothings.hub.dataindex.indexer import IndexManager
loop = asyncio.get_event_loop()
job_manager = JobManager(loop)
index_manager = IndexManager(job_manager=job_manager)
index_manager.configure(config.INDEX_CONFIG)
snapshot_manager = SnapshotManager(index_manager=index_manager,
job_manager=job_manager,
poll_schedule="* * * * * */10")
snapshot_manager.configure(config.SNAPSHOT_CONFIG)
# snapshot_manager.poll("snapshot",snapshot_manager.snapshot_build)
async def test_code():
snapshot_manager.snapshot('prod',
'mynews_202009170234_fjvg7skx',
steps="post")
asyncio.ensure_future(test_code())
loop.run_forever()
import config, biothings
biothings.config_for_app(config)
import logging
# shut some mouths...
logging.getLogger("elasticsearch").setLevel(logging.ERROR)
logging.getLogger("urllib3").setLevel(logging.ERROR)
logging.getLogger("requests").setLevel(logging.ERROR)
from biothings.utils.manager import JobManager
loop = asyncio.get_event_loop()
process_queue = concurrent.futures.ProcessPoolExecutor(max_workers=config.HUB_MAX_WORKERS)
thread_queue = concurrent.futures.ThreadPoolExecutor()
loop.set_default_executor(process_queue)
jmanager = JobManager(loop,
process_queue, thread_queue,
max_memory_usage=None,
)
import dataload
import biothings.dataload.uploader as uploader
import biothings.dataload.dumper as dumper
import biothings.databuild.builder as builder
import biothings.databuild.differ as differ
import biothings.databuild.syncer as syncer
import biothings.dataindex.indexer as indexer
from databuild.mapper import HasGeneMapper
from databuild.builder import TaxonomyDataBuilder
from dataindex.indexer import TaxonomyIndexer
differ_manager = differ.DifferManager(job_manager=jmanager)
differ_manager.configure()
biothings.config_for_app(config)
from biothings.utils.manager import JobManager
loop = asyncio.get_event_loop()
process_queue = concurrent.futures.ProcessPoolExecutor(
max_workers=config.HUB_MAX_WORKERS)
thread_queue = concurrent.futures.ThreadPoolExecutor()
loop.set_default_executor(process_queue)
max_mem = type(
config.HUB_MAX_MEM_USAGE
) == int and config.HUB_MAX_MEM_USAGE * 1024**3 or config.HUB_MAX_MEM_USAGE
job_manager = JobManager(
loop,
process_queue,
thread_queue,
max_memory_usage=max_mem,
)
import dataload
import biothings.dataload.uploader as uploader
import biothings.dataload.dumper as dumper
import biothings.databuild.builder as builder
import biothings.databuild.differ as differ
import biothings.databuild.syncer as syncer
import biothings.dataindex.indexer as indexer
from databuild.builder import MyVariantDataBuilder
from databuild.mapper import TagObserved
from dataindex.indexer import VariantIndexer
# will check every 10 seconds for sources to upload
logging.getLogger("boto").setLevel(logging.ERROR)
logging.getLogger("keylookup").setLevel(logging.INFO)
logging.info("Hub DB backend: %s" % biothings.config.HUB_DB_BACKEND)
logging.info("Hub database: %s" % biothings.config.DATA_HUB_DB_DATABASE)
from biothings.utils.hub import start_server, HubShell
from biothings.utils.manager import JobManager
loop = asyncio.get_event_loop()
process_queue = concurrent.futures.ProcessPoolExecutor(
max_workers=config.HUB_MAX_WORKERS)
thread_queue = concurrent.futures.ThreadPoolExecutor()
loop.set_default_executor(process_queue)
job_manager = JobManager(loop,
num_workers=config.HUB_MAX_WORKERS,
max_memory_usage=config.HUB_MAX_MEM_USAGE)
shell = HubShell(job_manager)
import hub.dataload
import biothings.hub.dataload.uploader as uploader
import biothings.hub.dataload.dumper as dumper
import biothings.hub.dataload.source as source
import biothings.hub.databuild.builder as builder
import biothings.hub.databuild.differ as differ
import biothings.hub.databuild.syncer as syncer
import biothings.hub.dataindex.indexer as indexer
import biothings.hub.datainspect.inspector as inspector
from biothings.hub.api.manager import APIManager
from hub.databuild.builder import MyGeneDataBuilder