import biothings.dataload.uploader as uploader import biothings.dataload.dumper as dumper import biothings.databuild.builder as builder import biothings.databuild.differ as differ import biothings.databuild.syncer as syncer import biothings.dataindex.indexer as indexer from databuild.mapper import HasGeneMapper from databuild.builder import TaxonomyDataBuilder from dataindex.indexer import TaxonomyIndexer differ_manager = differ.DifferManager(job_manager=jmanager) differ_manager.configure() syncer_manager = syncer.SyncerManager(job_manager=jmanager) syncer_manager.configure() dmanager = dumper.DumperManager(job_manager=jmanager) dmanager.register_sources(dataload.__sources__) dmanager.schedule_all() # manually register biothings source dumper # this dumper will download whatever is necessary to update an ES index from dataload.sources.biothings import BiothingsDumper from biothings.utils.es import ESIndexer from biothings.utils.backend import DocESBackend BiothingsDumper.BIOTHINGS_APP = "t.biothings.io" idxr = ESIndexer(index=config.ES_INDEX_NAME,doc_type=config.ES_DOC_TYPE,es_host=config.ES_HOST) partial_backend = partial(DocESBackend,idxr) BiothingsDumper.TARGET_BACKEND = partial_backend dmanager.register_classes([BiothingsDumper]) # will check every 10 seconds for sources to upload umanager = uploader.UploaderManager(poll_schedule = '* * * * * */10', job_manager=jmanager)
import config, biothings
biothings.config_for_app(config)
import contrib
import biothings.dataload.uploader as uploader
import biothings.dataload.dumper as dumper
# will check every 10 seconds for sources to upload
umanager = uploader.UploaderManager(poll_schedule='* * * * * */10',
event_loop=loop)
umanager.register_sources(contrib.__sources_dict__)
umanager.poll()
dmanager = dumper.DumperManager(loop)
dmanager.register_sources(contrib.__sources_dict__)
dmanager.schedule_all()
from biothings.utils.hub import schedule
COMMANDS = {
# dump commands
"dm": dmanager,
"dump": dmanager.dump_src,
"dump_all": dmanager.dump_all,
# upload commands
"um": umanager,
"upload": umanager.upload_src,
"upload_all": umanager.upload_all,
# admin/advanced