Python DumperManager Beispiele

Beispiel #1

import biothings.dataload.uploader as uploader
import biothings.dataload.dumper as dumper
import biothings.databuild.builder as builder
import biothings.databuild.differ as differ
import biothings.databuild.syncer as syncer
import biothings.dataindex.indexer as indexer
from databuild.mapper import HasGeneMapper
from databuild.builder import TaxonomyDataBuilder
from dataindex.indexer import TaxonomyIndexer 

differ_manager = differ.DifferManager(job_manager=jmanager)
differ_manager.configure()
syncer_manager = syncer.SyncerManager(job_manager=jmanager)
syncer_manager.configure()

dmanager = dumper.DumperManager(job_manager=jmanager)
dmanager.register_sources(dataload.__sources__)
dmanager.schedule_all()
# manually register biothings source dumper
# this dumper will download whatever is necessary to update an ES index
from dataload.sources.biothings import BiothingsDumper
from biothings.utils.es import ESIndexer
from biothings.utils.backend import DocESBackend
BiothingsDumper.BIOTHINGS_APP = "t.biothings.io"
idxr = ESIndexer(index=config.ES_INDEX_NAME,doc_type=config.ES_DOC_TYPE,es_host=config.ES_HOST)
partial_backend = partial(DocESBackend,idxr)
BiothingsDumper.TARGET_BACKEND = partial_backend
dmanager.register_classes([BiothingsDumper])

# will check every 10 seconds for sources to upload
umanager = uploader.UploaderManager(poll_schedule = '* * * * * */10', job_manager=jmanager)

Beispiel #2

import config, biothings

biothings.config_for_app(config)

import contrib
import biothings.dataload.uploader as uploader
import biothings.dataload.dumper as dumper

# will check every 10 seconds for sources to upload
umanager = uploader.UploaderManager(poll_schedule='* * * * * */10',
                                    event_loop=loop)
umanager.register_sources(contrib.__sources_dict__)
umanager.poll()

dmanager = dumper.DumperManager(loop)
dmanager.register_sources(contrib.__sources_dict__)
dmanager.schedule_all()

from biothings.utils.hub import schedule

COMMANDS = {
    # dump commands
    "dm": dmanager,
    "dump": dmanager.dump_src,
    "dump_all": dmanager.dump_all,
    # upload commands
    "um": umanager,
    "upload": umanager.upload_src,
    "upload_all": umanager.upload_all,
    # admin/advanced