def main():
conf = CommandOptions.parse()
Logger.init(conf.logfilepath)
log = Logger.get()
conf.info(log)
if conf.logfilepath:
conf.info(sys.stdout)
if isinstance(conf.dst_conf, MongoConfig):
from mongosync.mongo.syncer import MongoSyncer
syncer = MongoSyncer(conf)
syncer.run()
elif isinstance(conf.dst_conf, EsConfig):
from mongosync.es.syncer import EsSyncer
syncer = EsSyncer(conf)
syncer.run()
else:
raise Exception('invalid config type')
log.info('exit')
import pymongo
import gevent
import mmh3
import mongo_utils
from mongosync.mongo.syncer import MongoHandler
from mongosync.logger import Logger
log = Logger.get()
class OplogVector(object):
""" A set of oplogs with same namespace.
"""
def __init__(self, dbname, collname):
self._dbname = dbname
self._collname = collname
self._oplogs = []
class MultiOplogReplayer(object):
""" Concurrent oplog replayer for MongoDB.
"""
def __init__(self, mc, n_writers=10, batch_size=40):
"""
Parameter:
- n_writers: maximum coroutine count
- batch_size: maximum oplog count in a batch, 40 is empiric value
"""
assert isinstance(mc, MongoHandler)
assert n_writers > 0
assert batch_size > 0
import sys
import time
import multiprocessing
import threading
import Queue
from mongosync.logger import Logger
log = Logger.get()
class Message(object):
""" Progress change message.
"""
def __init__(self, ns, cnt, done):
self.ns = ns
self.cnt = cnt
self.done = done
class Progress(object):
""" Progress attibutes.
"""
def __init__(self, ns, total):
self.ns = ns
self.curr = 0
self.total = total
self.start_time = time.time()
self.done = False
class LoggerThread(threading.Thread):
# summary: MongoDB sync tool
# author: caosiyang
# date: 2013/09/16
from gevent import monkey
monkey.patch_all()
import sys
from mongosync.command_options import CommandOptions
from mongosync.config import MongoConfig, EsConfig
from mongosync.logger import Logger
if __name__ == '__main__':
conf = CommandOptions.parse()
Logger.init(conf.logfilepath)
log = Logger.get()
conf.info(log)
if conf.logfilepath:
conf.info(sys.stdout)
if isinstance(conf.dst_conf, MongoConfig):
from mongosync.mongo.syncer import MongoSyncer
syncer = MongoSyncer(conf)
syncer.run()
elif isinstance(conf.dst_conf, EsConfig):
from mongosync.es.syncer import EsSyncer
syncer = EsSyncer(conf)
syncer.run()
else:
raise Exception('invalid config type')
# summary: MongoDB sync tool
# author: caosiyang
# date: 2013/09/16
import pymongo
from mongosync.mongo_synchronizer import MongoSynchronizer
from mongosync.command_options import CommandOptions
from mongosync.logger import Logger
from mongosync.mongo_helper import get_version
if __name__ == '__main__':
opts = CommandOptions()
opts.parse()
Logger.init(opts.logfilepath)
logger = Logger.get()
logger.info('================================================')
logger.info('src hostportstr : %s' % opts.src_hostportstr)
logger.info('src engine : %s' % opts.src_engine)
logger.info('src db version : %s' % get_version(opts.src_host, opts.src_port))
if opts.src_username or opts.src_password:
logger.info('src authdb : %s' % opts.src_authdb)
else:
logger.info('src authdb : ')
logger.info('src username : %s' % opts.src_username)
logger.info('src password : %s' % opts.src_password)
logger.info('dst hostportstr : %s' % opts.dst_hostportstr)
logger.info('dst db version : %s' % get_version(opts.dst_host, opts.dst_port))
if opts.dst_username or opts.dst_password: