def __init__(self, conf):
if not isinstance(conf, Config):
raise RuntimeError('invalid config type')
self._conf = conf
self._ignore_dbs = ['admin', 'local']
self._ignore_colls = [
'system.indexes', 'system.profile', 'system.users'
]
if conf.optime_logfilepath:
self._optime_logger = OptimeLogger(conf.optime_logfilepath)
else:
self._optime_logger = None
self._optime_log_interval = 10 # default 10s
self._last_optime = None # optime of the last oplog was applied
self._last_optime_logtime = time.time()
self._log_interval = 2 # default 2s
self._last_logtime = time.time() # use in oplog replay
# for large collections
self._n_workers = 8 # multi-process
self._large_coll_docs = 1000000 # 100w
self._initial_sync_start_optime = None
self._initial_sync_end_optime = None
self._stage = Stage.STOPPED
def parse():
""" Parse command options and generate config.
"""
conf = Config()
parser = argparse.ArgumentParser(description='Sync data from a replica-set to another MongoDB/Elasticsearch.')
parser.add_argument('-f', '--config', nargs='?', required=False, help='configuration file, note that command options will override items in config file')
parser.add_argument('--src', nargs='?', required=False, help='source should be hostportstr of a replica-set member')
parser.add_argument('--src-authdb', nargs='?', required=False, help="src authentication database, default is 'admin'")
parser.add_argument('--src-username', nargs='?', required=False, help='src username')
parser.add_argument('--src-password', nargs='?', required=False, help='src password')
parser.add_argument('--dst', nargs='?', required=False, help='destination should be hostportstr of a mongos or mongod instance')
parser.add_argument('--dst-authdb', nargs='?', required=False, help="dst authentication database, default is 'admin', for MongoDB")
parser.add_argument('--dst-username', nargs='?', required=False, help='dst username, for MongoDB')
parser.add_argument('--dst-password', nargs='?', required=False, help='dst password, for MongoDB')
parser.add_argument('--start-optime', type=int, nargs='?', required=False, help='timestamp in second, indicates oplog based increment sync')
parser.add_argument('--optime-logfile', nargs='?', required=False, help="optime log file path, use this as start optime if without '--start-optime'")
parser.add_argument('--logfile', nargs='?', required=False, help='log file path')
args = parser.parse_args()
if args.config is not None:
conf = ConfigFile.load(args.config)
if args.src is not None:
conf.src_conf.hosts = args.src
if args.src_authdb is not None:
conf.src_conf.authdb = args.src_authdb
if args.src_username is not None:
conf.src_conf.username = args.src_username
if args.src_password is not None:
conf.src_conf.password = args.src_password
if args.dst is not None:
conf.dst_conf.hosts = args.dst
if args.dst_authdb is not None:
conf.dst_conf.authdb = args.dst_authdb
if args.dst_username is not None:
conf.dst_conf.username = args.dst_username
if args.dst_password is not None:
conf.dst_conf.password = args.dst_password
if args.start_optime is not None:
conf.start_optime = Timestamp(args.start_optime, 0)
if args.optime_logfile is not None:
conf.optime_logfilepath = args.optime_logfile
if args.start_optime is None:
optime_logger = OptimeLogger(args.optime_logfile)
conf.start_optime = optime_logger.read()
if args.logfile is not None:
conf.logfilepath = args.logfile
return conf
def parse():
""" Parse command options and generate config.
"""
conf = Config()
parser = argparse.ArgumentParser(description='Sync data from a replica-set to another MongoDB/Elasticsearch.')
parser.add_argument('-f', '--config', nargs='?', required=False, help='configuration file, note that command options will override items in config file')
parser.add_argument('--src', nargs='?', required=False, help='source should be hostportstr of a replica-set member')
parser.add_argument('--src-authdb', nargs='?', required=False, help="src authentication database, default is 'admin'")
parser.add_argument('--src-username', nargs='?', required=False, help='src username')
parser.add_argument('--src-password', nargs='?', required=False, help='src password')
parser.add_argument('--dst', nargs='?', required=False, help='destination should be hostportstr of a mongos or mongod instance')
parser.add_argument('--dst-authdb', nargs='?', required=False, help="dst authentication database, default is 'admin', for MongoDB")
parser.add_argument('--dst-username', nargs='?', required=False, help='dst username, for MongoDB')
parser.add_argument('--dst-password', nargs='?', required=False, help='dst password, for MongoDB')
parser.add_argument('--start-optime', type=int, nargs='?', required=False, help='timestamp in second, indicates oplog based increment sync')
parser.add_argument('--optime-logfile', nargs='?', required=False, help="optime log file path, use this as start optime if without '--start-optime'")
parser.add_argument('--logfile', nargs='?', required=False, help='log file path')
args = parser.parse_args()
if args.config is not None:
conf = ConfigFile.load(args.config)
if args.src is not None:
conf.src_conf.hosts = args.src
if args.src_authdb is not None:
conf.src_conf.authdb = args.src_authdb
if args.src_username is not None:
conf.src_conf.username = args.src_username
if args.src_password is not None:
conf.src_conf.password = args.src_password
if args.dst is not None:
conf.dst_conf.hosts = args.dst
if args.dst_authdb is not None:
conf.dst_conf.authdb = args.dst_authdb
if args.dst_username is not None:
conf.dst_conf.username = args.dst_username
if args.dst_password is not None:
conf.dst_conf.password = args.dst_password
if args.start_optime is not None:
conf.start_optime = Timestamp(args.start_optime, 0)
if args.optime_logfile is not None:
conf.optime_logfilepath = args.optime_logfile
if args.start_optime is None:
optime_logger = OptimeLogger(args.optime_logfile)
conf.start_optime = optime_logger.read()
if args.logfile is not None:
conf.logfilepath = args.logfile
return conf
def __init__(self, conf):
if not isinstance(conf, Config):
raise Exception('invalid config type')
self._conf = conf
self._ignore_dbs = ['admin', 'local']
self._ignore_colls = [
'system.indexes', 'system.profile', 'system.users'
]
if conf.optime_logfilepath:
self._optime_logger = OptimeLogger(conf.optime_logfilepath)
else:
self._optime_logger = None
self._optime_log_interval = 10 # default 10s
self._last_optime = None # optime of the last oplog has been replayed
self._last_optime_logtime = time.time()
self._log_interval = 2 # default 2s
self._last_logtime = time.time() # use in oplog replay
class CommonSyncer(object):
""" Common database synchronizer.
Specific database synchronizer should implement the following methods:
- __init__
- _initial_sync
- _sync_collection
- _sync_large_collection
- _replay_oplog
"""
def __init__(self, conf):
if not isinstance(conf, Config):
raise RuntimeError('invalid config type')
self._conf = conf
self._ignore_dbs = ['admin', 'local']
self._ignore_colls = [
'system.indexes', 'system.profile', 'system.users'
]
if conf.optime_logfilepath:
self._optime_logger = OptimeLogger(conf.optime_logfilepath)
else:
self._optime_logger = None
self._optime_log_interval = 10 # default 10s
self._last_optime = None # optime of the last oplog was applied
self._last_optime_logtime = time.time()
self._log_interval = 2 # default 2s
self._last_logtime = time.time() # use in oplog replay
# for large collections
self._n_workers = 8 # multi-process
self._large_coll_docs = 1000000 # 100w
self._initial_sync_start_optime = None
self._initial_sync_end_optime = None
self._stage = Stage.STOPPED
@property
def from_to(self):
return "%s => %s" % (self._conf.src_hostportstr,
self._conf.dst_hostportstr)
@property
def log_interval(self):
return self._log_interval
@log_interval.setter
def log_interval(self, n_secs):
if n_secs < 0:
n_secs = 0
self._log_interval = n_secs
def run(self):
""" Start to sync.
"""
# never drop database automatically
# clear data manually if necessary
try:
self._sync()
except KeyboardInterrupt:
log.info('keyboard interrupt')
def _sync(self):
""" Sync databases and oplog.
"""
if self._conf.start_optime:
log.info("locating oplog, it will take a while")
doc = None
cur = self._src.client()['local']['oplog.rs'].find(
{
'ts': {
'$lte': self._conf.start_optime
}
}, {
"ts": 1
}).sort("$natural", -1).limit(1)
try:
doc = cur.next()
except StopIteration:
pass
if not doc:
log.error('oplog is stale')
return
start_optime = doc['ts']
log.info('start timestamp is %s actually' % start_optime)
self._stage = Stage.OPLOG_SYNC
self._replay_oplog(start_optime)
else:
# initial sync
self._initial_sync_start_optime = get_optime(self._src.client())
self._stage = Stage.INITIAL_SYNC
self._initial_sync()
self._stage = Stage.POST_INITIAL_SYNC
self._initial_sync_end_optime = get_optime(self._src.client())
# oplog sync
if self._optime_logger:
self._optime_logger.write(self._initial_sync_start_optime)
self._replay_oplog(self._initial_sync_start_optime)
def _collect_colls(self):
""" Collect collections to sync.
"""
colls = []
for dbname in self._src.client().database_names():
if dbname in self._ignore_dbs:
continue
if not self._conf.data_filter.valid_db(dbname):
continue
for collname in self._src.client()[dbname].collection_names(
include_system_collections=False):
if collname in self._ignore_colls:
continue
if not self._conf.data_filter.valid_coll(dbname, collname):
continue
colls.append((dbname, collname))
return colls
def _split_coll(self, namespace_tuple, n_partitions):
""" Split a collection into n partitions.
Return a list of split points.
splitPointCount = partitionCount - 1
splitPointCount = keyTotalCount / (keyCount + 1)
keyCount = maxChunkSize / (2 * avgObjSize)
=>
maxChunkSize = (keyTotalCount / (partionCount - 1) - 1) * 2 * avgObjSize
Note: maxChunkObjects is default 250000.
"""
if n_partitions <= 1:
raise RuntimeError('n_partitions need greater than 1, but %s' %
n_partitions)
dbname, collname = namespace_tuple
ns = '.'.join(namespace_tuple)
db = self._src.client()[dbname]
collstats = db.command('collstats', collname)
if 'avgObjSize' not in collstats: # empty collection
return []
n_points = n_partitions - 1
max_chunk_size = int(
((collstats['count'] /
(n_partitions - 1) - 1) * 2 * collstats['avgObjSize']) / 1024 /
1024)
if max_chunk_size <= 0:
return []
res = db.command('splitVector',
ns,
keyPattern={'_id': 1},
maxSplitPoints=n_points,
maxChunkSize=max_chunk_size,
maxChunkObjects=collstats['count'])
if res['ok'] != 1:
return []
else:
return [doc['_id'] for doc in res['splitKeys']]
def _initial_sync(self):
""" Initial sync.
"""
def classify(ns_tuple, large_colls, small_colls):
""" Find out large and small collections.
"""
if self._is_large_collection(ns_tuple):
points = self._split_coll(ns_tuple, self._n_workers)
if points:
large_colls.append((ns_tuple, points))
else:
small_colls.append(ns_tuple)
else:
small_colls.append(ns_tuple)
large_colls = []
small_colls = []
pool = gevent.pool.Pool(8)
colls = self._collect_colls()
for ns in colls:
dbname, collname = ns
log.info('%d\t%s.%s' %
(self._src.client()[dbname][collname].count(), dbname,
collname))
pool.spawn(classify, ns, large_colls, small_colls)
pool.join()
if len(large_colls) + len(small_colls) != len(colls):
raise RuntimeError('classify collections error')
log.info('large collections: %s' %
['.'.join(ns) for ns, points in large_colls])
log.info('small collections: %s' %
['.'.join(ns) for ns in small_colls])
# create progress logger
self._progress_logger = LoggerThread(len(colls))
self._progress_logger.start()
# small collections first
pool = gevent.pool.Pool(8)
for res in pool.imap(self._sync_collection, small_colls):
if res is not None:
sys.exit(1)
# then large collections
for ns, points in large_colls:
self._sync_large_collection(ns, points)
def _sync_collection(self, namespace_tuple):
""" Sync a collection until success.
"""
raise NotImplementedError(
'you should implement %s.%s' %
(self.__class__.__name__, self._sync_collection.__name__))
def _is_large_collection(self, namespace_tuple):
""" Check if large collection or not.
"""
dbname, collname = namespace_tuple
return True if self._src.client()[dbname][collname].count(
) > self._large_coll_docs else False
def _sync_large_collection(self, namespace_tuple, split_points):
""" Sync large collection until success.
"""
raise NotImplementedError(
'you should implement %s.%s' %
(self.__class__.__name__, self._sync_large_collection.__name__))
def _replay_oplog(self, oplog_start):
""" Replay oplog.
"""
raise NotImplementedError(
'you should implement %s.%s' %
(self.__class__.__name__, self._replay_oplog.__name__))
def _log_progress(self, tag=''):
""" Print progress periodically.
"""
now = time.time()
if now - self._last_logtime >= self._log_interval:
delay = now - self._last_optime.time
time_unit = 'second' if delay <= 1 else 'seconds'
if tag:
log.info(
'%s - sync to %s - %d %s delay - %s - %s' %
(self.from_to,
datetime.datetime.fromtimestamp(self._last_optime.time),
delay, time_unit, self._last_optime, tag))
else:
log.info(
'%s - sync to %s - %d %s delay - %s' %
(self.from_to,
datetime.datetime.fromtimestamp(self._last_optime.time),
delay, time_unit, self._last_optime))
self._last_logtime = now
def _log_optime(self, optime):
""" Record optime periodically.
"""
if not self._optime_logger:
return
now = time.time()
if now - self._last_optime_logtime >= self._optime_log_interval:
self._optime_logger.write(optime)
self._last_optime_logtime = now
log.info("flush optime into file '%s': %s" %
(self._optime_logger.filepath, optime))
def load(filepath):
""" Load config file and generate conf.
"""
conf = Config()
tml = toml.load(filepath)
conf.src_conf = MongoConfig(tml['src']['hosts'],
tml['src'].get('authdb', 'admin'),
tml['src'].get('username', ''),
tml['src'].get('password', ''))
if type not in tml['dst'] or tml['dst']['type'] == 'mongo':
conf.dst_conf = MongoConfig(tml['dst']['hosts'],
tml['dst'].get('authdb', 'admin'),
tml['dst'].get('username', ''),
tml['dst'].get('password', ''))
elif tml['dst']['type'] == 'es':
conf.dst_conf = EsConfig(tml['dst']['hosts'])
else:
raise Exception('invalid dst.type')
if 'sync' in tml and 'dbs' in tml['sync']:
for dbentry in tml['sync']['dbs']:
if 'db' not in dbentry:
raise Exception("'db' is missing in sync.dbs")
if not dbentry['db']:
raise Exception("'db' is empty in sync.dbs")
dbname = dbentry['db'].strip()
rename_db = dbentry['rename_db'].strip(
) if 'rename_db' in dbentry else ""
# update db map
if dbname and rename_db:
if dbname in conf.dbmap:
raise Exception('duplicate dbname in sync.dbs: %s' %
dbname)
conf.dbmap[dbname] = rename_db
if 'colls' in dbentry and dbentry['colls']:
for collentry in dbentry['colls']:
if isinstance(collentry, str):
collname = collentry.strip()
ns = gen_namespace(dbname, collname)
conf.data_filter.add_include_coll(ns)
elif isinstance(collentry, dict):
if 'coll' not in collentry:
raise Exception(
"'coll' is missing in sync.dbs.colls")
if not collentry['coll']:
raise Exception(
"'coll' is empty in sync.dbs.colls")
collname = collentry['coll'].strip()
fields = frozenset(
[f.strip()
for f in collentry['fields']] if 'fields' in
collentry else [])
# update coll filter
ns = gen_namespace(dbname, collname)
conf.data_filter.add_include_coll(ns)
# update fields
if fields:
if ns in conf.fieldmap:
raise Exception(
"duplicate collname in sync.dbs.colls: %s"
% ns)
conf.fieldmap[ns] = fields
else:
raise Exception(
'invalid entry in sync.dbs.colls: %s' %
collentry)
else:
# update coll filter
conf.data_filter.add_include_coll(
gen_namespace(dbname, '*'))
if 'sync' in tml and 'start_optime' in tml['sync']:
conf.start_optime = Timestamp(tml['sync']['start_optime'], 0)
if 'log' in tml and 'filepath' in tml['log']:
conf.logfilepath = tml['log']['filepath']
if 'log' in tml and 'op_time_path' in tml['log']:
conf.optime_logfilepath = tml['log']['op_time_path']
optime_logger = OptimeLogger(conf.optime_logfilepath)
if optime_logger.read():
conf.start_optime = optime_logger.read()
return conf
class Synchronizer(object):
""" Common synchronizer.
Other synchronizer entities should implement methods:
- __init__
- __del__
- _sync_database
- _sync_collection
- _sync_oplog
"""
def __init__(self, conf):
if not isinstance(conf, Config):
raise Exception('invalid config type')
self._conf = conf
self._ignore_dbs = ['admin', 'local']
self._ignore_colls = [
'system.indexes', 'system.profile', 'system.users'
]
if conf.optime_logfilepath:
self._optime_logger = OptimeLogger(conf.optime_logfilepath)
else:
self._optime_logger = None
self._optime_log_interval = 10 # default 10s
self._last_optime = None # optime of the last oplog has been replayed
self._last_optime_logtime = time.time()
self._log_interval = 2 # default 2s
self._last_logtime = time.time() # use in oplog replay
@property
def from_to(self):
return "%s => %s" % (self._conf.src_hostportstr,
self._conf.dst_hostportstr)
@property
def log_interval(self):
return self._log_interval
@log_interval.setter
def log_interval(self, n_secs):
if n_secs < 0:
n_secs = 0
self._log_interval = n_secs
def run(self):
""" Start to sync.
"""
# never drop database automatically
# you should clear the databases manually if necessary
try:
self._sync()
except exceptions.KeyboardInterrupt:
log.info('keyboard interrupt')
def _sync(self):
""" Sync databases and oplog.
"""
if self._conf.start_optime:
# TODO optimize
log.info("locating oplog, it will take a while")
oplog_start = self._conf.start_optime
doc = self._src.client()['local']['oplog.rs'].find_one(
{'ts': {
'$gte': oplog_start
}})
if not doc:
log.error('no oplogs newer than the specified oplog')
return
oplog_start = doc['ts']
log.info('start timestamp is %s actually' % oplog_start)
self._last_optime = oplog_start
self._sync_oplog(oplog_start)
else:
oplog_start = get_optime(self._src.client())
if not oplog_start:
log.error('get oplog_start failed, terminate')
sys.exit(1)
self._last_optime = oplog_start
self._sync_databases()
if self._optime_logger:
self._optime_logger.write(oplog_start)
log.info('first %s' % oplog_start)
self._sync_oplog(oplog_start)
def _sync_databases(self):
""" Sync databases excluding 'admin' and 'local'.
"""
host, port = self._src.client().address
log.info('sync databases from %s:%d' % (host, port))
for dbname in self._src.client().database_names():
if dbname in self._ignore_dbs:
log.info("skip database '%s'" % dbname)
continue
if not self._conf.data_filter.valid_db(dbname):
log.info("skip database '%s'" % dbname)
continue
self._sync_database(dbname)
log.info('all databases done')
def _sync_database(self, dbname):
""" Sync a database.
"""
raise Exception(
'you should implement %s.%s' %
(self.__class__.__name__, self._sync_database.__name__))
def _sync_collections(self, dbname):
""" Sync collections in the database excluding system collections.
"""
collnames = self._src.client()[dbname].collection_names(
include_system_collections=False)
for collname in collnames:
if collname in self._ignore_colls:
log.info("skip collection '%s'" %
gen_namespace(dbname, collname))
continue
if not self._conf.data_filter.valid_coll(dbname, collname):
log.info("skip collection '%s'" %
gen_namespace(dbname, collname))
continue
self._sync_collection(dbname, collname)
def _sync_collection(self, dbname, collname):
""" Sync a collection until success.
"""
raise Exception(
'you should implement %s.%s' %
(self.__class__.__name__, self._sync_collection.__name__))
def _sync_oplog(self, oplog_start):
""" Replay oplog.
"""
raise Exception('you should implement %s.%s' %
(self.__class__.__name__, self._sync_oplog.__name__))
def _log_progress(self, tag=''):
""" Print progress.
"""
now = time.time()
if now - self._last_logtime >= self._log_interval:
delay = now - self._last_optime.time
time_unit = 'second' if delay <= 1 else 'seconds'
if tag:
log.info(
'%s - sync to %s - %d %s delay - %s - %s' %
(self.from_to,
datetime.datetime.fromtimestamp(self._last_optime.time),
delay, time_unit, self._last_optime, tag))
else:
log.info(
'%s - sync to %s - %d %s delay - %s' %
(self.from_to,
datetime.datetime.fromtimestamp(self._last_optime.time),
delay, time_unit, self._last_optime))
self._last_logtime = now
def _log_optime(self, optime):
""" Record optime.
"""
if not self._optime_logger:
return
now = time.time()
if now - self._last_optime_logtime >= self._optime_log_interval:
self._optime_logger.write(optime)
self._last_optime_logtime = now
log.info("flush optime into file '%s': %s" %
(self._optime_logger.filepath, optime))