def __init__(self, src_hostportstr, dst_hostportstr, **kwargs):
""" Constructor.
"""
if not src_hostportstr:
raise Exception('src hostportstr is empty')
if not dst_hostportstr:
raise Exception('dst hostportstr is empty')
self._src_mc = None
self._dst_mc = None
self._filter = None
self._query = None
self._w = 1 # write concern, default 1
self._start_optime = None # if true, only sync oplog
self._last_optime = None # optime of the last oplog has been replayed
self._logger = logging.getLogger()
self._ignore_dbs = ['admin', 'local']
self._ignore_colls = ['system.indexes', 'system.profile', 'system.users']
self._src_engine = kwargs.get('src_engine')
self._src_username = kwargs.get('src_username')
self._src_password = kwargs.get('src_password')
self._dst_username = kwargs.get('dst_username')
self._dst_password = kwargs.get('dst_password')
self._collections = kwargs.get('collections')
self._ignore_indexes = kwargs.get('ignore_indexes')
self._query = kwargs.get('query', None)
self._start_optime = kwargs.get('start_optime')
self._w = kwargs.get('write_concern', 1)
if self._collections:
self._filter = filter.CollectionFilter()
self._filter.add_target_collections(self._collections)
# init src mongo client
self._src_host = src_hostportstr.split(':')[0]
self._src_port = int(src_hostportstr.split(':')[1])
self._src_mc = mongo_helper.mongo_connect(
self._src_host,
self._src_port,
username=self._src_username,
password=self._src_password,
w=self._w)
# init dst mongo client
self._dst_host = dst_hostportstr.split(':')[0]
self._dst_port = int(dst_hostportstr.split(':')[1])
self._dst_mc = mongo_helper.mongo_connect(
self._dst_host,
self._dst_port,
username=self._dst_username,
password=self._dst_password,
w=self._w)
def initializer(host, port, dbname, collname):
""" Subprocess initial function.
"""
global g_mc, g_logger
try:
g_mc = mongo_connect(host, port, w=1)
g_logger = logging.getLogger()
except Exception as e:
g_logger.error(e)
def initializer(host, port, dbname, collname):
""" Subprocess initial function.
"""
global g_mc, g_logger
try:
g_mc = mongo_connect(host, port, w=1)
g_logger = logging.getLogger()
except Exception as e:
g_logger.error(e)
def reconnect(self, host, port, **kwargs):
""" Try to reconnect until success.
"""
while True:
try:
self._logger.info('try to reconnect %s:%d' % (host, port))
mc = mongo_helper.mongo_connect(host, port, **kwargs)
except:
pass
else:
return mc
def reconnect(self, host, port, **kwargs):
""" Try to reconnect until success.
"""
while True:
try:
self._logger.info('try to reconnect %s:%d' % (host, port))
mc = mongo_helper.mongo_connect(host, port, **kwargs)
mc.database_names(
) # excute command to confirm connection availability
return mc
except Exception as e:
self._logger.error('reconnect failed: %s' % e)
def __init__(self, src_hostportstr, dst_hostportstr, **kwargs):
""" Constructor.
"""
if not src_hostportstr:
raise Exception('source hostportstr is empty')
if not dst_hostportstr:
raise Exception('destination hostportstr is empty')
self._src_mc = None
self._dst_mc = None
self._filter = None
self._query = None
self._current_optime = None # optime of the last oplog has been replayed
self._w = 1 # write concern
self._start_optime = None # if true, only sync oplog
self._logger = logging.getLogger()
self._dst_username = kwargs.get('dst_username')
self._dst_password = kwargs.get('dst_password')
self._collections = kwargs.get('collections')
self._ignore_indexes = kwargs.get('ignore_indexes')
self._query = kwargs.get('query')
self._start_optime = kwargs.get('start_optime')
self._w = kwargs.get('write_concern', 1)
if self._collections:
self._filter = filter.CollectionFilter()
self._filter.add_target_collections(self._collections)
# init source mongo clients
self._src_host = src_hostportstr.split(':')[0]
self._src_port = int(src_hostportstr.split(':')[1])
self._src_mc = mongo_helper.mongo_connect(self._src_host, self._src_port, w=self._w)
# init destination mongo client
self._dst_host = dst_hostportstr.split(':')[0]
self._dst_port = int(dst_hostportstr.split(':')[1])
self._dst_mc = mongo_helper.mongo_connect(self._dst_host, self._dst_port, w=self._w)
if self._dst_username and self._dst_password:
self._dst_mc.admin.authenticate(self._dst_username, self._dst_password)
def reconnect(self, host, port, **kwargs):
""" Try to reconnect until done.
"""
while True:
try:
self._logger.info('try to reconnect %s:%d' % (host, port))
mc = mongo_helper.mongo_connect(host, port, **kwargs)
mc.database_names() # test connection
self._logger.info('reconnect ok')
return mc
except Exception as e:
mc.close()
self._logger.error(e)
time.sleep(1)
def __init__(self, src_hostportstr, dst_hostportstr, **kwargs):
""" Constructor.
"""
if not src_hostportstr:
raise Exception('src hostportstr is empty')
if not dst_hostportstr:
raise Exception('dst hostportstr is empty')
self._src_hostportstr = src_hostportstr
self._src_host, self._src_port = mongo_helper.parse_hostportstr(
src_hostportstr)
self._dst_hostportstr = dst_hostportstr
self._dst_host, self._dst_port = mongo_helper.parse_hostportstr(
dst_hostportstr)
self._src_engine = kwargs.get('src_engine', 'mongodb')
self._src_authdb = kwargs.get('src_authdb', 'admin')
self._src_username = kwargs.get('src_username', '')
self._src_password = kwargs.get('src_password', '')
self._dst_authdb = kwargs.get('dst_authdb', 'admin')
self._dst_username = kwargs.get('dst_username', '')
self._dst_password = kwargs.get('dst_password', '')
self._dbs = kwargs.get('dbs', [])
self._colls = kwargs.get('colls', [])
self._src_db = kwargs.get('src_db', '')
self._dst_db = kwargs.get('dst_db', '')
self._ignore_indexes = kwargs.get('ignore_indexes', False)
self._start_optime = kwargs.get('start_optime')
self._asyncio = kwargs.get('asyncio', False)
self._logger = Logger.get()
self._src_version = mongo_helper.get_version(self._src_host,
self._src_port)
self._dst_version = mongo_helper.get_version(self._dst_host,
self._dst_port)
self._ignore_dbs = ['admin', 'local']
self._ignore_colls = [
'system.indexes', 'system.profile', 'system.users'
]
self._last_optime = None # optime of the last oplog has been replayed
self._last_logtime = None # use in oplog replay
self._log_interval = 1
self._rename_db_mode = True if self._src_db and self._dst_db else False
if self._rename_db_mode:
assert len(self._dbs) == 0
self._dbs.append(self._src_db)
self._filter = None
if self._colls:
self._filter = filter.CollectionFilter()
self._filter.add_target_collections(self._colls)
elif self._dbs:
self._filter = filter.DatabaseFilter()
self._filter.add_target_databases(self._dbs)
# init src mongo client
self._src_mc = mongo_helper.mongo_connect(self._src_host,
self._src_port,
authdb=self._src_authdb,
username=self._src_username,
password=self._src_password,
w=1)
# init dst mongo client
self._dst_mc = mongo_helper.mongo_connect(self._dst_host,
self._dst_port,
authdb=self._dst_authdb,
username=self._dst_username,
password=self._dst_password,
w=1)
self._dst_is_mongos = self._dst_mc.is_mongos
def __init__(self, src_hostportstr, dst_hostportstr, **kwargs):
""" Constructor.
"""
if not src_hostportstr:
raise Exception('src hostportstr is empty')
if not dst_hostportstr:
raise Exception('dst hostportstr is empty')
self._src_mc = None
self._dst_mc = None
self._filter = None
self._query = None
self._w = 1 # write concern, default 1
self._start_optime = None # if true, only sync oplog
self._last_optime = None # optime of the last oplog has been replayed
self._logger = logging.getLogger()
self._ignore_dbs = ['admin', 'local']
self._ignore_colls = [
'system.indexes', 'system.profile', 'system.users'
]
self._src_engine = kwargs.get('src_engine')
self._src_authdb = kwargs.get('src_authdb')
self._src_username = kwargs.get('src_username')
self._src_password = kwargs.get('src_password')
self._dst_authdb = kwargs.get('dst_authdb')
self._dst_username = kwargs.get('dst_username')
self._dst_password = kwargs.get('dst_password')
self._dbs = kwargs.get('dbs')
self._colls = kwargs.get('colls')
self._query = kwargs.get('query', None)
self._ignore_indexes = kwargs.get('ignore_indexes')
self._start_optime = kwargs.get('start_optime')
self._w = kwargs.get('write_concern', 1)
if self._colls:
self._filter = filter.CollectionFilter()
self._filter.add_target_collections(self._colls)
elif self._dbs:
self._filter = filter.DatabaseFilter()
self._filter.add_target_databases(self._dbs)
# init src mongo client
self._src_host = src_hostportstr.split(':')[0]
self._src_port = int(src_hostportstr.split(':')[1])
self._src_mc = mongo_helper.mongo_connect(self._src_host,
self._src_port,
authdb=self._src_authdb,
username=self._src_username,
password=self._src_password,
w=self._w)
# init dst mongo client
self._dst_host = dst_hostportstr.split(':')[0]
self._dst_port = int(dst_hostportstr.split(':')[1])
self._dst_mc = mongo_helper.mongo_connect(self._dst_host,
self._dst_port,
authdb=self._dst_authdb,
username=self._dst_username,
password=self._dst_password,
w=self._w)
def main():
global g_src, g_src_username, g_src_password, g_dst, g_dst_username, g_dst_password
parse_args()
print '=' * 48
print 'src : %s' % g_src
print 'src username : %s' % g_src_username
print 'src password : %s' % g_src_password
print 'dst : %s' % g_dst
print 'dst username : %s' % g_dst_username
print 'dst password : %s' % g_dst_password
print '=' * 48
src_host = g_src.split(':')[0]
src_port = int(g_src.split(':')[1])
src_mc = mongo_helper.mongo_connect(
src_host,
src_port,
username=g_src_username,
password=g_src_password)
dst_host = g_dst.split(':')[0]
dst_port = int(g_dst.split(':')[1])
dst_mc = mongo_helper.mongo_connect(
dst_host,
dst_port,
username=g_dst_username,
password=g_dst_password)
ignore_dbs = ['admin', 'local']
ignore_colls = ['system.users', 'system.profile']
src_version = src_mc['admin'].command('serverStatus')['version']
dst_version = dst_mc['admin'].command('serverStatus')['version']
if src_version.startswith('2') and dst_version.startswith('3'):
ignore_colls.append('system.indexes')
# validate data
data_pass = True
print '-' * 96
print '%s%s%s%s' % ('RESULT'.ljust(16), 'COLL'.ljust(48), 'SRC'.rjust(16), 'DST'.rjust(16))
print '-' * 96
for dbname in sorted(src_mc.database_names()):
if dbname not in ignore_dbs:
for collname in sorted(src_mc[dbname].collection_names()):
if collname not in ignore_colls:
src_coll_cnt = src_mc[dbname][collname].count()
dst_coll_cnt = dst_mc[dbname][collname].count()
if src_coll_cnt == dst_coll_cnt:
res = 'OK'
else:
res = 'ERR'
data_pass = False
print '%s%s%s%s' % (res.ljust(16), (dbname + '.' + collname).ljust(48), str(src_coll_cnt).rjust(16), str(dst_coll_cnt).rjust(16))
print '-' * 96
if data_pass:
print 'data pass'
else:
print 'data not pass'
# validate index
index_pass = True
print '-' * 128
print '%s%s%s' % ('RESULT'.ljust(16), 'COLL'.ljust(48), 'INDEX'.ljust(64))
print '-' * 128
for dbname in sorted(src_mc.database_names()):
if dbname not in ignore_dbs:
for collname in sorted(src_mc[dbname].collection_names()):
if collname not in ignore_colls:
src_index_info = src_mc[dbname][collname].index_information()
dst_index_info = dst_mc[dbname][collname].index_information()
src_index_names = set()
dst_index_names = set()
for index_items in src_index_info.itervalues():
index_name = get_standard_index_name(index_items)
src_index_names.add(index_name)
for index_items in dst_index_info.itervalues():
index_name = get_standard_index_name(index_items)
dst_index_names.add(index_name)
for index_name in src_index_names:
if index_name in dst_index_names:
res = 'OK'
else:
res = 'ERR'
index_pass = False
print '%s%s%s' % (res.ljust(16), (dbname + '.' + collname).ljust(48), index_name.ljust(64))
print '-' * 128
if index_pass:
print 'index pass'
else:
print 'index not pass'
