def __init__(self, src_hostportstr, dst_hostportstr, **kwargs): """ Constructor. """ if not src_hostportstr: raise Exception('src hostportstr is empty') if not dst_hostportstr: raise Exception('dst hostportstr is empty') self._src_mc = None self._dst_mc = None self._filter = None self._query = None self._w = 1 # write concern, default 1 self._start_optime = None # if true, only sync oplog self._last_optime = None # optime of the last oplog has been replayed self._logger = logging.getLogger() self._ignore_dbs = ['admin', 'local'] self._ignore_colls = ['system.indexes', 'system.profile', 'system.users'] self._src_engine = kwargs.get('src_engine') self._src_username = kwargs.get('src_username') self._src_password = kwargs.get('src_password') self._dst_username = kwargs.get('dst_username') self._dst_password = kwargs.get('dst_password') self._collections = kwargs.get('collections') self._ignore_indexes = kwargs.get('ignore_indexes') self._query = kwargs.get('query', None) self._start_optime = kwargs.get('start_optime') self._w = kwargs.get('write_concern', 1) if self._collections: self._filter = filter.CollectionFilter() self._filter.add_target_collections(self._collections) # init src mongo client self._src_host = src_hostportstr.split(':')[0] self._src_port = int(src_hostportstr.split(':')[1]) self._src_mc = mongo_helper.mongo_connect( self._src_host, self._src_port, username=self._src_username, password=self._src_password, w=self._w) # init dst mongo client self._dst_host = dst_hostportstr.split(':')[0] self._dst_port = int(dst_hostportstr.split(':')[1]) self._dst_mc = mongo_helper.mongo_connect( self._dst_host, self._dst_port, username=self._dst_username, password=self._dst_password, w=self._w)
def initializer(host, port, dbname, collname): """ Subprocess initial function. """ global g_mc, g_logger try: g_mc = mongo_connect(host, port, w=1) g_logger = logging.getLogger() except Exception as e: g_logger.error(e)
def reconnect(self, host, port, **kwargs): """ Try to reconnect until success. """ while True: try: self._logger.info('try to reconnect %s:%d' % (host, port)) mc = mongo_helper.mongo_connect(host, port, **kwargs) except: pass else: return mc
def reconnect(self, host, port, **kwargs): """ Try to reconnect until success. """ while True: try: self._logger.info('try to reconnect %s:%d' % (host, port)) mc = mongo_helper.mongo_connect(host, port, **kwargs) mc.database_names( ) # excute command to confirm connection availability return mc except Exception as e: self._logger.error('reconnect failed: %s' % e)
def __init__(self, src_hostportstr, dst_hostportstr, **kwargs): """ Constructor. """ if not src_hostportstr: raise Exception('source hostportstr is empty') if not dst_hostportstr: raise Exception('destination hostportstr is empty') self._src_mc = None self._dst_mc = None self._filter = None self._query = None self._current_optime = None # optime of the last oplog has been replayed self._w = 1 # write concern self._start_optime = None # if true, only sync oplog self._logger = logging.getLogger() self._dst_username = kwargs.get('dst_username') self._dst_password = kwargs.get('dst_password') self._collections = kwargs.get('collections') self._ignore_indexes = kwargs.get('ignore_indexes') self._query = kwargs.get('query') self._start_optime = kwargs.get('start_optime') self._w = kwargs.get('write_concern', 1) if self._collections: self._filter = filter.CollectionFilter() self._filter.add_target_collections(self._collections) # init source mongo clients self._src_host = src_hostportstr.split(':')[0] self._src_port = int(src_hostportstr.split(':')[1]) self._src_mc = mongo_helper.mongo_connect(self._src_host, self._src_port, w=self._w) # init destination mongo client self._dst_host = dst_hostportstr.split(':')[0] self._dst_port = int(dst_hostportstr.split(':')[1]) self._dst_mc = mongo_helper.mongo_connect(self._dst_host, self._dst_port, w=self._w) if self._dst_username and self._dst_password: self._dst_mc.admin.authenticate(self._dst_username, self._dst_password)
def reconnect(self, host, port, **kwargs): """ Try to reconnect until done. """ while True: try: self._logger.info('try to reconnect %s:%d' % (host, port)) mc = mongo_helper.mongo_connect(host, port, **kwargs) mc.database_names() # test connection self._logger.info('reconnect ok') return mc except Exception as e: mc.close() self._logger.error(e) time.sleep(1)
def __init__(self, src_hostportstr, dst_hostportstr, **kwargs): """ Constructor. """ if not src_hostportstr: raise Exception('src hostportstr is empty') if not dst_hostportstr: raise Exception('dst hostportstr is empty') self._src_hostportstr = src_hostportstr self._src_host, self._src_port = mongo_helper.parse_hostportstr( src_hostportstr) self._dst_hostportstr = dst_hostportstr self._dst_host, self._dst_port = mongo_helper.parse_hostportstr( dst_hostportstr) self._src_engine = kwargs.get('src_engine', 'mongodb') self._src_authdb = kwargs.get('src_authdb', 'admin') self._src_username = kwargs.get('src_username', '') self._src_password = kwargs.get('src_password', '') self._dst_authdb = kwargs.get('dst_authdb', 'admin') self._dst_username = kwargs.get('dst_username', '') self._dst_password = kwargs.get('dst_password', '') self._dbs = kwargs.get('dbs', []) self._colls = kwargs.get('colls', []) self._src_db = kwargs.get('src_db', '') self._dst_db = kwargs.get('dst_db', '') self._ignore_indexes = kwargs.get('ignore_indexes', False) self._start_optime = kwargs.get('start_optime') self._asyncio = kwargs.get('asyncio', False) self._logger = Logger.get() self._src_version = mongo_helper.get_version(self._src_host, self._src_port) self._dst_version = mongo_helper.get_version(self._dst_host, self._dst_port) self._ignore_dbs = ['admin', 'local'] self._ignore_colls = [ 'system.indexes', 'system.profile', 'system.users' ] self._last_optime = None # optime of the last oplog has been replayed self._last_logtime = None # use in oplog replay self._log_interval = 1 self._rename_db_mode = True if self._src_db and self._dst_db else False if self._rename_db_mode: assert len(self._dbs) == 0 self._dbs.append(self._src_db) self._filter = None if self._colls: self._filter = filter.CollectionFilter() self._filter.add_target_collections(self._colls) elif self._dbs: self._filter = filter.DatabaseFilter() self._filter.add_target_databases(self._dbs) # init src mongo client self._src_mc = mongo_helper.mongo_connect(self._src_host, self._src_port, authdb=self._src_authdb, username=self._src_username, password=self._src_password, w=1) # init dst mongo client self._dst_mc = mongo_helper.mongo_connect(self._dst_host, self._dst_port, authdb=self._dst_authdb, username=self._dst_username, password=self._dst_password, w=1) self._dst_is_mongos = self._dst_mc.is_mongos
def __init__(self, src_hostportstr, dst_hostportstr, **kwargs): """ Constructor. """ if not src_hostportstr: raise Exception('src hostportstr is empty') if not dst_hostportstr: raise Exception('dst hostportstr is empty') self._src_mc = None self._dst_mc = None self._filter = None self._query = None self._w = 1 # write concern, default 1 self._start_optime = None # if true, only sync oplog self._last_optime = None # optime of the last oplog has been replayed self._logger = logging.getLogger() self._ignore_dbs = ['admin', 'local'] self._ignore_colls = [ 'system.indexes', 'system.profile', 'system.users' ] self._src_engine = kwargs.get('src_engine') self._src_authdb = kwargs.get('src_authdb') self._src_username = kwargs.get('src_username') self._src_password = kwargs.get('src_password') self._dst_authdb = kwargs.get('dst_authdb') self._dst_username = kwargs.get('dst_username') self._dst_password = kwargs.get('dst_password') self._dbs = kwargs.get('dbs') self._colls = kwargs.get('colls') self._query = kwargs.get('query', None) self._ignore_indexes = kwargs.get('ignore_indexes') self._start_optime = kwargs.get('start_optime') self._w = kwargs.get('write_concern', 1) if self._colls: self._filter = filter.CollectionFilter() self._filter.add_target_collections(self._colls) elif self._dbs: self._filter = filter.DatabaseFilter() self._filter.add_target_databases(self._dbs) # init src mongo client self._src_host = src_hostportstr.split(':')[0] self._src_port = int(src_hostportstr.split(':')[1]) self._src_mc = mongo_helper.mongo_connect(self._src_host, self._src_port, authdb=self._src_authdb, username=self._src_username, password=self._src_password, w=self._w) # init dst mongo client self._dst_host = dst_hostportstr.split(':')[0] self._dst_port = int(dst_hostportstr.split(':')[1]) self._dst_mc = mongo_helper.mongo_connect(self._dst_host, self._dst_port, authdb=self._dst_authdb, username=self._dst_username, password=self._dst_password, w=self._w)
def main(): global g_src, g_src_username, g_src_password, g_dst, g_dst_username, g_dst_password parse_args() print '=' * 48 print 'src : %s' % g_src print 'src username : %s' % g_src_username print 'src password : %s' % g_src_password print 'dst : %s' % g_dst print 'dst username : %s' % g_dst_username print 'dst password : %s' % g_dst_password print '=' * 48 src_host = g_src.split(':')[0] src_port = int(g_src.split(':')[1]) src_mc = mongo_helper.mongo_connect( src_host, src_port, username=g_src_username, password=g_src_password) dst_host = g_dst.split(':')[0] dst_port = int(g_dst.split(':')[1]) dst_mc = mongo_helper.mongo_connect( dst_host, dst_port, username=g_dst_username, password=g_dst_password) ignore_dbs = ['admin', 'local'] ignore_colls = ['system.users', 'system.profile'] src_version = src_mc['admin'].command('serverStatus')['version'] dst_version = dst_mc['admin'].command('serverStatus')['version'] if src_version.startswith('2') and dst_version.startswith('3'): ignore_colls.append('system.indexes') # validate data data_pass = True print '-' * 96 print '%s%s%s%s' % ('RESULT'.ljust(16), 'COLL'.ljust(48), 'SRC'.rjust(16), 'DST'.rjust(16)) print '-' * 96 for dbname in sorted(src_mc.database_names()): if dbname not in ignore_dbs: for collname in sorted(src_mc[dbname].collection_names()): if collname not in ignore_colls: src_coll_cnt = src_mc[dbname][collname].count() dst_coll_cnt = dst_mc[dbname][collname].count() if src_coll_cnt == dst_coll_cnt: res = 'OK' else: res = 'ERR' data_pass = False print '%s%s%s%s' % (res.ljust(16), (dbname + '.' + collname).ljust(48), str(src_coll_cnt).rjust(16), str(dst_coll_cnt).rjust(16)) print '-' * 96 if data_pass: print 'data pass' else: print 'data not pass' # validate index index_pass = True print '-' * 128 print '%s%s%s' % ('RESULT'.ljust(16), 'COLL'.ljust(48), 'INDEX'.ljust(64)) print '-' * 128 for dbname in sorted(src_mc.database_names()): if dbname not in ignore_dbs: for collname in sorted(src_mc[dbname].collection_names()): if collname not in ignore_colls: src_index_info = src_mc[dbname][collname].index_information() dst_index_info = dst_mc[dbname][collname].index_information() src_index_names = set() dst_index_names = set() for index_items in src_index_info.itervalues(): index_name = get_standard_index_name(index_items) src_index_names.add(index_name) for index_items in dst_index_info.itervalues(): index_name = get_standard_index_name(index_items) dst_index_names.add(index_name) for index_name in src_index_names: if index_name in dst_index_names: res = 'OK' else: res = 'ERR' index_pass = False print '%s%s%s' % (res.ljust(16), (dbname + '.' + collname).ljust(48), index_name.ljust(64)) print '-' * 128 if index_pass: print 'index pass' else: print 'index not pass'