def __init__(self, conf): CommonSyncer.__init__(self, conf) if not isinstance(self._conf.src_conf, MongoConfig): raise RuntimeError('invalid src config type') self._src = MongoHandler(self._conf.src_conf) if not self._src.connect(): raise RuntimeError('connect to mongodb(src) failed: %s' % self._conf.src_hostportstr) if not isinstance(self._conf.dst_conf, MongoConfig): raise RuntimeError('invalid dst config type') self._dst = MongoHandler(self._conf.dst_conf) if not self._dst.connect(): raise RuntimeError('connect to mongodb(dst) failed: %s' % self._conf.dst_hostportstr) self._multi_oplog_replayer = MultiOplogReplayer(self._dst, 10)
def __init__(self, conf): CommonSyncer.__init__(self, conf) if not isinstance(self._conf.src_conf, MongoConfig): raise Exception('invalid src config type') self._src = MongoHandler(self._conf.src_conf) if not self._src.connect(): raise Exception('connect to mongodb(src) failed: %s' % self._conf.src_hostportstr) if not isinstance(self._conf.dst_conf, EsConfig): raise Exception('invalid dst config type') self._dst = EsHandler(self._conf.dst_conf) if not self._dst.connect(): raise Exception('connect to elasticsearch(dst) failed: %s' % self._conf.dst_hostportstr) self._action_buf = [] # used to bulk write oplogs self._last_bulk_optime = None
def __init__(self, conf): CommonSyncer.__init__(self, conf) if not isinstance(self._conf.src_conf, MongoConfig): raise Exception('invalid src config type') self._src = MongoHandler(self._conf.src_conf) if not self._src.connect(): raise Exception('connect to mongodb(src) failed: %s' % self._conf.src_hostportstr) if not isinstance(self._conf.dst_conf, EsConfig): raise Exception('invalid dst config type') self._dst = EsHandler(self._conf.dst_conf) if not self._dst.connect(): raise Exception('connect to elasticsearch(dst) failed: %s' % self._conf.dst_hostportstr) self._action_buf = [] # used to bulk write oplogs self._last_bulk_optime = None
class MongoSyncer(CommonSyncer): """ MongoDB synchronizer. """ def __init__(self, conf): CommonSyncer.__init__(self, conf) if not isinstance(self._conf.src_conf, MongoConfig): raise RuntimeError('invalid src config type') self._src = MongoHandler(self._conf.src_conf) if not self._src.connect(): raise RuntimeError('connect to mongodb(src) failed: %s' % self._conf.src_hostportstr) if not isinstance(self._conf.dst_conf, MongoConfig): raise RuntimeError('invalid dst config type') self._dst = MongoHandler(self._conf.dst_conf) if not self._dst.connect(): raise RuntimeError('connect to mongodb(dst) failed: %s' % self._conf.dst_hostportstr) self._multi_oplog_replayer = MultiOplogReplayer(self._dst, 10) def _create_index(self, namespace_tuple): """ Create indexes. """ def format(key_direction_list): """ Format key and direction of index. """ res = [] for key, direction in key_direction_list: if isinstance(direction, float) or isinstance(direction, long): direction = int(direction) res.append((key, direction)) return res dbname, collname = namespace_tuple dst_dbname, dst_collname = self._conf.db_coll_mapping(dbname, collname) index_info = self._src.client()[dbname][collname].index_information() for name, info in index_info.iteritems(): keys = info['key'] options = {} options['name'] = name if 'unique' in info: options['unique'] = info['unique'] if 'sparse' in info: options['sparse'] = info['sparse'] if 'expireAfterSeconds' in info: options['expireAfterSeconds'] = info['expireAfterSeconds'] if 'partialFilterExpression' in info: options['partialFilterExpression'] = info[ 'partialFilterExpression'] if 'dropDups' in info: options['dropDups'] = info['dropDups'] ## create indexes before import documents, so not need 'background' option # if 'background' in info: # options['background'] = info['background'] # for text index if 'weights' in info: options['weights'] = info['weights'] if 'default_language' in info: options['default_language'] = info['default_language'] if 'language_override' in info: options['language_override'] = info['language_override'] self._dst.create_index(dst_dbname, dst_collname, format(keys), **options) def _sync_collection(self, namespace_tuple): """ Sync a collection until success. """ # create indexes first self._create_index(namespace_tuple) src_dbname, src_collname = namespace_tuple dst_dbname, dst_collname = self._conf.db_coll_mapping( src_dbname, src_collname) src_ns = '%s.%s' % (src_dbname, src_collname) total = self._src.client()[src_dbname][src_collname].count() self._progress_logger.register(src_ns, total) while True: try: cursor = self._src.client()[src_dbname][src_collname].find( filter=None, cursor_type=pymongo.cursor.CursorType.EXHAUST, no_cursor_timeout=True, modifiers={'$snapshot': True}) reqs = [] reqs_max = 100 groups = [] groups_max = 10 n = 0 for doc in cursor: reqs.append( pymongo.ReplaceOne({'_id': doc['_id']}, doc, upsert=True)) if len(reqs) == reqs_max: groups.append(reqs) reqs = [] if len(groups) == groups_max: threads = [ gevent.spawn(self._dst.bulk_write, dst_dbname, dst_collname, groups[i], ordered=False, ignore_duplicate_key_error=True) for i in xrange(groups_max) ] gevent.joinall(threads, raise_error=True) groups = [] n += 1 if n % 10000 == 0: self._progress_logger.add(src_ns, n) n = 0 if len(groups) > 0: threads = [ gevent.spawn(self._dst.bulk_write, dst_dbname, dst_collname, groups[i], ordered=False, ignore_duplicate_key_error=True) for i in xrange(len(groups)) ] gevent.joinall(threads, raise_error=True) if len(reqs) > 0: self._dst.bulk_write(dst_dbname, dst_collname, reqs, ordered=False, ignore_duplicate_key_error=True) self._progress_logger.add(src_ns, n, done=True) return except pymongo.errors.AutoReconnect: self._src.reconnect() def _sync_large_collection(self, namespace_tuple, split_points): """ Sync large collection. """ # create indexes first self._create_index(namespace_tuple) dbname, collname = namespace_tuple ns = '.'.join(namespace_tuple) log.info('pending to sync %s with %d processes' % (ns, len(split_points) + 1)) coll = self._src.client()[dbname][collname] total = coll.count() self._progress_logger.register(ns, total) prog_q = multiprocessing.Queue() res_q = multiprocessing.Queue() proc_logging = multiprocessing.Process(target=logging_progress, args=(ns, total, prog_q)) proc_logging.start() queries = [] lower_bound = None for point in split_points: if lower_bound is None: queries.append({'_id': {'$lt': point}}) else: queries.append({'_id': {'$gte': lower_bound, '$lt': point}}) lower_bound = point queries.append({'_id': {'$gte': lower_bound}}) procs = [] for query in queries: p = multiprocessing.Process( target=self._sync_collection_with_query, args=(namespace_tuple, query, prog_q, res_q)) p.start() procs.append(p) log.info('start process %s with query %s' % (p.name, query)) for p in procs: p.join() n_docs = 0 for p in procs: n_docs += res_q.get() self._progress_logger.add(ns, n_docs, done=True) prog_q.put(True) prog_q.close() prog_q.join_thread() proc_logging.join() def _sync_collection_with_query(self, namespace_tuple, query, prog_q, res_q): """ Sync collection with query. """ self._src.reconnect() self._dst.reconnect() src_dbname, src_collname = namespace_tuple dst_dbname, dst_collname = self._conf.db_coll_mapping( src_dbname, src_collname) while True: try: cursor = self._src.client()[src_dbname][src_collname].find( filter=query, cursor_type=pymongo.cursor.CursorType.EXHAUST, no_cursor_timeout=True, # snapshot cause blocking, maybe bug # modifiers={'$snapshot': True} ) total = 0 n = 0 reqs = [] reqs_max = 100 groups = [] groups_max = 10 for doc in cursor: reqs.append( pymongo.ReplaceOne({'_id': doc['_id']}, doc, upsert=True)) if len(reqs) == reqs_max: groups.append(reqs) reqs = [] if len(groups) == groups_max: threads = [ gevent.spawn(self._dst.bulk_write, dst_dbname, dst_collname, groups[i], ordered=False, ignore_duplicate_key_error=True) for i in xrange(groups_max) ] gevent.joinall(threads, raise_error=True) groups = [] n += 1 total += 1 if n % 10000 == 0: prog_q.put(n) n = 0 if len(groups) > 0: threads = [ gevent.spawn(self._dst.bulk_write, dst_dbname, dst_collname, groups[i], ordered=False, ignore_duplicate_key_error=True) for i in xrange(len(groups)) ] gevent.joinall(threads, raise_error=True) if len(reqs) > 0: self._dst.bulk_write(dst_dbname, dst_collname, reqs, ordered=False, ignore_duplicate_key_error=True) if n > 0: prog_q.put(n) res_q.put(total) prog_q.close() prog_q.join_thread() res_q.close() res_q.join_thread() return except pymongo.errors.AutoReconnect: self._src.reconnect() def _replay_oplog(self, start_optime): """ Replay oplog. """ self._last_optime = start_optime n_total = 0 n_skip = 0 while True: try: start_optime_valid = False need_log = False host, port = self._src.client().address log.info('try to sync oplog from %s on %s:%d' % (self._last_optime, host, port)) cursor = self._src.tail_oplog(start_optime) except IndexError as e: log.error(e) log.error('%s not found, terminate' % self._last_optime) return except Exception as e: log.error('get oplog cursor failed: %s' % e) continue # loop: read and apply oplog while True: try: if need_log: self._log_optime(self._last_optime) self._log_progress() need_log = False if not cursor.alive: log.error('cursor is dead') raise pymongo.errors.AutoReconnect oplog = cursor.next() n_total += 1 # check start optime once if not start_optime_valid: if oplog['ts'] == self._last_optime: log.info('oplog is ok: %s' % self._last_optime) start_optime_valid = True else: log.error('oplog %s is stale, terminate' % self._last_optime) return if oplog['op'] == 'n': # no-op self._last_optime = oplog['ts'] need_log = True continue # validate oplog if not self._conf.data_filter.valid_oplog(oplog): n_skip += 1 self._last_optime = oplog['ts'] need_log = True continue dbname, collname = mongo_utils.parse_namespace(oplog['ns']) dst_dbname, dst_collname = self._conf.db_coll_mapping( dbname, collname) if dst_dbname != dbname or dst_collname != collname: oplog['ns'] = '%s.%s' % (dst_dbname, dst_collname) if self._stage == Stage.post_initial_sync: if self._multi_oplog_replayer: if mongo_utils.is_command(oplog): self._multi_oplog_replayer.apply( ignore_duplicate_key_error=True) self._multi_oplog_replayer.clear() self._dst.apply_oplog(oplog) self._last_optime = oplog['ts'] need_log = True else: self._multi_oplog_replayer.push(oplog) if oplog[ 'ts'] == self._initial_sync_end_optime or self._multi_oplog_replayer.count( ) == self._oplog_batchsize: self._multi_oplog_replayer.apply( ignore_duplicate_key_error=True) self._multi_oplog_replayer.clear() self._last_optime = oplog['ts'] need_log = True else: self._dst.apply_oplog( oplog, ignore_duplicate_key_error=True) self._last_optime = oplog['ts'] need_log = True if oplog['ts'] == self._initial_sync_end_optime: log.info('step into stage: oplog_sync') self._stage = Stage.oplog_sync else: if self._multi_oplog_replayer: if mongo_utils.is_command(oplog): self._multi_oplog_replayer.apply() self._multi_oplog_replayer.clear() self._dst.apply_oplog(oplog) self._last_optime = oplog['ts'] need_log = True else: self._multi_oplog_replayer.push(oplog) if self._multi_oplog_replayer.count( ) == self._oplog_batchsize: self._multi_oplog_replayer.apply() self._multi_oplog_replayer.clear() self._last_optime = oplog['ts'] need_log = True else: self._dst.apply_oplog(oplog) self._last_optime = oplog['ts'] need_log = True except StopIteration as e: if self._multi_oplog_replayer and self._multi_oplog_replayer.count( ) > 0: self._multi_oplog_replayer.apply() self._multi_oplog_replayer.clear() self._last_optime = self._multi_oplog_replayer.last_optime( ) need_log = True # no more oplogs, wait a moment time.sleep(0.1) self._log_optime(self._last_optime) self._log_progress('latest') except pymongo.errors.DuplicateKeyError as e: if self._stage == Stage.oplog_sync: log.error(e) log.error('terminate') return else: log.error('ignore duplicate key error: %s' % e) continue except pymongo.errors.AutoReconnect as e: log.error(e) self._src.reconnect() break
class EsSyncer(CommonSyncer): """ Elasticsearch synchronizer. """ def __init__(self, conf): CommonSyncer.__init__(self, conf) if not isinstance(self._conf.src_conf, MongoConfig): raise Exception('invalid src config type') self._src = MongoHandler(self._conf.src_conf) if not self._src.connect(): raise Exception('connect to mongodb(src) failed: %s' % self._conf.src_hostportstr) if not isinstance(self._conf.dst_conf, EsConfig): raise Exception('invalid dst config type') self._dst = EsHandler(self._conf.dst_conf) if not self._dst.connect(): raise Exception('connect to elasticsearch(dst) failed: %s' % self._conf.dst_hostportstr) self._action_buf = [] # used to bulk write oplogs self._last_bulk_optime = None def _action_buf_full(self): return len(self._action_buf) >= 40 def _sync_database(self, dbname): """ Sync a database. """ log.info("sync database '%s'" % dbname) # create index idxname = self._conf.db_mapping(dbname) if self._dst.client().indices.exists(index=idxname): log.info('index already existed: %s' % idxname) else: log.info('create index: %s' % idxname) self._dst.client().indices.create(index=idxname) self._sync_collections(dbname) def _sync_collection(self, namespace_tuple): """ Sync a collection until success. """ src_dbname, src_collname = namespace_tuple[0], namespace_tuple[1] idxname, typename = self._conf.db_coll_mapping(src_dbname, src_collname) fields = self._conf.fieldmap.get( gen_namespace(src_dbname, src_collname)) while True: try: log.info("sync collection '%s.%s' => '%s.%s'" % (src_dbname, src_collname, idxname, typename)) cursor = self._src.client()[src_dbname][src_collname].find( filter=None, cursor_type=pymongo.cursor.CursorType.EXHAUST, no_cursor_timeout=True, modifiers={'$snapshot': True}) count = cursor.count() if count == 0: log.info(' skip empty collection') return n = 0 actions = [] actions_max = 20 groups = [] groups_max = 10 for doc in cursor: id = str(doc['_id']) del doc['_id'] source = gen_doc_with_fields(doc, fields) if fields else doc if source: actions.append({ '_op_type': 'index', '_index': idxname, '_type': typename, '_id': id, '_source': source }) if len(actions) == actions_max: groups.append(actions) actions = [] if len(groups) == groups_max: threads = [ gevent.spawn(self._dst.bulk_write, groups[i]) for i in xrange(groups_max) ] gevent.joinall(threads) groups = [] n += 1 if n % 1000 == 0: log.info(' %s.%s %d/%d (%.2f%%)' % (src_dbname, src_collname, n, count, float(n) / count * 100)) if len(groups) > 0: threads = [ gevent.spawn(self._dst.bulk_write, groups[i]) for i in xrange(len(groups)) ] gevent.joinall(threads) if len(actions) > 0: elasticsearch.helpers.bulk(client=self._dst.client(), actions=actions) log.info(' %s.%s %d/%d (%.2f%%)' % (src_dbname, src_collname, n, count, float(n) / count * 100)) return except pymongo.errors.AutoReconnect: self._src.reconnect() def _replay_oplog(self, oplog_start): """ Replay oplog. """ self._last_bulk_optime = oplog_start n_total = 0 n_skip = 0 while True: # try to get cursor until success try: host, port = self._src.client().address log.info('try to sync oplog from %s on %s:%d' % (self._last_bulk_optime, host, port)) # set codec options to guarantee the order of keys in command coll = self._src.client()['local'].get_collection( 'oplog.rs', codec_options=bson.codec_options.CodecOptions( document_class=bson.son.SON)) cursor = coll.find( {'ts': { '$gte': oplog_start }}, cursor_type=pymongo.cursor.CursorType.TAILABLE_AWAIT, no_cursor_timeout=True) # New in version 3.2 # src_version = mongo_utils.get_version(self._src.client()) # if mongo_utils.version_higher_or_equal(src_version, '3.2.0'): # cursor.max_await_time_ms(1000) valid_start_optime = False # need to validate while True: try: if not cursor.alive: log.error('cursor is dead') raise pymongo.errors.AutoReconnect oplog = cursor.next() n_total += 1 if not valid_start_optime: if oplog['ts'] == oplog_start: log.info('oplog is ok: %s' % oplog_start) valid_start_optime = True else: log.error('oplog %s is stale, terminate' % oplog_start) return # validate oplog if not self._conf.data_filter.valid_oplog(oplog): n_skip += 1 self._last_optime = oplog['ts'] continue op = oplog['op'] ns = oplog['ns'] if op == 'i': # insert dbname, collname = parse_namespace(ns) idxname, typename = self._conf.db_coll_mapping( dbname, collname) fields = self._conf.fieldmap.get( gen_namespace(dbname, collname)) doc = oplog['o'] id = str(doc['_id']) del doc['_id'] if fields: doc = gen_doc_with_fields(doc, fields) if doc: self._action_buf.append({ '_op_type': 'index', '_index': idxname, '_type': typename, '_id': id, '_source': doc }) elif op == 'u': # update dbname, collname = parse_namespace(ns) idxname, typename = self._conf.db_coll_mapping( dbname, collname) fields = self._conf.fieldmap.get( gen_namespace(dbname, collname)) id = str(oplog['o2']['_id']) if '$set' in oplog['o']: doc = {} for k, v in oplog['o']['$set'].iteritems(): if not fields or k in fields: sub_doc = doc_flat_to_nested( k.split('.'), v) merge_doc(doc, sub_doc) if doc: self._action_buf.append({ '_op_type': 'update', '_index': idxname, '_type': typename, '_id': id, '_retry_on_conflict': 3, 'doc': doc, 'doc_as_upsert': True }) if '$unset' in oplog['o']: script_statements = [] for keypath in oplog['o']['$unset'].iterkeys(): if not fields or keypath in fields: pos = keypath.rfind('.') if pos >= 0: script_statements.append( 'ctx._source.%s.remove("%s")' % (keypath[:pos], keypath[pos + 1:])) else: script_statements.append( 'ctx._source.remove("%s")' % keypath) if script_statements: doc = { 'script': '; '.join(script_statements) } self._action_buf.append({ '_op_type': 'update', '_index': idxname, '_type': typename, '_id': id, '_retry_on_conflict': 3, 'script': doc['script'] }) if '$set' not in oplog[ 'o'] and '$unset' not in oplog['o']: log.warn('unexpect oplog: %s', oplog['o']) elif op == 'd': # delete dbname, collname = parse_namespace(ns) idxname, typename = self._conf.db_coll_mapping( dbname, collname) id = str(oplog['o']['_id']) self._action_buf.append({ '_op_type': 'delete', '_index': idxname, '_type': typename, '_id': id }) elif op == 'c': # command dbname, _ = parse_namespace(ns) idxname = self._conf.db_mapping(dbname) if 'drop' in oplog['o']: # TODO # how to delete type? pass log.warn( 'you should implement document type deletion.' ) if 'dropDatabase' in oplog['o']: # delete index self._dst.client().indices.delete( index=idxname) elif op == 'n': # no-op pass else: log.error('invalid optype: %s' % oplog) # flush if self._action_buf_full(): self._dst.bulk_write(self._action_buf) self._action_buf = [] self._last_bulk_optime = oplog['ts'] self._last_optime = oplog['ts'] self._log_optime(oplog['ts']) self._log_progress() except StopIteration as e: # flush if len(self._action_buf) > 0: self._dst.bulk_write(self._action_buf) self._action_buf = [] self._last_bulk_optime = self._last_optime self._log_optime(self._last_optime) self._log_progress('latest') time.sleep(0.1) except pymongo.errors.AutoReconnect as e: log.error(e) self._src.reconnect() break except elasticsearch.helpers.BulkIndexError as e: log.error(e) self._action_buf = [] except IndexError as e: log.error(e) log.error('%s not found, terminate' % oplog_start) return
class EsSyncer(CommonSyncer): """ Elasticsearch synchronizer. """ def __init__(self, conf): CommonSyncer.__init__(self, conf) if not isinstance(self._conf.src_conf, MongoConfig): raise Exception('invalid src config type') self._src = MongoHandler(self._conf.src_conf) if not self._src.connect(): raise Exception('connect to mongodb(src) failed: %s' % self._conf.src_hostportstr) if not isinstance(self._conf.dst_conf, EsConfig): raise Exception('invalid dst config type') self._dst = EsHandler(self._conf.dst_conf) if not self._dst.connect(): raise Exception('connect to elasticsearch(dst) failed: %s' % self._conf.dst_hostportstr) self._action_buf = [] # used to bulk write oplogs self._last_bulk_optime = None def _action_buf_full(self): return len(self._action_buf) >= 40 def _sync_database(self, dbname): """ Sync a database. """ log.info("sync database '%s'" % dbname) # create index idxname = self._conf.db_mapping(dbname) if self._dst.client().indices.exists(index=idxname): log.info('index already existed: %s' % idxname) else: log.info('create index: %s' % idxname) self._dst.client().indices.create(index=idxname) self._sync_collections(dbname) def _sync_collection(self, namespace_tuple): """ Sync a collection until success. """ src_dbname, src_collname = namespace_tuple[0], namespace_tuple[1] idxname, typename = self._conf.db_coll_mapping(src_dbname, src_collname) fields = self._conf.fieldmap.get(gen_namespace(src_dbname, src_collname)) while True: try: log.info("sync collection '%s.%s' => '%s.%s'" % (src_dbname, src_collname, idxname, typename)) cursor = self._src.client()[src_dbname][src_collname].find(filter=None, cursor_type=pymongo.cursor.CursorType.EXHAUST, no_cursor_timeout=True, modifiers={'$snapshot': True}) count = cursor.count() if count == 0: log.info(' skip empty collection') return n = 0 actions = [] actions_max = 20 groups = [] groups_max = 10 for doc in cursor: id = str(doc['_id']) del doc['_id'] source = gen_doc_with_fields(doc, fields) if fields else doc if source: actions.append({'_op_type': 'index', '_index': idxname, '_type': typename, '_id': id, '_source': source}) if len(actions) == actions_max: groups.append(actions) actions = [] if len(groups) == groups_max: threads = [gevent.spawn(self._dst.bulk_write, groups[i]) for i in xrange(groups_max)] gevent.joinall(threads) groups = [] n += 1 if n % 1000 == 0: log.info(' %s.%s %d/%d (%.2f%%)' % (src_dbname, src_collname, n, count, float(n)/count*100)) if len(groups) > 0: threads = [gevent.spawn(self._dst.bulk_write, groups[i]) for i in xrange(len(groups))] gevent.joinall(threads) if len(actions) > 0: elasticsearch.helpers.bulk(client=self._dst.client(), actions=actions) log.info(' %s.%s %d/%d (%.2f%%)' % (src_dbname, src_collname, n, count, float(n)/count*100)) return except pymongo.errors.AutoReconnect: self._src.reconnect() def _replay_oplog(self, oplog_start): """ Replay oplog. """ self._last_bulk_optime = oplog_start n_total = 0 n_skip = 0 while True: # try to get cursor until success try: host, port = self._src.client().address log.info('try to sync oplog from %s on %s:%d' % (self._last_bulk_optime, host, port)) # set codec options to guarantee the order of keys in command coll = self._src.client()['local'].get_collection('oplog.rs', codec_options=bson.codec_options.CodecOptions(document_class=bson.son.SON)) cursor = coll.find({'ts': {'$gte': oplog_start}}, cursor_type=pymongo.cursor.CursorType.TAILABLE_AWAIT, no_cursor_timeout=True) # New in version 3.2 # src_version = mongo_utils.get_version(self._src.client()) # if mongo_utils.version_higher_or_equal(src_version, '3.2.0'): # cursor.max_await_time_ms(1000) valid_start_optime = False # need to validate while True: try: if not cursor.alive: log.error('cursor is dead') raise pymongo.errors.AutoReconnect oplog = cursor.next() n_total += 1 if not valid_start_optime: if oplog['ts'] == oplog_start: log.info('oplog is ok: %s' % oplog_start) valid_start_optime = True else: log.error('oplog %s is stale, terminate' % oplog_start) return # validate oplog if not self._conf.data_filter.valid_oplog(oplog): n_skip += 1 self._last_optime = oplog['ts'] continue op = oplog['op'] ns = oplog['ns'] if op == 'i': # insert dbname, collname = parse_namespace(ns) idxname, typename = self._conf.db_coll_mapping(dbname, collname) fields = self._conf.fieldmap.get(gen_namespace(dbname, collname)) doc = oplog['o'] id = str(doc['_id']) del doc['_id'] if fields: doc = gen_doc_with_fields(doc, fields) if doc: self._action_buf.append({'_op_type': 'index', '_index': idxname, '_type': typename, '_id': id, '_source': doc}) elif op == 'u': # update dbname, collname = parse_namespace(ns) idxname, typename = self._conf.db_coll_mapping(dbname, collname) fields = self._conf.fieldmap.get(gen_namespace(dbname, collname)) id = str(oplog['o2']['_id']) if '$set' in oplog['o']: doc = {} for k, v in oplog['o']['$set'].iteritems(): if not fields or k in fields: sub_doc = doc_flat_to_nested(k.split('.'), v) merge_doc(doc, sub_doc) if doc: self._action_buf.append({'_op_type': 'update', '_index': idxname, '_type': typename, '_id': id, '_retry_on_conflict': 3, 'doc': doc, 'doc_as_upsert': True}) if '$unset' in oplog['o']: script_statements = [] for keypath in oplog['o']['$unset'].iterkeys(): if not fields or keypath in fields: pos = keypath.rfind('.') if pos >= 0: script_statements.append('ctx._source.%s.remove("%s")' % (keypath[:pos], keypath[pos+1:])) else: script_statements.append('ctx._source.remove("%s")' % keypath) if script_statements: doc = {'script': '; '.join(script_statements)} self._action_buf.append({'_op_type': 'update', '_index': idxname, '_type': typename, '_id': id, '_retry_on_conflict': 3, 'script': doc['script']}) if '$set' not in oplog['o'] and '$unset' not in oplog['o']: log.warn('unexpect oplog: %s', oplog['o']) elif op == 'd': # delete dbname, collname = parse_namespace(ns) idxname, typename = self._conf.db_coll_mapping(dbname, collname) id = str(oplog['o']['_id']) self._action_buf.append({'_op_type': 'delete', '_index': idxname, '_type': typename, '_id': id}) elif op == 'c': # command dbname, _ = parse_namespace(ns) idxname = self._conf.db_mapping(dbname) if 'drop' in oplog['o']: # TODO # how to delete type? pass log.warn('you should implement document type deletion.') if 'dropDatabase' in oplog['o']: # delete index self._dst.client().indices.delete(index=idxname) elif op == 'n': # no-op pass else: log.error('invalid optype: %s' % oplog) # flush if self._action_buf_full(): self._dst.bulk_write(self._action_buf) self._action_buf = [] self._last_bulk_optime = oplog['ts'] self._last_optime = oplog['ts'] self._log_optime(oplog['ts']) self._log_progress() except StopIteration as e: # flush if len(self._action_buf) > 0: self._dst.bulk_write(self._action_buf) self._action_buf = [] self._last_bulk_optime = self._last_optime self._log_optime(self._last_optime) self._log_progress('latest') time.sleep(0.1) except pymongo.errors.AutoReconnect as e: log.error(e) self._src.reconnect() break except elasticsearch.helpers.BulkIndexError as e: log.error(e) self._action_buf = [] except IndexError as e: log.error(e) log.error('%s not found, terminate' % oplog_start) return