def get_cluster_state(mongos): """ returns a dictionary that contains the subset of cluster state we care about """ global shard_clients # this won't work well with a large (thousands?) number of shards shards_collection = client['config']['shards'] shards = [shard for shard in shards_collection.find()] state = {} state['shard_names'] = [shard['_id'] for shard in shards] state['shard_names'].sort() members = {} oplog_positions = {} for shard in shards: # get statuses for all replica set members try: repl_set, host = shard['host'].split('/') except ValueError: print >> sys.stderr, "ERROR: can't get replica set status for %s" % shard[ '_id'] sys.exit(1) # get cached connection, if one exists if repl_set in shard_clients: shard_client = shard_clients[repl_set] else: shard_client = pymongo.MongoClient( host, replicaSet=repl_set, read_preference=ReadPreference.PRIMARY, socketTimeoutMS=120000) shard_clients[repl_set] = shard_client rs_status = shard_client.admin.command('replSetGetStatus') for member in rs_status['members']: members[member['name']] = member['stateStr'] # get last oplog positions last_oplog_entry = utils.get_last_oplog_entry(shard_client) oplog_positions[repl_set] = last_oplog_entry['ts'] state['members'] = members state['oplog_positions'] = oplog_positions return state
def get_cluster_state(mongos): """ returns a dictionary that contains the subset of cluster state we care about """ global shard_clients # this won't work well with a large (thousands?) number of shards shards_collection = client['config']['shards'] shards = [shard for shard in shards_collection.find()] state = {} state['shard_names'] = [shard['_id'] for shard in shards] state['shard_names'].sort() members = {} oplog_positions = {} for shard in shards: # get statuses for all replica set members try: repl_set, host = shard['host'].split('/') except ValueError: print >>sys.stderr, "ERROR: can't get replica set status for %s" % shard['_id'] sys.exit(1) # get cached connection, if one exists if repl_set in shard_clients: shard_client = shard_clients[repl_set] else: shard_client = pymongo.MongoClient(host, replicaSet=repl_set, read_preference=ReadPreference.PRIMARY, socketTimeoutMS=120000) shard_clients[repl_set] = shard_client rs_status = shard_client.admin.command('replSetGetStatus') for member in rs_status['members']: members[member['name']] = member['stateStr'] # get last oplog positions last_oplog_entry = utils.get_last_oplog_entry(shard_client) oplog_positions[repl_set] = last_oplog_entry['ts'] state['members'] = members state['oplog_positions'] = oplog_positions return state
def copy_collection(source, dest, state_path, percent): """ Copies all documents from source to destination collection. Inserts documents in batches using insert workers, which are each run in their own greenlet. Ensures that the destination is empty before starting the copy. Does no safety checks -- this is up to the caller. @param source dict of (host, port, db, collection) for the source @param dest dict of (host, port, db, collection) for the destination @param state_path path of state database @param percent percentage of documents to copy """ gevent.monkey.patch_socket() # open state database state_db = CopyStateDB(state_path) # connect to mongo source_client = utils.mongo_connect( source['host'], source['port'], ensure_direct=True, max_pool_size=30, read_preference=ReadPreference.SECONDARY, document_class=FasterOrderedDict) source_collection = source_client[source['db']][source['collection']] if source_client.is_mongos: raise Exception( "for performance reasons, sources must be mongod instances; %s:%d is not", source['host'], source['port']) dest_client = utils.mongo_connect(dest['host'], dest['port'], max_pool_size=30, document_class=FasterOrderedDict) dest_collection = dest_client[dest['db']][dest['collection']] # record timestamp of last oplog entry, so that we know where to start applying ops # later oplog_ts = utils.get_last_oplog_entry(source_client)['ts'] state_db.update_oplog_ts(source, dest, oplog_ts) # for testing copying of indices quickly if percent == 0: log.info("skipping copy because of --percent 0 parameters") state_db.update_state(source, dest, CopyStateDB.STATE_WAITING_FOR_INDICES) return stats = Stats() stats.total_docs = int(source_collection.count()) if percent: # hack-ish but good enough for a testing-only feature stats.total_docs = int(stats.total_docs * (float(percent) / 100.0)) # get all _ids, which works around a mongo bug/feature that causes massive slowdowns # of long-running, large reads over time ids = [] cursor = source_collection.find(fields=["_id"], snapshot=True, timeout=False) cursor.batch_size(5000) insert_pool = Pool(INSERT_POOL_SIZE) stats_greenlet = gevent.spawn(_copy_stats_worker, stats) for doc in cursor: _id = doc['_id'] if percent is not None and not utils.id_in_subset(_id, percent): continue # when we've gathered enough _ids, spawn a worker greenlet to batch copy the # documents corresponding to them ids.append(_id) if len(ids) % INSERT_SIZE == 0: outgoing_ids = ids ids = [] insert_pool.spawn(_find_and_insert_batch_worker, source_collection=source_collection, dest_collection=dest_collection, ids=outgoing_ids, stats=stats) gevent.sleep() # insert last batch of documents if len(ids) > 0: _find_and_insert_batch_worker(source_collection=source_collection, dest_collection=dest_collection, ids=ids, stats=stats) stats.log() # wait until all other outstanding inserts have finished insert_pool.join() stats_greenlet.kill() log.info("done with initial copy") state_db.update_state(source, dest, CopyStateDB.STATE_WAITING_FOR_INDICES)
def copy_collection(source, dest, state_path, percent): """ Copies all documents from source to destination collection. Inserts documents in batches using insert workers, which are each run in their own greenlet. Ensures that the destination is empty before starting the copy. Does no safety checks -- this is up to the caller. @param source dict of (host, port, db, collection) for the source @param dest dict of (host, port, db, collection) for the destination @param state_path path of state database @param percent percentage of documents to copy """ gevent.monkey.patch_socket() # open state database state_db = CopyStateDB(state_path) # connect to mongo source_client = utils.mongo_connect(source, ensure_direct=True, max_pool_size=30, read_preference=ReadPreference.SECONDARY, document_class=FasterOrderedDict) source_collection = source_client[source['db']][source['collection']] if source_client.is_mongos: raise Exception("for performance reasons, sources must be mongod instances; %s:%d is not", source['host'], source['port']) dest_client = utils.mongo_connect(dest, max_pool_size=30, document_class=FasterOrderedDict) dest_collection = dest_client[dest['db']][dest['collection']] # record timestamp of last oplog entry, so that we know where to start applying ops # later oplog_ts = utils.get_last_oplog_entry(source_client)['ts'] state_db.update_oplog_ts(source, dest, oplog_ts) # for testing copying of indices quickly if percent == 0: log.info("skipping copy because of --percent 0 parameters") state_db.update_state(source, dest, CopyStateDB.STATE_WAITING_FOR_INDICES) return stats = Stats() stats.total_docs = int(source_collection.count()) if percent: # hack-ish but good enough for a testing-only feature stats.total_docs = int(stats.total_docs * (float(percent)/100.0)) # get all _ids, which works around a mongo bug/feature that causes massive slowdowns # of long-running, large reads over time ids = [] cursor = source_collection.find( projection={'_id':True}, modifiers={'$snapshot':True} ) cursor.batch_size(5000) insert_pool = Pool(INSERT_POOL_SIZE) stats_greenlet = gevent.spawn(_copy_stats_worker, stats) for doc in cursor: _id = doc['_id'] if percent is not None and not utils.id_in_subset(_id, percent): continue # when we've gathered enough _ids, spawn a worker greenlet to batch copy the # documents corresponding to them ids.append(_id) if len(ids) % INSERT_SIZE == 0: outgoing_ids = ids ids = [] insert_pool.spawn(_find_and_insert_batch_worker, source_collection=source_collection, dest_collection=dest_collection, ids=outgoing_ids, stats=stats) gevent.sleep() # insert last batch of documents if len(ids) > 0: _find_and_insert_batch_worker(source_collection=source_collection, dest_collection=dest_collection, ids=ids, stats=stats) stats.log() # wait until all other outstanding inserts have finished insert_pool.join() stats_greenlet.kill() log.info("done with initial copy") state_db.update_state(source, dest, CopyStateDB.STATE_WAITING_FOR_INDICES)