def test_configured(self): configure_controller( test_settings.CONTROLLER['uri'], test_settings.CONTROLLER['db_name'], ) self.assertIsNone(shardmonster.connection._controlling_db) get_controlling_db() self.assertIsNotNone(shardmonster.connection._controlling_db)
def _sync_from_oplog(collection_name, shard_key, oplog_pos): """Syncs the oplog to within a reasonable timeframe of "now". """ conn = get_controlling_db().connection repl_coll = conn['local']['oplog.rs'] cursor = repl_coll.find({'ts': {'$gt': oplog_pos}}, tailable=True) cursor = cursor.add_option(_QUERY_OPTIONS['oplog_replay']) cursor = cursor.hint([('$natural', 1)]) realm = metadata._get_realm_for_collection(collection_name) shard_field = realm['shard_field'] shards_coll = api._get_shards_coll() shard_metadata, = shards_coll.find( {'realm': realm['name'], 'shard_key': shard_key}) current_location = shard_metadata['location'] new_location = shard_metadata['new_location'] current_collection = _get_collection_from_location_string( current_location, collection_name) new_collection = _get_collection_from_location_string( new_location, collection_name) shard_query = {shard_field: shard_key} current_namespace = "%s.%s" % ( current_collection.database.name, current_collection.name) for r in cursor: if r['ns'] != current_namespace: continue if r['op'] in ['u', 'i']: # Check that this doc is part of our query set oid = r.get('o2', r['o'])['_id'] object_query = {'_id': oid} object_query.update(shard_query) match = bool( current_collection.find(object_query).count()) elif r['op'] == 'd': oid = r.get('o2', r['o'])['_id'] object_query = {'_id': oid} object_query.update(shard_query) match = bool( new_collection.find(object_query).count()) else: print 'Ignoring op', r['op'], r continue if not match: continue if r['op'] == 'u': blue(' - Updating %s with %s' % (oid, r['o'])) new_collection.update( {'_id': oid}, r['o'], safe=True) elif r['op'] == 'i': try: new_collection.insert(r['o'], safe=True) except pymongo.errors.DuplicateKeyError: pass elif r['op'] == 'd': blue(' - Removing %s' % oid) new_collection.remove({'_id': oid}, safe=True) oplog_pos = r['ts'] return oplog_pos
def _get_oplog_pos(): conn = get_controlling_db().connection repl_coll = conn['local']['oplog.rs'] most_recent_op = repl_coll.find({}, sort=[('$natural', -1)])[0] ts_from = most_recent_op['ts'] return ts_from
def _get_shards_coll(): return get_controlling_db().shards
def _get_realm_coll(): return get_controlling_db().realms
def _sync_from_oplog(collection_name, shard_key, oplog_pos): """Syncs the oplog to within a reasonable timeframe of "now". """ conn = get_controlling_db().connection repl_coll = conn['local']['oplog.rs'] cursor = repl_coll.find({'ts': {'$gt': oplog_pos}}, tailable=True) cursor = cursor.add_option(_QUERY_OPTIONS['oplog_replay']) cursor = cursor.hint([('$natural', 1)]) realm = metadata._get_realm_for_collection(collection_name) shard_field = realm['shard_field'] shards_coll = api._get_shards_coll() shard_metadata, = shards_coll.find({ 'realm': realm['name'], 'shard_key': shard_key }) current_location = shard_metadata['location'] new_location = shard_metadata['new_location'] current_collection = _get_collection_from_location_string( current_location, collection_name) new_collection = _get_collection_from_location_string( new_location, collection_name) shard_query = {shard_field: shard_key} current_namespace = "%s.%s" % (current_collection.database.name, current_collection.name) for r in cursor: if r['ns'] != current_namespace: continue if r['op'] in ['u', 'i']: # Check that this doc is part of our query set oid = r.get('o2', r['o'])['_id'] object_query = {'_id': oid} object_query.update(shard_query) match = bool(current_collection.find(object_query).count()) elif r['op'] == 'd': oid = r.get('o2', r['o'])['_id'] object_query = {'_id': oid} object_query.update(shard_query) match = bool(new_collection.find(object_query).count()) else: print 'Ignoring op', r['op'], r continue if not match: continue if r['op'] == 'u': blue(' - Updating %s with %s' % (oid, r['o'])) new_collection.update({'_id': oid}, r['o'], safe=True) elif r['op'] == 'i': try: new_collection.insert(r['o'], safe=True) except pymongo.errors.DuplicateKeyError: pass elif r['op'] == 'd': blue(' - Removing %s' % oid) new_collection.remove({'_id': oid}, safe=True) oplog_pos = r['ts'] return oplog_pos
def test_not_configured(self): with self.assertRaises(Exception) as cm: get_controlling_db() self.assertEqual( 'Call connect_to_controller or configure_controller ' 'before attempting to get a connection', str(cm.exception))