def test_configured(self):
configure_controller(
test_settings.CONTROLLER['uri'],
test_settings.CONTROLLER['db_name'],
)
self.assertIsNone(shardmonster.connection._controlling_db)
get_controlling_db()
self.assertIsNotNone(shardmonster.connection._controlling_db)
def _sync_from_oplog(collection_name, shard_key, oplog_pos):
"""Syncs the oplog to within a reasonable timeframe of "now".
"""
conn = get_controlling_db().connection
repl_coll = conn['local']['oplog.rs']
cursor = repl_coll.find({'ts': {'$gt': oplog_pos}}, tailable=True)
cursor = cursor.add_option(_QUERY_OPTIONS['oplog_replay'])
cursor = cursor.hint([('$natural', 1)])
realm = metadata._get_realm_for_collection(collection_name)
shard_field = realm['shard_field']
shards_coll = api._get_shards_coll()
shard_metadata, = shards_coll.find(
{'realm': realm['name'], 'shard_key': shard_key})
current_location = shard_metadata['location']
new_location = shard_metadata['new_location']
current_collection = _get_collection_from_location_string(
current_location, collection_name)
new_collection = _get_collection_from_location_string(
new_location, collection_name)
shard_query = {shard_field: shard_key}
current_namespace = "%s.%s" % (
current_collection.database.name, current_collection.name)
for r in cursor:
if r['ns'] != current_namespace:
continue
if r['op'] in ['u', 'i']:
# Check that this doc is part of our query set
oid = r.get('o2', r['o'])['_id']
object_query = {'_id': oid}
object_query.update(shard_query)
match = bool(
current_collection.find(object_query).count())
elif r['op'] == 'd':
oid = r.get('o2', r['o'])['_id']
object_query = {'_id': oid}
object_query.update(shard_query)
match = bool(
new_collection.find(object_query).count())
else:
print 'Ignoring op', r['op'], r
continue
if not match:
continue
if r['op'] == 'u':
blue(' - Updating %s with %s' % (oid, r['o']))
new_collection.update(
{'_id': oid}, r['o'], safe=True)
elif r['op'] == 'i':
try:
new_collection.insert(r['o'], safe=True)
except pymongo.errors.DuplicateKeyError:
pass
elif r['op'] == 'd':
blue(' - Removing %s' % oid)
new_collection.remove({'_id': oid}, safe=True)
oplog_pos = r['ts']
return oplog_pos
def _get_oplog_pos():
conn = get_controlling_db().connection
repl_coll = conn['local']['oplog.rs']
most_recent_op = repl_coll.find({}, sort=[('$natural', -1)])[0]
ts_from = most_recent_op['ts']
return ts_from
def _get_shards_coll():
return get_controlling_db().shards
def _get_realm_coll():
return get_controlling_db().realms
def _sync_from_oplog(collection_name, shard_key, oplog_pos):
"""Syncs the oplog to within a reasonable timeframe of "now".
"""
conn = get_controlling_db().connection
repl_coll = conn['local']['oplog.rs']
cursor = repl_coll.find({'ts': {'$gt': oplog_pos}}, tailable=True)
cursor = cursor.add_option(_QUERY_OPTIONS['oplog_replay'])
cursor = cursor.hint([('$natural', 1)])
realm = metadata._get_realm_for_collection(collection_name)
shard_field = realm['shard_field']
shards_coll = api._get_shards_coll()
shard_metadata, = shards_coll.find({
'realm': realm['name'],
'shard_key': shard_key
})
current_location = shard_metadata['location']
new_location = shard_metadata['new_location']
current_collection = _get_collection_from_location_string(
current_location, collection_name)
new_collection = _get_collection_from_location_string(
new_location, collection_name)
shard_query = {shard_field: shard_key}
current_namespace = "%s.%s" % (current_collection.database.name,
current_collection.name)
for r in cursor:
if r['ns'] != current_namespace:
continue
if r['op'] in ['u', 'i']:
# Check that this doc is part of our query set
oid = r.get('o2', r['o'])['_id']
object_query = {'_id': oid}
object_query.update(shard_query)
match = bool(current_collection.find(object_query).count())
elif r['op'] == 'd':
oid = r.get('o2', r['o'])['_id']
object_query = {'_id': oid}
object_query.update(shard_query)
match = bool(new_collection.find(object_query).count())
else:
print 'Ignoring op', r['op'], r
continue
if not match:
continue
if r['op'] == 'u':
blue(' - Updating %s with %s' % (oid, r['o']))
new_collection.update({'_id': oid}, r['o'], safe=True)
elif r['op'] == 'i':
try:
new_collection.insert(r['o'], safe=True)
except pymongo.errors.DuplicateKeyError:
pass
elif r['op'] == 'd':
blue(' - Removing %s' % oid)
new_collection.remove({'_id': oid}, safe=True)
oplog_pos = r['ts']
return oplog_pos
def _get_oplog_pos():
conn = get_controlling_db().connection
repl_coll = conn['local']['oplog.rs']
most_recent_op = repl_coll.find({}, sort=[('$natural', -1)])[0]
ts_from = most_recent_op['ts']
return ts_from
def test_not_configured(self):
with self.assertRaises(Exception) as cm:
get_controlling_db()
self.assertEqual(
'Call connect_to_controller or configure_controller '
'before attempting to get a connection', str(cm.exception))
def _get_shards_coll():
return get_controlling_db().shards
def _get_realm_coll():
return get_controlling_db().realms
