def _create_collection_iterator(collection_name, query):
"""Creates an iterator that returns collections and queries that can then
be used to perform multishard operations:
for collection, query in _create_collection_iterator(...):
for doc in collection.find(query):
yield doc
This does all the hardwork of figuring out what collections to query and how
to adjust the query to account for any shards that are currently moving.
"""
realm = _get_realm_for_collection(collection_name)
shard_field = realm['shard_field']
shard_key = _get_query_target(collection_name, query)
if shard_key:
location = _get_location_for_shard(realm, shard_key)
locations = {location.location: location}
else:
locations = _get_all_locations_for_realm(realm)
for location, location_meta in locations.iteritems():
cluster_name, database_name = location.split('/')
connection = get_connection(cluster_name)
collection = connection[database_name][collection_name]
if location_meta.excludes:
if len(location_meta.excludes) == 1:
query = {'$and': [
query, {shard_field: {'$ne': location_meta.excludes[0]}}]}
else:
raise Exception('Multiple shards in transit. Aborting')
yield collection, query
if location_meta.excludes:
query = query['$and'][0]
def where_is(collection_name, shard_key):
"""Returns a string of the form cluster/database that says where a
particular shard of data resides.
:param collection_name: The collection name for the shard
:param shard_key: The shard key to look for
"""
realm = _get_realm_for_collection(collection_name)
location = _get_location_for_shard(realm, shard_key)
return location.location
def where_is(collection_name, shard_key):
"""Returns a string of the form cluster/database that says where a
particular shard of data resides.
:param collection_name: The collection name for the shard
:param shard_key: The shard key to look for
"""
realm = _get_realm_for_collection(collection_name)
location = _get_location_for_shard(realm, shard_key)
return location.location
def _get_collection_for_targetted_upsert(
collection_name, query, update, with_options={}):
shard_key = _get_query_target(collection_name, update)
if not shard_key:
shard_key = _get_query_target(collection_name, update['$set'])
realm = _get_realm_for_collection(collection_name)
location = _get_location_for_shard(realm, shard_key)
cluster_name, database_name = parse_location(location.location)
connection = get_connection(cluster_name)
collection = connection[database_name][collection_name]
if with_options:
collection = collection.with_options(with_options)
return collection
def _get_collection_for_targetted_upsert(collection_name,
query,
update,
with_options={}):
shard_key = _get_query_target(collection_name, update)
if not shard_key:
shard_key = _get_query_target(collection_name, update['$set'])
realm = _get_realm_for_collection(collection_name)
location = _get_location_for_shard(realm, shard_key)
cluster_name, database_name = parse_location(location.location)
connection = get_connection(cluster_name)
collection = connection[database_name][collection_name]
if with_options:
collection = collection.with_options(with_options)
return collection
def start_migration(realm_name, shard_key, new_location):
"""Marks a shard as being in the process of being migrated.
"""
shards_coll = _get_shards_coll()
realm = _get_realm_by_name(realm_name)
existing_location = _get_location_for_shard(realm, shard_key)
if existing_location.location == new_location:
raise Exception('Shard is already at %s' % new_location)
shards_coll.update(
{'realm': realm_name, 'shard_key': shard_key},
{'$set': {
'status': ShardStatus.MIGRATING_COPY,
'new_location': new_location,
}},
)
def start_migration(realm_name, shard_key, new_location):
"""Marks a shard as being in the process of being migrated.
"""
shards_coll = _get_shards_coll()
realm = _get_realm_by_name(realm_name)
existing_location = _get_location_for_shard(realm, shard_key)
if existing_location.location == new_location:
raise Exception('Shard is already at %s' % new_location)
shards_coll.update(
{
'realm': realm_name,
'shard_key': shard_key
},
{
'$set': {
'status': ShardStatus.MIGRATING_COPY,
'new_location': new_location,
}
},
)
