def _create_collection_iterator(collection_name, query): """Creates an iterator that returns collections and queries that can then be used to perform multishard operations: for collection, query in _create_collection_iterator(...): for doc in collection.find(query): yield doc This does all the hardwork of figuring out what collections to query and how to adjust the query to account for any shards that are currently moving. """ realm = _get_realm_for_collection(collection_name) shard_field = realm['shard_field'] shard_key = _get_query_target(collection_name, query) if shard_key: location = _get_location_for_shard(realm, shard_key) locations = {location.location: location} else: locations = _get_all_locations_for_realm(realm) for location, location_meta in locations.iteritems(): cluster_name, database_name = location.split('/') connection = get_connection(cluster_name) collection = connection[database_name][collection_name] if location_meta.excludes: if len(location_meta.excludes) == 1: query = {'$and': [ query, {shard_field: {'$ne': location_meta.excludes[0]}}]} else: raise Exception('Multiple shards in transit. Aborting') yield collection, query if location_meta.excludes: query = query['$and'][0]
def test_get_location_ordering(self): # Exposes a bug that was found in caching and default locations api.create_realm('dummy-realm', 'some_field', 'dummy_collection', 'cluster-1/some_db') api.set_shard_at_rest('dummy-realm', 1, 'dest2/some_db') realm = metadata._get_realm_for_collection('dummy_collection') meta = metadata._get_metadata_for_shard(realm, 2) expected_meta = { 'status': metadata.ShardStatus.AT_REST, 'realm': 'dummy-realm', 'location': 'cluster-1/some_db' } self.assertEquals(meta, expected_meta) all_locations = metadata._get_all_locations_for_realm(realm) self.assertEquals([], all_locations['cluster-1/some_db'].contains) self.assertEquals([], all_locations['cluster-1/some_db'].excludes) self.assertEquals([1], all_locations['dest2/some_db'].contains) self.assertEquals([], all_locations['dest2/some_db'].excludes)
def test_get_location_ordering(self): # Exposes a bug that was found in caching and default locations api.create_realm( 'dummy-realm', 'some_field', 'dummy_collection', 'cluster-1/some_db') api.set_shard_at_rest('dummy-realm', 1, 'dest2/some_db') realm = metadata._get_realm_for_collection('dummy_collection') meta = metadata._get_metadata_for_shard(realm, 2) expected_meta = { 'status': metadata.ShardStatus.AT_REST, 'realm': 'dummy-realm', 'location': 'cluster-1/some_db' } self.assertEquals(meta, expected_meta) all_locations = metadata._get_all_locations_for_realm(realm) self.assertEquals([], all_locations['cluster-1/some_db'].contains) self.assertEquals([], all_locations['cluster-1/some_db'].excludes) self.assertEquals([1], all_locations['dest2/some_db'].contains) self.assertEquals([], all_locations['dest2/some_db'].excludes)