def test_update(self):
# Put the same document in multiple locations (a mid-migration status)
# then do an update and ensure that only the correct place has been
# updated.
api.set_shard_at_rest('dummy', 1, "dest1/test_sharding")
doc1 = {'x': 1, 'y': 1}
self.db1.dummy.insert(doc1)
api.start_migration('dummy', 1, 'dest2/test_sharding')
api.set_shard_to_migration_status(
'dummy', 1, api.ShardStatus.MIGRATING_COPY)
self.db2.dummy.insert(doc1)
result = operations.multishard_update('dummy', {}, {'$inc': {'y': 1}})
self.assertEquals(1, result['n'])
# Query the correct shard first and see that the counter has been
# incremented
result, = operations.multishard_find('dummy', {'x': 1})
self.assertEquals(2, result['y'])
# Now spoof the metadata such that the system thinks the data is on
# shard2. The counter should still be 1 here.
api.set_shard_at_rest('dummy', 1, "dest2/test_sharding", force=True)
result, = operations.multishard_find('dummy', {'x': 1})
self.assertEquals(1, result['y'])
def test_multishard_skip(self):
doc1 = {'x': 1, 'y': 1}
doc2 = {'x': 1, 'y': 2}
doc3 = {'x': 2, 'y': 1}
doc4 = {'x': 2, 'y': 2}
self.db1.dummy.insert(doc1)
self.db1.dummy.insert(doc2)
self.db2.dummy.insert(doc3)
self.db2.dummy.insert(doc4)
# TODO There is a bug here. The skip is done BEFORE the sory. Poo :(
results = operations.multishard_find(
'dummy', {}, sort=[('x', 1), ('y', 1)]).skip(1)
results = list(results)
self.assertEquals([doc2, doc3, doc4], results)
results = operations.multishard_find(
'dummy', {}, sort=[('x', 1), ('y', 1)]).skip(2)
self.assertEquals([doc3, doc4], list(results))
results = operations.multishard_find(
'dummy', {}, sort=[('x', 1), ('y', 1)]).skip(3)
self.assertEquals([doc4], list(results))
results = operations.multishard_find(
'dummy', {}, sort=[('x', 1), ('y', 1)]).skip(4)
self.assertEquals([], list(results))
def test_multishard_skip(self):
doc1 = {'x': 1, 'y': 1}
doc2 = {'x': 1, 'y': 2}
doc3 = {'x': 2, 'y': 1}
doc4 = {'x': 2, 'y': 2}
self.db1.dummy.insert(doc1)
self.db1.dummy.insert(doc2)
self.db2.dummy.insert(doc3)
self.db2.dummy.insert(doc4)
results = operations.multishard_find('dummy', {},
sort=[('x', 1), ('y', 1)]).skip(1)
self.assertEquals([doc2, doc3, doc4], list(results))
results = operations.multishard_find('dummy', {},
sort=[('x', 1), ('y', 1)]).skip(2)
self.assertEquals([doc3, doc4], list(results))
results = operations.multishard_find('dummy', {},
sort=[('x', 1), ('y', 1)]).skip(3)
self.assertEquals([doc4], list(results))
results = operations.multishard_find('dummy', {},
sort=[('x', 1), ('y', 1)]).skip(4)
self.assertEquals([], list(results))
def test_multishard_skip(self):
doc1 = {'x': 1, 'y': 1}
doc2 = {'x': 1, 'y': 2}
doc3 = {'x': 2, 'y': 1}
doc4 = {'x': 2, 'y': 2}
self.db1.dummy.insert(doc1)
self.db1.dummy.insert(doc2)
self.db2.dummy.insert(doc3)
self.db2.dummy.insert(doc4)
results = operations.multishard_find(
'dummy', {}, sort=[('x', 1), ('y', 1)]).skip(1)
self.assertEquals([doc2, doc3, doc4], list(results))
results = operations.multishard_find(
'dummy', {}, sort=[('x', 1), ('y', 1)]).skip(2)
self.assertEquals([doc3, doc4], list(results))
results = operations.multishard_find(
'dummy', {}, sort=[('x', 1), ('y', 1)]).skip(3)
self.assertEquals([doc4], list(results))
results = operations.multishard_find(
'dummy', {}, sort=[('x', 1), ('y', 1)]).skip(4)
self.assertEquals([], list(results))
def test_multishard_skip(self):
doc1 = {'x': 1, 'y': 1}
doc2 = {'x': 1, 'y': 2}
doc3 = {'x': 2, 'y': 1}
doc4 = {'x': 2, 'y': 2}
self.db1.dummy.insert(doc1)
self.db1.dummy.insert(doc2)
self.db2.dummy.insert(doc3)
self.db2.dummy.insert(doc4)
# TODO There is a bug here. The skip is done BEFORE the sory. Poo :(
results = operations.multishard_find('dummy', {},
sort=[('x', 1), ('y', 1)]).skip(1)
results = list(results)
self.assertEquals([doc2, doc3, doc4], results)
results = operations.multishard_find('dummy', {},
sort=[('x', 1), ('y', 1)]).skip(2)
self.assertEquals([doc3, doc4], list(results))
results = operations.multishard_find('dummy', {},
sort=[('x', 1), ('y', 1)]).skip(3)
self.assertEquals([doc4], list(results))
results = operations.multishard_find('dummy', {},
sort=[('x', 1), ('y', 1)]).skip(4)
self.assertEquals([], list(results))
def test_getitem_on_non_targetted_query(self):
"""This tests a bug that was found in a production environment. If a
scatter-gather query is performed and data is only on one shard then if
the queries are performed in a certain order the getitem will fail due
to no results being found.
"""
# Test db1 with all the data and db2 without any.
self.db1.dummy.insert({'x': 1, 'y': 1})
self.db1.dummy.insert({'x': 1, 'y': 2})
expected = {'x': 1, 'y': 3}
self.db1.dummy.insert(expected)
result = operations.multishard_find('dummy', {})\
.sort([('y', 1)])[2]
self.assertEquals(result, expected)
# Now test the other way around to ensure we capture all orderings.
# Add a z field for querying to ensure db1 returns 0 results.
self.db2.dummy.insert({'x': 2, 'y': 1, 'z': 1})
self.db2.dummy.insert({'x': 2, 'y': 2, 'z': 1})
expected = {'x': 2, 'y': 3, 'z': 1}
self.db2.dummy.insert(expected)
result = operations.multishard_find('dummy', {'z': 1})\
.sort([('y', 1)])[2]
self.assertEquals(result, expected)
def test_skips_earlier_rows_across_shards(self):
doc1 = {'x': 1, 'y': 1}
doc2 = {'x': 1, 'y': 2}
doc3 = {'x': 2, 'y': 1}
doc4 = {'x': 2, 'y': 2}
self.db1.dummy.insert(doc1)
self.db1.dummy.insert(doc2)
self.db2.dummy.insert(doc3)
self.db2.dummy.insert(doc4)
cursor = operations.multishard_find('dummy', {},
sort=[('x', 1), ('y', 1)]).skip(1)
self.assertEquals([doc2, doc3, doc4], list(cursor))
cursor = operations.multishard_find('dummy', {},
sort=[('x', 1), ('y', 1)]).skip(2)
self.assertEquals([doc3, doc4], list(cursor))
cursor = operations.multishard_find('dummy', {},
sort=[('x', 1), ('y', 1)]).skip(3)
self.assertEquals([doc4], list(cursor))
cursor = operations.multishard_find('dummy', {},
sort=[('x', 1), ('y', 1)]).skip(4)
self.assertEquals([], list(cursor))
def test_update(self):
# Put the same document in multiple locations (a mid-migration status)
# then do an update and ensure that only the correct place has been
# updated.
api.set_shard_at_rest('dummy', 1, "dest1/test_sharding")
doc1 = {'x': 1, 'y': 1}
self.db1.dummy.insert(doc1)
api.start_migration('dummy', 1, 'dest2/test_sharding')
api.set_shard_to_migration_status('dummy', 1,
api.ShardStatus.MIGRATING_COPY)
self.db2.dummy.insert(doc1)
result = operations.multishard_update('dummy', {}, {'$inc': {'y': 1}})
self.assertEquals(1, result['n'])
# Query the correct shard first and see that the counter has been
# incremented
result, = operations.multishard_find('dummy', {'x': 1})
self.assertEquals(2, result['y'])
# Now spoof the metadata such that the system thinks the data is on
# shard2. The counter should still be 1 here.
api.set_shard_at_rest('dummy', 1, "dest2/test_sharding", force=True)
result, = operations.multishard_find('dummy', {'x': 1})
self.assertEquals(1, result['y'])
def test_multishard_find_with_sort(self):
doc1 = {'x': 1, 'y': 1}
doc2 = {'x': 1, 'y': 2}
doc3 = {'x': 2, 'y': 1}
doc4 = {'x': 2, 'y': 2}
self.db1.dummy.insert(doc1)
self.db1.dummy.insert(doc2)
self.db2.dummy.insert(doc3)
self.db2.dummy.insert(doc4)
results = operations.multishard_find(
'dummy', {}, sort=[('x', 1), ('y', 1)])
self.assertEquals([doc1, doc2, doc3, doc4], list(results))
results = operations.multishard_find(
'dummy', {}, sort=[('x', -1), ('y', 1)])
self.assertEquals([doc3, doc4, doc1, doc2], list(results))
results = operations.multishard_find(
'dummy', {}, sort=[('x', 1), ('y', -1)])
self.assertEquals([doc2, doc1, doc4, doc3], list(results))
results = operations.multishard_find(
'dummy', {}, sort=[('x', -1), ('y', -1)])
self.assertEquals([doc4, doc3, doc2, doc1], list(results))
# Insert a document the same as doc4 to ensure sorts will cope with
# things that are basically the same
doc5 = {'x': 2, 'y': 2, 'z': 1}
self.db2.dummy.insert(doc5)
results = operations.multishard_find(
'dummy', {}, sort=[('x', -1), ('y', -1)])
results = results[:2]
self.assertTrue(doc4 in results)
self.assertTrue(doc5 in results)
def test_multishard_count_with_motion(self):
api.set_shard_at_rest('dummy', 1, "dest1/test_sharding")
api.set_shard_at_rest('dummy', 2, "dest1/test_sharding")
doc1 = {'x': 1, 'y': 1}
doc2 = {'x': 1, 'y': 2}
doc3 = {'x': 2, 'y': 1}
doc4 = {'x': 2, 'y': 2}
self.db1.dummy.insert(doc1)
self.db1.dummy.insert(doc2)
self.db1.dummy.insert(doc3)
self.db1.dummy.insert(doc4)
results = operations.multishard_find('dummy', {}).count()
self.assertEquals(4, results)
# Mimic the shard now being in the second location and there being
# documents left here
api.start_migration('dummy', 2, "dest2/test_sharding")
api.set_shard_to_migration_status(
'dummy', 2, api.ShardStatus.POST_MIGRATION_PAUSED_AT_DESTINATION)
self.db2.dummy.insert(doc3)
self.db2.dummy.insert(doc4)
results = operations.multishard_find('dummy', {}).count()
self.assertEquals(4, results)
def test_multishard_count_with_motion(self):
api.set_shard_at_rest('dummy', 1, "dest1/test_sharding")
api.set_shard_at_rest('dummy', 2, "dest1/test_sharding")
doc1 = {'x': 1, 'y': 1}
doc2 = {'x': 1, 'y': 2}
doc3 = {'x': 2, 'y': 1}
doc4 = {'x': 2, 'y': 2}
self.db1.dummy.insert(doc1)
self.db1.dummy.insert(doc2)
self.db1.dummy.insert(doc3)
self.db1.dummy.insert(doc4)
results = operations.multishard_find('dummy', {}).count()
self.assertEquals(4, results)
# Mimic the shard now being in the second location and there being
# documents left here
api.start_migration('dummy', 2, "dest2/test_sharding")
api.set_shard_to_migration_status(
'dummy', 2, api.ShardStatus.POST_MIGRATION_PAUSED_AT_DESTINATION)
self.db2.dummy.insert(doc3)
self.db2.dummy.insert(doc4)
results = operations.multishard_find('dummy', {}).count()
self.assertEquals(4, results)
def test_multishard_find_with_sort(self):
doc1 = {'x': 1, 'y': 1}
doc2 = {'x': 1, 'y': 2}
doc3 = {'x': 2, 'y': 1}
doc4 = {'x': 2, 'y': 2}
self.db1.dummy.insert(doc1)
self.db1.dummy.insert(doc2)
self.db2.dummy.insert(doc3)
self.db2.dummy.insert(doc4)
results = operations.multishard_find('dummy', {},
sort=[('x', 1), ('y', 1)])
self.assertEquals([doc1, doc2, doc3, doc4], list(results))
results = operations.multishard_find('dummy', {},
sort=[('x', -1), ('y', 1)])
self.assertEquals([doc3, doc4, doc1, doc2], list(results))
results = operations.multishard_find('dummy', {},
sort=[('x', 1), ('y', -1)])
self.assertEquals([doc2, doc1, doc4, doc3], list(results))
results = operations.multishard_find('dummy', {},
sort=[('x', -1), ('y', -1)])
self.assertEquals([doc4, doc3, doc2, doc1], list(results))
# Insert a document the same as doc4 to ensure sorts will cope with
# things that are basically the same
doc5 = {'x': 2, 'y': 2, 'z': 1}
self.db2.dummy.insert(doc5)
results = operations.multishard_find('dummy', {},
sort=[('x', -1), ('y', -1)])
results = results[:2]
self.assertTrue(doc4 in results)
self.assertTrue(doc5 in results)
def test_getitem_on_non_targetted_query(self):
"""This tests a bug that was found in a production environment. If a
scatter-gather query is performed and data is only on one shard then if
the queries are performed in a certain order the getitem will fail due
to no results being found.
"""
# Test db1 with all the data and db2 without any.
self.db1.dummy.insert({'x': 1, 'y': 1})
self.db1.dummy.insert({'x': 1, 'y': 2})
expected = {'x': 1, 'y': 3}
self.db1.dummy.insert(expected)
result = operations.multishard_find('dummy', {})\
.sort([('y', 1)])[2]
self.assertEquals(result, expected)
# Now test the other way around to ensure we capture all orderings.
# Add a z field for querying to ensure db1 returns 0 results.
self.db2.dummy.insert({'x': 2, 'y': 1, 'z': 1})
self.db2.dummy.insert({'x': 2, 'y': 2, 'z': 1})
expected = {'x': 2, 'y': 3, 'z': 1}
self.db2.dummy.insert(expected)
result = operations.multishard_find('dummy', {'z': 1})\
.sort([('y', 1)])[2]
self.assertEquals(result, expected)
def test_multishard_find_with_sort_as_single_arg(self):
doc1 = {'x': 1, 'y': 1}
doc2 = {'x': 2, 'y': 1}
self.db1.dummy.insert(doc1)
self.db2.dummy.insert(doc2)
results = operations.multishard_find('dummy', {}).sort('x', 1)
self.assertEquals([doc1, doc2], list(results))
results = operations.multishard_find('dummy', {}).sort('x', -1)
self.assertEquals([doc2, doc1], list(results))
def test_indexed_read(self):
doc1 = {'x': 1, 'y': 1}
doc2 = {'x': 2, 'y': 1}
self.db1.dummy.insert(doc1)
self.db2.dummy.insert(doc2)
cursor = operations.multishard_find('dummy', {'y': 1},
sort=[('x', 1), ('y', 1)])
self.assertEquals(doc1, cursor[0])
cursor = operations.multishard_find('dummy', {'y': 1},
sort=[('x', -1), ('y', 1)])
self.assertEquals(doc2, cursor[0])
def test_indexed_read(self):
doc1 = {'x': 1, 'y': 1}
doc2 = {'x': 2, 'y': 1}
self.db1.dummy.insert(doc1)
self.db2.dummy.insert(doc2)
cursor = operations.multishard_find(
'dummy', {'y': 1}, sort=[('x', 1), ('y', 1)])
self.assertEquals(doc1, cursor[0])
cursor = operations.multishard_find(
'dummy', {'y': 1}, sort=[('x', -1), ('y', 1)])
self.assertEquals(doc2, cursor[0])
def test_multishard_find_with_sort_as_single_arg(self):
doc1 = {'x': 1, 'y': 1}
doc2 = {'x': 2, 'y': 1}
self.db1.dummy.insert(doc1)
self.db2.dummy.insert(doc2)
results = operations.multishard_find(
'dummy', {}).sort('x', 1)
self.assertEquals([doc1, doc2], list(results))
results = operations.multishard_find(
'dummy', {}).sort('x', -1)
self.assertEquals([doc2, doc1], list(results))
def test_indexed_read(self):
api.set_shard_at_rest('dummy', 1, 'dest1/test_sharding')
api.set_shard_at_rest('dummy', 2, 'dest2/test_sharding')
doc1 = {'x': 1, 'y': 1}
doc2 = {'x': 2, 'y': 1}
self.db1.dummy.insert(doc1)
self.db2.dummy.insert(doc2)
cursor = operations.multishard_find('dummy', {'y': 1},
sort=[('x', 1), ('y', 1)])
self.assertEquals(doc1, cursor[0])
cursor = operations.multishard_find('dummy', {'y': 1},
sort=[('x', -1), ('y', 1)])
self.assertEquals(doc2, cursor[0])
def test_indexed_read(self):
api.set_shard_at_rest('dummy', 1, 'dest1/test_sharding')
api.set_shard_at_rest('dummy', 2, 'dest2/test_sharding')
doc1 = {'x': 1, 'y': 1}
doc2 = {'x': 2, 'y': 1}
self.db1.dummy.insert(doc1)
self.db2.dummy.insert(doc2)
cursor = operations.multishard_find(
'dummy', {'y': 1}, sort=[('x', 1), ('y', 1)])
self.assertEquals(doc1, cursor[0])
cursor = operations.multishard_find(
'dummy', {'y': 1}, sort=[('x', -1), ('y', 1)])
self.assertEquals(doc2, cursor[0])
def test_multi_update(self):
# Test that an update will hit multiple clusters at once
doc1 = {'x': 1, 'y': 1}
doc2 = {'x': 2, 'y': 1}
self.db1.dummy.insert(doc1)
self.db2.dummy.insert(doc2)
result = operations.multishard_update('dummy', {}, {'$inc': {'y': 1}})
self.assertEquals(2, result['n'])
result, = operations.multishard_find('dummy', {'x': 1})
self.assertEquals(2, result['y'])
result, = operations.multishard_find('dummy', {'x': 2})
self.assertEquals(2, result['y'])
def test_multi_update(self):
# Test that an update will hit multiple clusters at once
doc1 = {'x': 1, 'y': 1}
doc2 = {'x': 2, 'y': 1}
self.db1.dummy.insert(doc1)
self.db2.dummy.insert(doc2)
result = operations.multishard_update('dummy', {}, {'$inc': {'y': 1}})
self.assertEquals(2, result['n'])
result, = operations.multishard_find('dummy', {'x': 1})
self.assertEquals(2, result['y'])
result, = operations.multishard_find('dummy', {'x': 2})
self.assertEquals(2, result['y'])
def test_alive(self):
api.set_shard_at_rest('dummy', 1, "dest1/test_sharding")
doc1 = {'x': 1, 'y': 1}
self.db1.dummy.insert(doc1)
c = operations.multishard_find('dummy', {})
self.assertTrue(c.alive)
def test_alive(self):
api.set_shard_at_rest('dummy', 1, "dest1/test_sharding")
doc1 = {'x': 1, 'y': 1}
self.db1.dummy.insert(doc1)
c = operations.multishard_find('dummy', {})
self.assertTrue(c.alive)
def test_multishard_find_with_sort_fn(self):
doc1 = {'x': 1, 'y': 1}
doc2 = {'x': 1, 'y': 2}
doc3 = {'x': 2, 'y': 1}
doc4 = {'x': 2, 'y': 2}
self.db1.dummy.insert(doc1)
self.db1.dummy.insert(doc2)
self.db2.dummy.insert(doc3)
self.db2.dummy.insert(doc4)
results = operations.multishard_find('dummy', {}).sort([('x', 1),
('y', 1)])
self.assertEquals([doc1, doc2, doc3, doc4], list(results))
results = operations.multishard_find('dummy', {}).sort([('x', -1),
('y', 1)])
self.assertEquals([doc3, doc4, doc1, doc2], list(results))
def test_non_zero_indexing(self):
doc1 = {'x': 1, 'y': 1}
doc2 = {'x': 2, 'y': 1}
self.db1.dummy.insert(doc1)
self.db2.dummy.insert(doc2)
result = operations.multishard_find('dummy', {'y': 1})[1]
self.assertEquals(doc2, result)
def test_multishard_find_with_sort_fn(self):
doc1 = {'x': 1, 'y': 1}
doc2 = {'x': 1, 'y': 2}
doc3 = {'x': 2, 'y': 1}
doc4 = {'x': 2, 'y': 2}
self.db1.dummy.insert(doc1)
self.db1.dummy.insert(doc2)
self.db2.dummy.insert(doc3)
self.db2.dummy.insert(doc4)
results = operations.multishard_find(
'dummy', {}).sort([('x', 1), ('y', 1)])
self.assertEquals([doc1, doc2, doc3, doc4], list(results))
results = operations.multishard_find(
'dummy', {}).sort([('x', -1), ('y', 1)])
self.assertEquals([doc3, doc4, doc1, doc2], list(results))
def test_non_zero_indexing(self):
doc1 = {'x': 1, 'y': 1}
doc2 = {'x': 2, 'y': 1}
self.db1.dummy.insert(doc1)
self.db2.dummy.insert(doc2)
result = operations.multishard_find('dummy', {'y': 1})[1]
self.assertEquals(doc2, result)
def test_multishard_find_clone_with_read_preference(self):
cursor = operations.multishard_find(
collection_name='dummy',
query={},
with_options={'read_preference': ReadPreference.SECONDARY})
cloned_cursor = cursor.clone()
self.assertEqual(cloned_cursor.with_options,
{'read_preference': ReadPreference.SECONDARY})
def test_skip_slice(self):
doc1 = {'x': 1, 'y': 1}
doc2 = {'x': 2, 'y': 1}
self.db1.dummy.insert(doc1)
self.db2.dummy.insert(doc2)
c = operations.multishard_find('dummy', {'y': 1})[1:]
results = sorted(list(c), key=lambda d: d['x'])
self.assertEquals([doc2], results)
def test_multishard_find_args(self):
doc1 = {'x': 1, 'y': 1}
doc2 = {'x': 2, 'y': 1}
self.db1.dummy.insert(doc1)
self.db2.dummy.insert(doc2)
c = operations.multishard_find('dummy', {'y': 1}, {'x': 1, '_id': 0})
results = sorted(list(c), key=lambda d: d['x'])
self.assertEquals([{'x': 1}, {'x': 2}], results)
def test_multishard_find_args(self):
doc1 = {'x': 1, 'y': 1}
doc2 = {'x': 2, 'y': 1}
self.db1.dummy.insert(doc1)
self.db2.dummy.insert(doc2)
c = operations.multishard_find('dummy', {'y': 1}, {'x': 1, '_id': 0})
results = sorted(list(c), key=lambda d: d['x'])
self.assertEquals([{'x': 1}, {'x': 2}], results)
def test_skip_slice(self):
doc1 = {'x': 1, 'y': 1}
doc2 = {'x': 2, 'y': 1}
self.db1.dummy.insert(doc1)
self.db2.dummy.insert(doc2)
c = operations.multishard_find('dummy', {'y': 1})[1:]
results = sorted(list(c), key=lambda d: d['x'])
self.assertEquals([doc2], results)
def test_can_index_an_item_across_shards(self):
doc1 = {'x': 1, 'y': 1}
doc2 = {'x': 2, 'y': 1}
self.db1.dummy.insert(doc1)
self.db2.dummy.insert(doc2)
result = operations.multishard_find('dummy', {'y': 1},
sort=[('x', 1)])[1]
self.assertEquals(doc2, result)
def test_multishard_find_compound_key(self):
doc1 = {'x': 1, 'a': 1, 'y': {'z': 1}}
doc2 = {'x': 2, 'a': 1, 'y': {'z': 2}}
self.db1.dummy.insert(doc1)
self.db2.dummy.insert(doc2)
c = operations.multishard_find('dummy', {'a': 1}).sort([('y.z', 1)])
results = sorted(list(c), key=lambda d: d['x'])
self.assertEquals([doc1, doc2], results)
def test_can_slice_ordered_data_across_shards(self):
doc1 = {'x': 1, 'y': 1}
doc2 = {'x': 2, 'y': 1}
self.db1.dummy.insert(doc1)
self.db2.dummy.insert(doc2)
cursor = operations.multishard_find('dummy', {'y': 1},
sort=[('x', 1)])[1:]
self.assertEquals([doc2], list(cursor))
def test_skip_beyond_limit(self):
self.db1.dummy.insert({'x': 1, 'y': 1})
self.db1.dummy.insert({'x': 1, 'y': 1})
self.db1.dummy.insert({'x': 1, 'y': 1})
self.db1.dummy.insert({'x': 1, 'y': 1})
expected_doc = {'x': 2, 'y': 1}
self.db2.dummy.insert(expected_doc)
result = operations.multishard_find('dummy', {'y': 1}).limit(1).skip(4)
self.assertEquals([expected_doc], list(result))
def test_skip_beyond_limit(self):
self.db1.dummy.insert({'x': 1, 'y': 1})
self.db1.dummy.insert({'x': 1, 'y': 1})
self.db1.dummy.insert({'x': 1, 'y': 1})
self.db1.dummy.insert({'x': 1, 'y': 1})
expected_doc = {'x': 2, 'y': 1}
self.db2.dummy.insert(expected_doc)
result = operations.multishard_find('dummy', {'y': 1}).limit(1).skip(4)
self.assertEquals([expected_doc], list(result))
def test_multishard_find(self):
api.set_shard_at_rest('dummy', 1, 'dest1/test_sharding')
api.set_shard_at_rest('dummy', 2, 'dest2/test_sharding')
doc1 = {'x': 1, 'y': 1}
doc2 = {'x': 2, 'y': 1}
self.db1.dummy.insert(doc1)
self.db2.dummy.insert(doc2)
c = operations.multishard_find('dummy', {'y': 1})
results = sorted(list(c), key=lambda d: d['x'])
self.assertEquals([doc1, doc2], results)
def test_multishard_find(self):
api.set_shard_at_rest('dummy', 1, 'dest1/test_sharding')
api.set_shard_at_rest('dummy', 2, 'dest2/test_sharding')
doc1 = {'x': 1, 'y': 1}
doc2 = {'x': 2, 'y': 1}
self.db1.dummy.insert(doc1)
self.db2.dummy.insert(doc2)
c = operations.multishard_find('dummy', {'y': 1})
results = sorted(list(c), key=lambda d: d['x'])
self.assertEquals([doc1, doc2], results)
def test_multishard_rewind(self):
doc1 = {'x': 1, 'y': 1}
doc2 = {'x': 2, 'y': 1}
self.db1.dummy.insert(doc1)
self.db2.dummy.insert(doc2)
cursor = operations.multishard_find('dummy', {'y': 1}, sort=[('x', 1)])
found = cursor.next()
self.assertEquals((1, 1), (found['x'], found['y']))
cursor.rewind()
found = cursor.next()
self.assertEquals((1, 1), (found['x'], found['y']))
def test_alive_with_sort(self):
doc1 = {'x': 1, 'y': 1}
doc2 = {'x': 2, 'y': 1}
self.db1.dummy.insert(doc1)
self.db2.dummy.insert(doc2)
c = operations.multishard_find('dummy', {}).sort('x', 1)
self.assertTrue(c.alive)
c.next()
self.assertTrue(c.alive)
c.next()
self.assertFalse(c.alive)
def test_cursor_explain_not_called_on_find(self):
doc1 = {'x': 1, 'y': 1}
doc2 = {'x': 2, 'y': 1}
self.db1.dummy.insert(doc1)
self.db2.dummy.insert(doc2)
with patch.object(Cursor, 'explain') as explain_mock:
c = operations.multishard_find('dummy', {'y': 1}, sort=[('x', 1)])
list(c)
self.assertFalse(explain_mock.called)
c.explain()
self.assertTrue(explain_mock.called)
def test_multishard_rewind(self):
doc1 = {'x': 1, 'y': 1}
doc2 = {'x': 2, 'y': 1}
self.db1.dummy.insert(doc1)
self.db2.dummy.insert(doc2)
cursor = operations.multishard_find('dummy', {'y': 1}, sort=[('x', 1)])
found = cursor.next()
self.assertEquals((1, 1), (found['x'], found['y']))
cursor.rewind()
found = cursor.next()
self.assertEquals((1, 1), (found['x'], found['y']))
def test_untargetted_query_callback(self):
_callback = Mock()
doc1 = {'x': 1, 'y': 1}
doc2 = {'x': 2, 'y': 1}
self.db1.dummy.insert(doc1)
self.db2.dummy.insert(doc2)
api.set_untargetted_query_callback(_callback)
list(operations.multishard_find('dummy', {'y': 1}))
_callback.assert_called_with('dummy', {'y': 1})
def test_cursor_explain_not_called_on_find(self):
doc1 = {'x': 1, 'y': 1}
doc2 = {'x': 2, 'y': 1}
self.db1.dummy.insert(doc1)
self.db2.dummy.insert(doc2)
with patch.object(Cursor, 'explain') as explain_mock:
c = operations.multishard_find('dummy', {'y': 1}, sort=[('x', 1)])
list(c)
self.assertFalse(explain_mock.called)
c.explain()
self.assertTrue(explain_mock.called)
def test_multishard_find_with_limit_as_method(self):
doc1 = {'x': 1, 'y': 1}
doc2 = {'x': 1, 'y': 2}
doc3 = {'x': 2, 'y': 1}
doc4 = {'x': 2, 'y': 2}
self.db1.dummy.insert(doc1)
self.db1.dummy.insert(doc2)
self.db2.dummy.insert(doc3)
self.db2.dummy.insert(doc4)
results = operations.multishard_find('dummy', {}).limit(3)
self.assertEquals(3, len(list(results)))
def test_multishard_find_with_limit_as_method(self):
doc1 = {'x': 1, 'y': 1}
doc2 = {'x': 1, 'y': 2}
doc3 = {'x': 2, 'y': 1}
doc4 = {'x': 2, 'y': 2}
self.db1.dummy.insert(doc1)
self.db1.dummy.insert(doc2)
self.db2.dummy.insert(doc3)
self.db2.dummy.insert(doc4)
results = operations.multishard_find('dummy', {}).limit(3)
self.assertEquals(3, len(list(results)))
def test_untargetted_query_callback(self):
_callback = Mock()
doc1 = {'x': 1, 'y': 1}
doc2 = {'x': 2, 'y': 1}
self.db1.dummy.insert(doc1)
self.db2.dummy.insert(doc2)
api.set_untargetted_query_callback(_callback)
list(operations.multishard_find('dummy', {'y': 1}))
_callback.assert_called_with('dummy', {'y': 1})
def test_alive_with_sort(self):
doc1 = {'x': 1, 'y': 1}
doc2 = {'x': 2, 'y': 1}
self.db1.dummy.insert(doc1)
self.db2.dummy.insert(doc2)
c = operations.multishard_find('dummy', {}).sort('x', 1)
self.assertTrue(c.alive)
c.next()
self.assertTrue(c.alive)
c.next()
self.assertFalse(c.alive)
def test_non_zero_indexing(self):
doc1 = {'x': 1, 'y': 1}
doc2 = {'x': 2, 'y': 1}
self.db1.dummy.insert(doc1)
self.db2.dummy.insert(doc2)
result = operations.multishard_find('dummy', {'y': 1})[1]
# As there is no sort order on the query and the data is in two
# locations the order of the results is not defined. Therefore, we have
# to check for one of two possible outcomes based on which shard was
# queried first.
self.assertTrue(result == doc1 or result == doc2)
def test_skip_slice(self):
doc1 = {'x': 1, 'y': 1}
doc2 = {'x': 2, 'y': 1}
self.db1.dummy.insert(doc1)
self.db2.dummy.insert(doc2)
c = operations.multishard_find('dummy', {'y': 1})[1:]
results = list(c)
# As there is no sort order on the query and the data is in two
# locations the order of the results is not defined. Therefore, we have
# to test that we have skipped *a* document.
self.assertTrue(results == [doc1] or results == [doc2])
def test_non_zero_indexing(self):
doc1 = {'x': 1, 'y': 1}
doc2 = {'x': 2, 'y': 1}
self.db1.dummy.insert(doc1)
self.db2.dummy.insert(doc2)
result = operations.multishard_find('dummy', {'y': 1})[1]
# As there is no sort order on the query and the data is in two
# locations the order of the results is not defined. Therefore, we have
# to check for one of two possible outcomes based on which shard was
# queried first.
self.assertTrue(result == doc1 or result == doc2)
def test_multishard_find_with_sort_and_limit(self):
doc1 = {'x': 1, 'y': 1}
doc2 = {'x': 1, 'y': 2}
doc3 = {'x': 2, 'y': 1}
doc4 = {'x': 2, 'y': 2}
self.db1.dummy.insert(doc1)
self.db1.dummy.insert(doc2)
self.db2.dummy.insert(doc3)
self.db2.dummy.insert(doc4)
results = operations.multishard_find(
'dummy', {}, sort=[('x', 1), ('y', 1)], limit=3)
self.assertEquals([doc1, doc2, doc3], list(results))
def test_multishard_batch_size_paging(self):
for i in xrange(10):
self.db1.dummy.insert({'x': 1, 'y': i})
self.db2.dummy.insert({'x': 2, 'y': i + 10})
qs = operations.multishard_find('dummy', {}).batch_size(5)
qs = qs.sort([('y', 1)]).limit(15)
cnt = 0
for c in qs:
cnt += 1
self.assertEqual(cnt, 15)
self.assertEqual(c, {'x': 2, 'y': 14, '_id': c['_id']})
def test_multishard_find_with_limit(self):
api.set_shard_at_rest('dummy', 1, "dest1/test_sharding")
api.set_shard_at_rest('dummy', 2, "dest2/test_sharding")
doc1 = {'x': 1, 'y': 1}
doc2 = {'x': 1, 'y': 2}
doc3 = {'x': 2, 'y': 1}
doc4 = {'x': 2, 'y': 2}
self.db1.dummy.insert(doc1)
self.db1.dummy.insert(doc2)
self.db2.dummy.insert(doc3)
self.db2.dummy.insert(doc4)
results = operations.multishard_find('dummy', {}, limit=3)
self.assertEquals(3, len(list(results)))
def test_multishard_find_with_shardkey_present(self):
# Create an unlikely scenario where data is present in both locations
# even though the data is "at rest". Then do a query with the shard
# key present. As the shard key is present the query should be
# targetted enough to ignore the dummy data
doc1 = {'x': 1, 'y': 1}
doc2_bad = {'x': 2, 'y': 1, 'bad': True}
doc2_good = {'x': 2, 'y': 1, 'bad': False}
self.db1.dummy.insert(doc1)
self.db1.dummy.insert(doc2_bad)
self.db2.dummy.insert(doc2_good)
results = operations.multishard_find('dummy', {'x': 2, 'y': 1})
self.assertEquals([doc2_good], list(results))
def test_hint(self):
# The easier way to test if a hint is being applied is to apply a bad
# hint and watch things blow up
doc1 = {'x': 1, 'y': 1}
doc2 = {'x': 2, 'y': 1}
self.db1.dummy.insert(doc1)
self.db2.dummy.insert(doc2)
# This should explode due to a lack of index
try:
c = operations.multishard_find('dummy', {'x': 1})
c = c.hint([('apples', 1)])
list(c)
except OperationFailure as e:
self.assertTrue("bad hint" in str(e))
def test_targetted_skip_and_limit_with_sort(self):
doc1 = {'x': 1, 'y': 1}
self.db1.dummy.insert(doc1)
doc2 = {'x': 1, 'y': 2}
self.db1.dummy.insert(doc2)
doc3 = {'x': 1, 'y': 3}
self.db1.dummy.insert(doc3)
doc4 = {'x': 1, 'y': 4}
self.db1.dummy.insert(doc4)
doc5 = {'x': 1, 'y': 5}
self.db2.dummy.insert(doc5)
qs = operations.multishard_find('dummy', {'x': 1})
qs = qs.sort([('y', 1)]).skip(1).limit(3)
result = list(qs)
self.assertEquals([doc2, doc3, doc4], result)
def test_explain(self):
doc1 = {'x': 1, 'y': 1}
doc2 = {'x': 2, 'y': 1}
self.db1.dummy.insert(doc1)
self.db2.dummy.insert(doc2)
c = operations.multishard_find('dummy', {'y': 1}, sort=[('x', 1)])
list(c)
explains = c.explain()
# mongo 2.6 and 3+ have differing explain output makes looking at more
# interesting values (like queryPlanner->parsedQuery) difficult
# without enforcing a particular version of mongo on the tester.
self.assertTrue(all([
set(['queryPlanner', 'allPlans']) & set(e.keys()) != set()
for (location, e) in explains.iteritems()
]))
