def _attempt_migration(self, num_records):
api.set_shard_at_rest('dummy', 1, "dest1/test_sharding")
api.set_shard_at_rest('dummy', 2, "dest1/test_sharding")
account_1 = self._prepare_account_data(
self.db1, 1, xrange(0, num_records))
account_2 = self._prepare_account_data(
self.db1, 2, xrange(0, num_records))
shard_manager = sharder._begin_migration(
'dummy', 1, "dest2/test_sharding")
self._modify_data(account_1, account_2, num_records, num_records)
while not shard_manager.is_finished():
time.sleep(0.01)
self._verify_end_state(
account_1, account_2, self.unwrapped_dummy_1, self.unwrapped_dummy_2)
# Check that the data for the other account has remained intact and in
# the same place
account_2_actual = list(self.unwrapped_dummy_1.find({'account_id': 2}))
account_2_actual = list(sorted(
account_2_actual, key=lambda r: r['some_key']))
self.assertEquals(account_2, account_2_actual)
# Now migrate back to the source
print 'Now migrate backwards...'
shard_manager = sharder._begin_migration(
'dummy', 1, "dest1/test_sharding")
self._modify_data(account_1, account_2, num_records * 2, num_records)
while not shard_manager.is_finished():
time.sleep(0.01)
self._verify_end_state(
account_1, account_2, self.unwrapped_dummy_2, self.unwrapped_dummy_1)
def _attempt_migration(self, num_records):
api.set_shard_at_rest('dummy', 1, "dest1/test_sharding")
api.set_shard_at_rest('dummy', 2, "dest1/test_sharding")
account_1 = self._prepare_account_data(self.db1, 1,
xrange(0, num_records))
account_2 = self._prepare_account_data(self.db1, 2,
xrange(0, num_records))
shard_manager = sharder._begin_migration('dummy', 1,
"dest2/test_sharding")
self._modify_data(account_1, account_2, num_records, num_records)
while not shard_manager.is_finished():
time.sleep(0.01)
self._verify_end_state(account_1, account_2, self.unwrapped_dummy_1,
self.unwrapped_dummy_2)
# Check that the data for the other account has remained intact and in
# the same place
account_2_actual = list(self.unwrapped_dummy_1.find({'account_id': 2}))
account_2_actual = list(
sorted(account_2_actual, key=lambda r: r['some_key']))
self.assertEquals(account_2, account_2_actual)
# Now migrate back to the source
print 'Now migrate backwards...'
shard_manager = sharder._begin_migration('dummy', 1,
"dest1/test_sharding")
self._modify_data(account_1, account_2, num_records * 2, num_records)
while not shard_manager.is_finished():
time.sleep(0.01)
self._verify_end_state(account_1, account_2, self.unwrapped_dummy_2,
self.unwrapped_dummy_1)
def test_index_inversion(self):
num_records = 200
api.set_shard_at_rest('dummy', 1, "dest1/test_sharding")
account_1 = self._prepare_account_data(
self.db1, 1, xrange(0, num_records))
shard_manager = sharder._begin_migration(
'dummy', 1, "dest2/test_sharding")
self._modify_data(account_1)
while not shard_manager.is_finished():
time.sleep(0.01)
self._verify_end_state(
account_1, self.unwrapped_dummy_1, self.unwrapped_dummy_2)
def test_index_inversion(self):
num_records = 200
api.set_shard_at_rest('dummy', 1, "dest1/test_sharding")
account_1 = self._prepare_account_data(self.db1, 1,
xrange(0, num_records))
shard_manager = sharder._begin_migration('dummy', 1,
"dest2/test_sharding")
self._modify_data(account_1)
while not shard_manager.is_finished():
time.sleep(0.01)
self._verify_end_state(account_1, self.unwrapped_dummy_1,
self.unwrapped_dummy_2)
def _attempt_migration(self, records):
api.set_shard_at_rest('dummy', 1, "dest1/test_sharding")
api.set_shard_at_rest('dummy', 2, "dest1/test_sharding")
account_1 = self._prepare_account_data(self.db1, 1, xrange(0, records))
account_2 = self._prepare_account_data(self.db1, 2, xrange(0, records))
shard_manager = sharder._begin_migration(
'dummy', 1, "dest2/test_sharding")
# Increment all the counters a few times
for cnt in range(3):
for record in account_1:
self.db1.dummy.update(
{'account_id': 1, 'some_key': record['some_key']},
{'$inc': {'counter': 1}},
)
record['counter'] += 1
# Delete all the counters that are divisible by 97 (including 0)
to_delete = []
for record in account_1:
if record['some_key'] % 97 == 0:
to_delete.append(record)
for record in to_delete:
account_1.remove(record)
self.db1.dummy.remove(
{'account_id': 1, 'some_key': record['some_key']})
# Add some additional records
account_1 += self._prepare_account_data(
self.db1, 1, xrange(records, 2 * records))
while not shard_manager.is_finished():
time.sleep(0.01)
# Fetch the data from the second server and check all the records match
account_1_actual = list(self.original_dummy_2.find({'account_id': 1}))
account_1_actual = list(sorted(
account_1_actual, key=lambda r: r['some_key']))
if account_1 != account_1_actual:
print 'Account 1 Expected'
print '------------------'
for doc in account_1:
print '(%s, %s) -> %d' % (
doc['account_id'], doc['some_key'], doc['counter'])
print 'Account 1 Actual'
print '----------------'
for doc in account_1_actual:
print '(%s, %s) -> %d (%s)' % (
doc['account_id'], doc['some_key'], doc['counter'], doc['_id'])
self.assertEquals(account_1, account_1_actual)
# Check that the data for the other account has remained intact and in
# the same place
account_2_actual = list(self.original_dummy_1.find({'account_id': 2}))
account_2_actual = list(sorted(
account_2_actual, key=lambda r: r['some_key']))
self.assertEquals(account_2, account_2_actual)
self.assertEquals(
0, self.original_dummy_1.find({'account_id': 1}).count())
def _attempt_migration(self, records):
api.set_shard_at_rest('dummy', 1, "dest1/test_sharding")
api.set_shard_at_rest('dummy', 2, "dest1/test_sharding")
account_1 = self._prepare_account_data(self.db1, 1, xrange(0, records))
account_2 = self._prepare_account_data(self.db1, 2, xrange(0, records))
shard_manager = sharder._begin_migration('dummy', 1,
"dest2/test_sharding")
# Increment all the counters a few times
for cnt in range(3):
for record in account_1:
self.db1.dummy.update(
{
'account_id': 1,
'some_key': record['some_key']
},
{'$inc': {
'counter': 1
}},
)
record['counter'] += 1
# Delete all the counters that are divisible by 97 (including 0)
to_delete = []
for record in account_1:
if record['some_key'] % 97 == 0:
to_delete.append(record)
for record in to_delete:
account_1.remove(record)
self.db1.dummy.remove({
'account_id': 1,
'some_key': record['some_key']
})
# Add some additional records
account_1 += self._prepare_account_data(self.db1, 1,
xrange(records, 2 * records))
while not shard_manager.is_finished():
time.sleep(0.01)
# Fetch the data from the second server and check all the records match
account_1_actual = list(self.original_dummy_2.find({'account_id': 1}))
account_1_actual = list(
sorted(account_1_actual, key=lambda r: r['some_key']))
if account_1 != account_1_actual:
print 'Account 1 Expected'
print '------------------'
for doc in account_1:
print '(%s, %s) -> %d' % (doc['account_id'], doc['some_key'],
doc['counter'])
print 'Account 1 Actual'
print '----------------'
for doc in account_1_actual:
print '(%s, %s) -> %d (%s)' % (doc['account_id'],
doc['some_key'],
doc['counter'], doc['_id'])
self.assertEquals(account_1, account_1_actual)
# Check that the data for the other account has remained intact and in
# the same place
account_2_actual = list(self.original_dummy_1.find({'account_id': 2}))
account_2_actual = list(
sorted(account_2_actual, key=lambda r: r['some_key']))
self.assertEquals(account_2, account_2_actual)
self.assertEquals(
0,
self.original_dummy_1.find({
'account_id': 1
}).count())