def test_recover_of_one_key(cluster):
"""Test the case when cluster consists of 2 groups on different nodes and one of it has the only alive record.
Expect: the record will be server-sent to second group.
"""
mocked_server_send = elliptics.newapi.Session.return_value.server_send
mocked_server_send.return_value = [mock.MagicMock()]
recovery(one_node=False,
remotes=cluster.remotes,
backend_id=None,
address=None,
groups=cluster.groups,
rtype=RECOVERY.DC,
log_file='recovery.log',
tmp_dir='test_recover_of_one_key',
no_meta=True,
no_server_send=False,
expected_ret_code=0)
# expect that server-send will be called for the only record
mocked_server_send.assert_called_once_with(
keys=[record.key for record in cluster.backends[0].records],
src_group=cluster.backends[0].group_id,
dst_groups=[cluster.backends[1].group_id],
flags=0,
chunk_commit_timeout=1000,
chunk_retry_count=0,
chunk_size=1024,
chunk_write_timeout=1000)
def test_recover_of_one_uncommitted_key(cluster):
"""Test the case when cluster consists of 2 groups on different nodes and one of it has the only uncommitted record.
Uncommitted record is also out-dated.
Expect: the record will be removed and nothing will be server-sent.
"""
mocked_remove = elliptics.newapi.Session.return_value.remove
mocked_remove.return_value.get.return_value = [mock.MagicMock()]
mocked_server_send = elliptics.newapi.Session.return_value.server_send
recovery(one_node=False,
remotes=cluster.remotes,
backend_id=None,
address=None,
groups=cluster.groups,
rtype=RECOVERY.DC,
log_file='recovery.log',
tmp_dir='test_recover_of_one_uncommitted_key',
no_meta=True,
no_server_send=False,
expected_ret_code=0)
# nothing should be server-sent
mocked_server_send.assert_not_called()
# uncommitted key should be removed
mocked_remove.assert_called_once_with(cluster.backends[0].records[0].key)
def test_server_send_with_source_failure(cluster, error):
"""Test the case when the key should be recovered from two groups to third, but first group is experiencing an error
Make second try of server-send successful.
Expect: server-send will be called twice for different groups and recovery will be successful. The key should be
server-sent from second group to first one.
"""
test_key = cluster.backends[0].records[0].key
mocked_server_send = elliptics.newapi.Session.return_value.server_send
mocked_server_send.side_effect = [
mock.MagicMock(__iter__=mock.MagicMock(
return_value=iter([mock.MagicMock(key=test_key, status=error)]))),
mock.MagicMock(__iter__=mock.MagicMock(
return_value=iter([mock.MagicMock(key=test_key, status=0)])))
]
recovery(one_node=False,
remotes=cluster.remotes,
backend_id=None,
address=None,
groups=cluster.groups,
rtype=RECOVERY.DC,
log_file='recovery.log',
tmp_dir='test_failed_server_send' + str(error),
no_meta=True,
no_server_send=False,
expected_ret_code=0)
# server-send should be called twice
assert mocked_server_send.call_count == 2
mocked_server_send.assert_has_calls(calls=[
mock.call(
keys=[test_key],
flags=0, # default flags
chunk_size=1024, # default chunk_size
src_group=cluster.backends[0].
group_id, # first try should be to first group
dst_groups=[cluster.backends[2].group_id
], # first try should use third group as a destination
chunk_write_timeout=1000, # default timeout
chunk_commit_timeout=1000, # default timeout
chunk_retry_count=0), # default retry count
mock.call(
keys=[test_key],
flags=0, # default flags
chunk_size=1024, # default chunk_size
src_group=cluster.backends[1].
group_id, # second try should be to second group
# second try should use first and third groups as destinations
dst_groups=[
cluster.backends[0].group_id, cluster.backends[2].group_id
],
chunk_write_timeout=1000, # default timeout
chunk_commit_timeout=1000, # default timeout
chunk_retry_count=0), # default retry count
])
def test_dc_isolated_groups(self, servers):
'''
Write one key into every group,
run dc server-send recovery,
check that keys were recovered into all groups.
'''
scope.node = elliptics.Node(
elliptics.Logger("client.log", elliptics.log_level.debug))
scope.node.add_remotes(servers.remotes)
session = make_session(
node=scope.node,
test_name='TestIsolatedRecovery.test_dc_isolated_groups',
test_namespace=self.namespace)
groups = session.routes.groups()
scope.test_group = groups[0]
scope.test_group2 = groups[1]
routes = session.routes.filter_by_group(scope.test_group)
scope.test_address = routes[0].address
keys = []
data = 'isolated_data'
groups = (scope.test_group, scope.test_group2)
session.timestamp = elliptics.Time.now()
for group_id in groups:
key = 'isolated_key_{}'.format(group_id)
session.groups = [group_id]
write_data(scope, session, [key], [data])
check_data(scope, session, [key], [data], session.timestamp)
keys.append(key)
recovery(one_node=False,
remotes=map(elliptics.Address.from_host_port_family,
servers.remotes),
backend_id=None,
address=scope.test_address,
groups=groups,
rtype=RECOVERY.DC,
log_file='dc_isolated_groups.log',
tmp_dir='dc_isolated_groups')
for group_id in groups:
session.groups = [group_id]
check_data(scope, session, keys, [data] * len(keys),
session.timestamp)
def test_specific_case_with_readonly_groups(cluster):
"""Test the case when old version of recovery did wrong and left some keys non-recovered.
Initial state:
group #1 and #2 marked read-only
the key that is presented in group #1 and #2 but group #2 has newer version of the key
two keys that are presented in group #1 and #3
the key that is presented in group #1 and uncommitted in group #3
Expect: uncommitted replica will be removed from group #3 and will be server-sent from group #1 to #3,
the key with different replicas will be server-sent from group #2 to #3.
"""
mocked_remove = elliptics.newapi.Session.return_value.remove
mocked_remove.return_value.get.return_value = [mock.MagicMock(status=0)]
uncommitted_key = elliptics.Id.from_hex(
hashlib.sha512('key uncommitted in third group').hexdigest())
different_replicas_key = elliptics.Id.from_hex(
hashlib.sha512(
'key with different replicas in two groups and missed in third').
hexdigest())
mocked_server_send = elliptics.newapi.Session.return_value.server_send
mocked_server_send.side_effect = [
mock.MagicMock(__iter__=mock.MagicMock(return_value=iter([
mock.MagicMock(key=uncommitted_key, status=0),
]))),
mock.MagicMock(__iter__=mock.MagicMock(return_value=iter([
mock.MagicMock(key=different_replicas_key, status=0),
]))),
]
recovery(one_node=False,
remotes=cluster.remotes,
backend_id=None,
address=None,
groups=cluster.groups,
rtype=RECOVERY.DC,
log_file='recovery.log',
tmp_dir='test_specific_case_with_readonly_groups',
no_meta=True,
no_server_send=False,
expected_ret_code=0,
ro_groups={1, 2})
# uncommitted key should be removed from third group
mocked_remove.assert_called_once_with(uncommitted_key)
mocked_server_send.assert_has_calls(calls=[
mock.call(
keys=[uncommitted_key
], # firstly uncommitted key should be recovered
flags=0, # default flags
chunk_size=1024, # default chunk_size
# first server-send should be sent to group #1 because it has the largest number of keys to recover
src_group=1,
dst_groups=[3], # the key should be server-sent to group #3
chunk_write_timeout=1000, # default timeout
chunk_commit_timeout=1000, # default timeout
chunk_retry_count=0), # default retry count
mock.call(
keys=[
different_replicas_key
], # secondly key with different replicas should be recovered
flags=0, # default flags
chunk_size=1024, # default chunk_size
# second server-send should be to group #2, because all other server-sends should be skipped
# since group #1 and #2 are read-only
src_group=2,
dst_groups=[3], # the key should be server-sent to group #3
chunk_write_timeout=1000, # default timeout
chunk_commit_timeout=1000, # default timeout
chunk_retry_count=0), # default retry count
])
# only the two calls above should be made
assert mocked_server_send.call_count == 2