def setUp(self):
subprocess.call = self.__subprocess_mock
self.__backup_controller = BackupController(COMMIT_WAIT_IN_SECONDS)
self.__backup_controller.set_retry_count(1)
self.__backup_controller.set_retry_wait(0)
self.__backup_controller.set_restore_retry_count(1)
self.__backup_controller.set_restore_retry_wait(0)
class TestBackupController(TestCase):
__backup_controller = None
__subprocess_mock = MagicMock(return_value=0)
def setUp(self):
subprocess.call = self.__subprocess_mock
self.__backup_controller = BackupController(COMMIT_WAIT_IN_SECONDS)
self.__backup_controller.set_retry_count(1)
self.__backup_controller.set_retry_wait(0)
self.__backup_controller.set_restore_retry_count(1)
self.__backup_controller.set_restore_retry_wait(0)
def test_should_restore_backup_for_local_shard(self):
timestamp = datetime.utcnow().strftime('%Y%m%d%H%M')
backup_dir = BACKUP_ROOT_DIR + timestamp
test_backup_file_name = 'backup_' + timestamp + '_' + TEST_COLLECTION + '_' + TEST_SHARD + '.tar.gz'
test_core_name = TEST_COLLECTION + '_' + TEST_SHARD + '_' + TEST_REPLICA_1
local_cores_url = LOCAL_URL + '/admin/cores?action=STATUS&wt=json'
trigger_restore_url = LOCAL_URL + '/' + test_core_name + '/replication?command=restore&wt=json&location='\
+ backup_dir + '&name=' + TEST_COLLECTION + '_' + TEST_SHARD
check_restore_status_url = LOCAL_URL + '/' + test_core_name + '/replication?command=restorestatus&wt=json'
http_responses = [
self.__side_effect_local_cores(urllib.request.Request(local_cores_url)),
self.__side_effect_all_ok(urllib.request.Request(trigger_restore_url)),
self.__side_effect_restore_in_progress(urllib.request.Request(check_restore_status_url)),
self.__side_effect_restore_done(urllib.request.Request(check_restore_status_url)),
]
http_mock = MagicMock(side_effect=http_responses)
urllib.request.urlopen = http_mock
makedirs_mock = MagicMock(return_value=0)
os.makedirs = makedirs_mock
os_remove_mock = MagicMock(return_value=0)
os.remove = os_remove_mock
os.listdir = MagicMock(return_value=[timestamp])
os_path_isfile_mock = MagicMock(return_value=False)
os.path.isfile = os_path_isfile_mock
is_dir_responses = [
False,
True,
True,
True
]
os.path.isdir = MagicMock(side_effect=is_dir_responses)
shutil_rmtree_mock = MagicMock(return_value=0)
shutil.rmtree = shutil_rmtree_mock
self.__backup_controller.restore_backup(bucket=S3_BUCKET, timestamp=timestamp)
# Verify that backup directory is created
makedirs_mock.assert_called_once_with(BACKUP_ROOT_DIR + timestamp)
# Verify that existence of tarball is checked before start of restoring
os_path_isfile_mock.assert_called_once_with(BACKUP_ROOT_DIR + test_backup_file_name)
# Verify that backup tarball is downloaded
s3_file_url = 's3://' + S3_BUCKET + '/' + timestamp + '/' + test_backup_file_name
self.__subprocess_mock.assert_any_call(['aws', 's3', 'cp', s3_file_url, BACKUP_ROOT_DIR])
# Verify that backup tarball is unzipped
self.__subprocess_mock.assert_any_call(
['tar', '-xzf', BACKUP_ROOT_DIR + test_backup_file_name, '-C', BACKUP_ROOT_DIR + timestamp]
)
# Verify that downloaded file is deleted after unzipping
os_remove_mock.assert_called_once_with(BACKUP_ROOT_DIR + test_backup_file_name)
# Verify HTTP requests
called_urls = list(map(lambda call_args: call_args[0][0].get_full_url(), http_mock.call_args_list))
expected_urls = [
local_cores_url,
trigger_restore_url,
check_restore_status_url,
check_restore_status_url
]
self.assertListEqual(called_urls, expected_urls)
# Verify that backup directory is cleaned up afterwards
shutil_rmtree_mock.assert_called_once_with(backup_dir)
def test_should_create_backup_of_local_shards(self):
self.__test_and_verify_backup_creation(TEST_COLLECTION, TEST_SHARD, self.__side_effect_local_cores)
def test_should_create_backup_of_local_split_shards(self):
self.__test_and_verify_backup_creation(TEST_COLLECTION, TEST_SPLIT_SHARD, self.__side_effect_local_split_cores)
def test_should_create_backup_of_single_core_setup(self):
self.__test_and_verify_backup_creation(TEST_COLLECTION, '', self.__side_effect_local_single_cores)
def __test_and_verify_backup_creation(self, collection: str, shard: str, cores_func):
timestamp = datetime.utcnow().strftime('%Y%m%d%H%M')
backup_dir = BACKUP_ROOT_DIR + timestamp
normalized_shard = self.__normalize_split_shard(shard)
if shard != '':
backup_name = collection + '_' + shard
normalized_backup_name = collection + '_' + normalized_shard
core_name = backup_name + '_' + TEST_REPLICA_1
else:
backup_name = collection
normalized_backup_name = collection
core_name = backup_name
backup_file_name = 'backup_' + timestamp + '_' + normalized_backup_name + '.tar.gz'
normalized_snapshot_dir = 'snapshot.' + normalized_backup_name
snapshot_dir = 'snapshot.' + backup_name
dir_lists = [
[], # backup dir before start of backup
[snapshot_dir], # snapshot dirs for zipping and uploading to S3
[timestamp] # backup dir for clean up
]
os.listdir = MagicMock(side_effect=dir_lists)
os.path.isdir = MagicMock(return_value=True)
shutil_rmtree_mock = MagicMock(return_value=0)
shutil.rmtree = shutil_rmtree_mock
os_rename_mock = MagicMock(return_value=0)
os.rename = os_rename_mock
local_cores_url = LOCAL_URL + '/admin/cores?action=STATUS&wt=json'
trigger_commit_url = LOCAL_URL + '/' + collection + '/update?commit=true&wt=json'
trigger_backup_url = LOCAL_URL + '/' + core_name + '/replication?command=backup&wt=json&location='\
+ backup_dir + '&name=' + backup_name
check_backup_status_url = LOCAL_URL + '/' + core_name + '/replication?command=details&wt=json'
http_responses = [
cores_func(urllib.request.Request(local_cores_url)),
self.__side_effect_all_ok(urllib.request.Request(trigger_commit_url)),
cores_func(urllib.request.Request(local_cores_url)),
self.__side_effect_all_ok(urllib.request.Request(trigger_backup_url)),
self.__side_effect_backup_in_progress(urllib.request.Request(check_backup_status_url)),
self.__side_effect_backup_done(urllib.request.Request(check_backup_status_url)),
]
http_mock = MagicMock(side_effect=http_responses)
urllib.request.urlopen = http_mock
self.__backup_controller.create_backup(bucket=S3_BUCKET)
# Verify HTTP requests
called_urls = list(map(lambda call_args: call_args[0][0].get_full_url(), http_mock.call_args_list))
expected_urls = [
local_cores_url,
trigger_commit_url,
local_cores_url,
trigger_backup_url,
check_backup_status_url,
check_backup_status_url
]
self.assertListEqual(called_urls, expected_urls)
# Verify that snapshot directory was renamed
if shard != normalized_shard:
split_snapshot_dir_name = backup_dir + '/snapshot.' + backup_name
normalized_snapshot_dir_name = backup_dir + '/snapshot.' + normalized_backup_name
os_rename_mock.assert_called_once_with(split_snapshot_dir_name, normalized_snapshot_dir_name)
# Verify that backup tarball is zipped
self.__subprocess_mock.assert_any_call(['tar', '-czf', backup_dir + '/' + backup_file_name, '-C',
backup_dir, normalized_snapshot_dir])
# Verify that backup tarball is uploaded
s3_url = 's3://' + S3_BUCKET + '/' + timestamp + '/'
self.__subprocess_mock.assert_any_call(['aws', 's3', 'cp', backup_dir + '/' + backup_file_name, s3_url])
# Verify that backup directory is cleaned up afterwards
shutil_rmtree_mock.assert_called_once_with(backup_dir)
def __normalize_split_shard(self, shard: str):
if len(shard.split('_')) > 1:
return TEST_SPLIT_SHARD_NORMALIZED
else:
return shard
def __side_effect_all_ok(self, value):
self.assertIsNotNone(value)
response_mock = MagicMock()
response_mock.getcode.return_value = HTTP_CODE_OK
return response_mock
def __side_effect_backup_in_progress(self, value):
self.assertIsNotNone(value)
response_mock = MagicMock()
response_mock.getcode.return_value = HTTP_CODE_OK
response_mock.read.return_value = \
bytes(json.dumps({"details": {"backup": ["", "", "", "", "", "In Progress"]}}), 'utf-8')
return response_mock
def __side_effect_backup_done(self, value):
self.assertIsNotNone(value)
response_mock = MagicMock()
response_mock.getcode.return_value = HTTP_CODE_OK
response_mock.read.return_value = \
bytes(json.dumps({"details": {"backup": ["", "", "", "", "", "success"]}}), 'utf-8')
return response_mock
def __side_effect_restore_in_progress(self, value):
self.assertIsNotNone(value)
response_mock = MagicMock()
response_mock.getcode.return_value = HTTP_CODE_OK
response_mock.read.return_value = \
bytes(json.dumps({"restorestatus": {"status": "In Progress"}}), 'utf-8')
return response_mock
def __side_effect_restore_done(self, value):
self.assertIsNotNone(value)
response_mock = MagicMock()
response_mock.getcode.return_value = HTTP_CODE_OK
response_mock.read.return_value = \
bytes(json.dumps({"restorestatus": {"status": "success"}}), 'utf-8')
return response_mock
def __side_effect_local_cores(self, value):
self.assertIsNotNone(value)
response_mock = MagicMock()
response_mock.getcode.return_value = HTTP_CODE_OK
response_mock.read.return_value = \
bytes(json.dumps({"status": {TEST_COLLECTION + '_' + TEST_SHARD + '_' + TEST_REPLICA_1: {}}}), 'utf-8')
return response_mock
def __side_effect_local_split_cores(self, value):
self.assertIsNotNone(value)
response_mock = MagicMock()
response_mock.getcode.return_value = HTTP_CODE_OK
response_mock.read.return_value = \
bytes(json.dumps({"status": {TEST_COLLECTION + '_' + TEST_SPLIT_SHARD + '_' + TEST_REPLICA_1: {}}}),
'utf-8')
return response_mock
def __side_effect_local_single_cores(self, value):
self.assertIsNotNone(value)
response_mock = MagicMock()
response_mock.getcode.return_value = HTTP_CODE_OK
response_mock.read.return_value = \
bytes(json.dumps({"status": {TEST_COLLECTION: {}}}),
'utf-8')
return response_mock
