def file_md5(self, local_file): #pylint: disable=no-self-use
'''
Get md5sum of local file
local_file : Full path of local file
'''
return utils.md5(local_file)
def test_object_get(mocker):
test_data = '01234'
with TemporaryDirectory() as tmp_dir:
with utils.temp_file(tmp_dir) as input_file:
with open(input_file, 'w') as writer:
writer.write(test_data)
with open(input_file, 'rb') as reader:
class MockRawRequest():
def __init__(self):
self.raw = reader
class MockOCI():
def __init__(self, *args, **kwargs):
pass
def get_object(self, *args, **kwargs):
return MockResponse(200, MockRawRequest())
mocker.patch('backup_tool.oci_client.from_file',
return_value='')
mocker.patch('backup_tool.oci_client.ObjectStorageClient',
return_value=MockOCI)
mocker.patch('backup_tool.oci_client.to_dict',
side_effect=to_dict_mock)
client = OCIObjectStorageClient(FAKE_CONFIG, FAKE_SECTION)
# Make sure to pass page limit of 1
with utils.temp_file(tmp_dir) as temp_file:
objects = client.object_get(FAKE_NAMESPACE, FAKE_BUCKET,
'some-object-name', temp_file)
md5 = utils.md5(temp_file)
assert md5 == 'QQDE1E2pF3JH5EpfwVRneA=='
def test_md5():
'''
Run md5 and make sure it matches expected
'''
# Hardcoded, this is what the base64 hash of foo is
value = '07BzhNET7exJ6qYjitX/AA=='
with TemporaryDirectory() as tmp_dir:
with utils.temp_file(tmp_dir) as temp:
with open(temp, 'w') as writer:
writer.write('foo\n')
md5_value = utils.md5(temp)
assert md5_value == value, 'MD5 value not equal to expected'
def __consume_backup_file(self, local_file_path, overwrite):
self.client.logger.debug(f'Backup up file {str(local_file_path)}')
local_file_md5 = utils.md5(local_file_path)
self.client.logger.debug(f'Local file "{str(local_file_path)}" has md5 {local_file_md5}')
should_upload_file, local_backup_file = self.client._file_backup_ensure_database_entry(local_file_path, #pylint:disable=protected-access
local_file_md5,
overwrite)
if not should_upload_file:
self.cache_json['backup']['processed'].append(str(local_file_path))
return None
encryption_data = self.client._file_backup_encrypt(local_file_path, local_file_md5) #pylint:disable=protected-access
encryption_data_key = encryption_data.get('local_file')
encryption_data['local_backup_file_id'] = local_backup_file.id
self.cache_json['backup']['pending_upload'][encryption_data_key] = encryption_data
return encryption_data
def file_backup(self, local_file, overwrite=False):
'''
Backup file to object storage
local_file : Full path of local file
overwrite : Upload new file is md5 is changed
automatically_upload_files : Upload new files automatically
'''
# Use local file as the full path of the file
# Use local file path as relative path for the database
local_file_path = Path(local_file).resolve()
self.logger.info(f'Backing up local file: "{str(local_file_path)}"')
local_file_md5 = utils.md5(local_file_path)
self.logger.debug(f'Local file "{str(local_file_path)}" has md5 {local_file_md5}')
should_upload_file, local_backup_file = self._file_backup_ensure_database_entry(local_file, local_file_md5, overwrite)
if not should_upload_file:
return False
encryption_data = self._file_backup_encrypt(local_file_path, local_file_md5)
self._file_backup_upload(encryption_data['encrypted_file'],
encryption_data['encrypted_file_md5'],
encryption_data['local_file_md5'],
local_backup_file)
Path(encryption_data['encrypted_file']).unlink()
return True
def file_restore(self, local_file_id, overwrite=False, set_restore=False): #pylint: disable=too-many-return-statements
'''
Restore file from object storage
local_file_id : ID of local file database entry to restore locally
overwrite : Overwrite local file if md5 does not match
set_restore : If object is archived, attempt to restore
'''
self.logger.info(f'Restoring local file: {local_file_id}')
local_file = self.db_session.query(BackupEntryLocalFile).get(local_file_id)
if not local_file:
self.logger.error(f'Unable to find local file: {local_file_id}')
return False
if not local_file.backup_entry_id:
self.logger.error(f'No backup entry for local file: {local_file_id}')
return False
backup_entry = self.db_session.query(BackupEntry).get(local_file.backup_entry_id)
if not backup_entry:
self.logger.error(f'Expecting backup entry {local_file.backup_entry_id} does not exist')
local_file_path = Path(local_file.local_file_path)
if self.relative_path:
local_file_path = self.relative_path / local_file_path
if local_file_path.is_file():
self.logger.debug(f'Checking local file "{str(local_file_path)}" md5')
local_file_md5 = utils.md5(str(local_file_path))
self.logger.debug(f'Local file "{str(local_file_path)}" has md5 sum {local_file_md5}')
if backup_entry.original_md5_checksum == local_file_md5:
if not overwrite:
self.logger.info(f'Local file "{str(local_file_path)}" has expected md5 {local_file_md5}')
return True
# Write file to temp dir
with utils.temp_file(self.work_directory) as encrypted_file:
self.logger.info(f'Downloading object {backup_entry.uploaded_file_path} to temp file "{str(encrypted_file)}"')
self.os_client.object_get(self.oci_namespace, self.oci_bucket,
backup_entry.uploaded_file_path, str(encrypted_file), set_restore=set_restore)
self.logger.info(f'Downloaded of object {backup_entry.uploaded_file_path} complete, written to temp file "{str(encrypted_file)}"')
# Ensure dir of new decrypted file is created
if not local_file_path.parent.exists():
local_file_path.mkdir(parents=True)
self.logger.debug(f'Decrypting temp file "{str(encrypted_file)}" to file "{str(local_file_path)}"')
encrypted_file_md5, local_file_md5 = crypto.decrypt_file(str(encrypted_file),
str(local_file_path),
self.crypto_key)
self.logger.debug(f'Decrypted file "{str(encrypted_file)}" with md5 "{encrypted_file_md5}" to '
f'file "{str(local_file_path)}" with md5 "{local_file_md5}"')
if backup_entry.uploaded_md5_checksum != encrypted_file_md5:
self.logger.error(f'Downloaded file "{str(encrypted_file)}" has unexpected md5 {encrypted_file_md5}, '
f'expected {backup_entry.uploaded_md5_checksum}')
return False
if local_file_md5 != backup_entry.original_md5_checksum:
self.logger.error(f'MD5 {local_file_md5} of decrypted file "{str(local_file_path)}" does not match expected {backup_entry.original_md5_checksum}')
return False
return True