def setUp(self):
super(BaseFreezerTest, self).setUp()
self.storage = Temp_Tree()
self.source_trees = []
self.backup_count = 0
self.backup_name = 'backup_test'
self.get_environ()
def setUp(self):
super(TestFreezerCompressGzip, self).setUp()
# create a source tree to backup with a few empty files
# (files must be empty to avoid encoding errors with pure random data)
self.source_tree = Temp_Tree()
self.source_tree.add_random_data(size=0)
self.storage_tree = Temp_Tree()
self.dest_tree = Temp_Tree()
self.environ = super(TestFreezerCompressGzip, self).get_environ()
def setUp(self):
super(TestFreezerMetadataChecksum, self).setUp()
# create a source tree to backup with a few empty files
# (files must be empty to avoid encoding errors with pure random data)
self.source_tree = Temp_Tree()
self.source_tree.add_random_data(size=0)
self.storage_tree = Temp_Tree()
self.dest_tree = Temp_Tree()
self.freezer_backup_name = 'freezer-test-backup-checksum-0'
self.metadata_out = os.path.join(
self.storage_tree.path,
self.freezer_backup_name + '.json')
self.environ = super(TestFreezerMetadataChecksum, self).get_environ()
def setUp(self):
super(TestFreezerCompressGzip, self).setUp()
# create a source tree to backup with a few empty files
# (files must be empty to avoid encoding errors with pure random data)
self.source_tree = Temp_Tree()
self.source_tree.add_random_data(size=0)
self.storage_tree = Temp_Tree()
self.dest_tree = Temp_Tree()
self.environ = super(TestFreezerCompressGzip, self).get_environ()
def setUp(self):
super(TestFreezerMetadataChecksum, self).setUp()
# create a source tree to backup with a few empty files
# (files must be empty to avoid encoding errors with pure random data)
self.source_tree = Temp_Tree()
self.source_tree.add_random_data(size=0)
self.storage_tree = Temp_Tree()
self.dest_tree = Temp_Tree()
self.freezer_backup_name = 'freezer-test-backup-checksum-0'
self.metadata_out = os.path.join(self.storage_tree.path,
self.freezer_backup_name + '.json')
self.environ = super(TestFreezerMetadataChecksum, self).get_environ()
class TestFreezerCompressGzip(base.BaseFreezerTest):
def __init__(self, *args, **kwargs):
super(TestFreezerCompressGzip, self).__init__(*args, **kwargs)
# noinspection PyAttributeOutsideInit
def setUp(self):
super(TestFreezerCompressGzip, self).setUp()
# create a source tree to backup with a few empty files
# (files must be empty to avoid encoding errors with pure random data)
self.source_tree = Temp_Tree()
self.source_tree.add_random_data(size=0)
self.storage_tree = Temp_Tree()
self.dest_tree = Temp_Tree()
self.environ = super(TestFreezerCompressGzip, self).get_environ()
def tearDown(self):
super(TestFreezerCompressGzip, self).tearDown()
self.source_tree.cleanup()
self.dest_tree.cleanup()
self.storage_tree.cleanup()
def _backup(self, name, method):
# perform a normal backup, with gzip specified
backup_args = [
'freezer-agent', '--path-to-backup', self.source_tree.path,
'--container', self.storage_tree.path, '--backup-name', name,
'--storage', 'local', '--compress', method, '--metadata-out',
os.path.join(self.storage_tree.path, 'metadata.json')
]
self.run_subprocess(backup_args, 'Test gzip backup to local storage.')
def _restore(self, name, method):
restore_args = [
'freezer-agent', '--action', 'restore', '--restore-abs-path',
self.dest_tree.path, '--container', self.storage_tree.path,
'--backup-name', name, '--storage', 'local', '--compress', method
]
self.run_subprocess(restore_args, 'Test restore from local storage.')
def _metadata(self):
path = os.path.join(self.storage_tree.path, 'metadata.json')
with open(path, 'r') as f:
return json.load(f)
def _file_get_mimetype(self, metadata):
"""Given some file metadata, find its mimetype using the file command
:param metadata: the parsed json file metadata
:return: the mimetype
"""
"""
Data is stored like data/tar/localhost_False/1469786264/0_1469786264 so
we need build the same directory structure.
data: the directory that holds the backup data
tar: the engine used to create backup
localhost: the hostname of the machine where the backup was taken
False: it should be backup name or False is backup is not provided
1469786264: timestamp
0_1469786264: level zero timestamp
"""
data_file_path = 'data{0}{1}{0}{2}_{3}{0}{4}{0}{5}_{4}{0}data'.format(
os.path.sep,
"tar", # currently we support only tar
metadata['hostname'],
metadata['backup_name'],
metadata['time_stamp'],
metadata['curr_backup_level'])
data_file_path = os.path.join(self.storage_tree.path, data_file_path)
self.assertEqual(True, os.path.exists(data_file_path))
# run 'file' in brief mode to only output the values we want
proc = subprocess.Popen(['file', '-b', '--mime-type', data_file_path],
stdout=subprocess.PIPE)
out, err = proc.communicate()
self.assertEqual(0, proc.returncode)
return out.strip()
@test.attr(type="gate")
def test_freezer_backup_compress_gzip(self):
backup_name = 'freezer-test-backup-gzip-0'
self._backup(backup_name, 'gzip')
self._restore(backup_name, 'gzip')
# metadata should show the correct algorithm
metadata = self._metadata()
self.assertIn('compression', metadata)
self.assertEqual('gzip', metadata['compression'])
# file utility should detect the correct mimetype
mimetype = self._file_get_mimetype(metadata)
self.assertEqual('application/gzip', mimetype)
# actual contents should be the same
diff_args = [
'diff', '-r', '-q', self.source_tree.path, self.dest_tree.path
]
self.run_subprocess(
diff_args, 'Verify restored copy is identical to '
'original.')
@test.attr(type="gate")
def test_freezer_backup_compress_bzip2(self):
backup_name = 'freezer-test-backup-bzip2-0'
self._backup(backup_name, 'bzip2')
self._restore(backup_name, 'bzip2')
metadata = self._metadata()
self.assertIn('compression', metadata)
self.assertEqual('bzip2', metadata['compression'])
mimetype = self._file_get_mimetype(metadata)
self.assertEqual('application/x-bzip2', mimetype)
diff_args = [
'diff', '-r', '-q', self.source_tree.path, self.dest_tree.path
]
self.run_subprocess(
diff_args, 'Verify restored copy is identical to '
'original.')
@test.attr(type="gate")
def test_freezer_backup_compress_xz(self):
backup_name = 'freezer-test-backup-xz-0'
self._backup(backup_name, 'xz')
self._restore(backup_name, 'xz')
metadata = self._metadata()
self.assertIn('compression', metadata)
self.assertEqual('xz', metadata['compression'])
mimetype = self._file_get_mimetype(metadata)
self.assertEqual('application/x-xz', mimetype)
diff_args = [
'diff', '-r', '-q', self.source_tree.path, self.dest_tree.path
]
self.run_subprocess(
diff_args, 'Verify restored copy is identical to '
'original.')
def create_local_backup(self,
hostname=None,
compression=None,
consistency_check=None,
incremental=True,
always_level=None,
restart_always_level=None,
max_level=None):
"""Creates a new backup with the given parameters.
The backup will immediately be created using a randomly-generated
source tree on the local filesystem, and will be stored in a random
temporary directory using the 'local' storage mode. All generated data
files will be automatically removed during `tearDown()`, though
implementations are responsible for cleaning up any additional copies
or restores created via other methods.
:param hostname: if set, set `--hostname` to the given value
:param compression: if set, set `--compression` to the given value
:param consistency_check: if True, set `--consistency_check`
:param incremental: if False, set `--no-incremental`
:param always_level: sets `--always-level` to the given value
:param restart_always_level: sets `--restart-always-level`
:param max_level: sets `--max-level` to the given value
:return: the path to the stored backup metadata
"""
metadata_path = os.path.join(
self.storage.path, 'metadata-{}.json'.format(self.backup_count))
self.backup_count += 1
tree = Temp_Tree()
tree.add_random_data()
self.source_trees.append(tree)
backup_args = [
'freezer-agent',
'--path-to-backup',
tree.path,
'--container',
self.storage.path,
'--backup-name',
self.backup_name,
'--storage',
'local',
'--metadata-out',
metadata_path,
]
if hostname:
backup_args += ['--hostname', hostname]
if compression:
backup_args += ['--compression', compression]
if consistency_check:
backup_args += ['--consistency-check']
if incremental:
if always_level is not None:
backup_args += ['--always-level', str(always_level)]
if max_level is not None:
backup_args += ['--max-level', str(max_level)]
if restart_always_level:
backup_args += [
'--restart-always-level',
str(restart_always_level)
]
else:
backup_args += ['--no-incremental', 'NO_INCREMENTAL']
self.run_subprocess(backup_args, 'Test backup to local storage.')
return metadata_path
class BaseFreezerTest(test.BaseTestCase):
credentials = ['primary']
def __init__(self, *args, **kwargs):
super(BaseFreezerTest, self).__init__(*args, **kwargs)
# noinspection PyAttributeOutsideInit
def setUp(self):
super(BaseFreezerTest, self).setUp()
self.storage = Temp_Tree()
self.source_trees = []
self.backup_count = 0
self.backup_name = 'backup_test'
self.get_environ()
def tearDown(self):
super(BaseFreezerTest, self).tearDown()
for tree in self.source_trees:
tree.cleanup()
self.storage.cleanup()
@classmethod
def get_auth_url(cls):
return cls.os_primary.auth_provider.auth_client.auth_url[:-len(
'/auth/tokens')]
@classmethod
def setup_clients(cls):
super(BaseFreezerTest, cls).setup_clients()
cls.get_environ()
@classmethod
def get_environ(cls):
os.environ['OS_PASSWORD'] = cls.os_primary.credentials.password
os.environ['OS_USERNAME'] = cls.os_primary.credentials.username
os.environ['OS_PROJECT_NAME'] = cls.os_primary.credentials.tenant_name
os.environ['OS_TENANT_NAME'] = cls.os_primary.credentials.tenant_name
os.environ['OS_PROJECT_DOMAIN_NAME'] = \
cls.os_primary.credentials.project_domain_name
os.environ['OS_USER_DOMAIN_NAME'] = \
cls.os_primary.credentials.user_domain_name
# Allow developers to set OS_AUTH_URL when developing so that
# Keystone may be on a host other than localhost.
if 'OS_AUTH_URL' not in os.environ:
os.environ['OS_AUTH_URL'] = cls.get_auth_url()
# Mac OS X uses gtar located in /usr/local/bin
os.environ['PATH'] = '/usr/local/bin:' + os.environ['PATH']
return os.environ
def run_subprocess(self, sub_process_args, fail_message):
proc = subprocess.Popen(sub_process_args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=self.environ,
shell=False)
out, err = proc.communicate()
self.assertEqual(
0, proc.returncode, fail_message + " Output: {0}. "
"Error: {1}".format(out, err))
self.assertEqual(
'', err, fail_message + " Output: {0}. "
"Error: {1}".format(out, err))
def create_local_backup(self,
hostname=None,
compression=None,
consistency_check=None,
incremental=True,
always_level=None,
restart_always_level=None,
max_level=None):
"""Creates a new backup with the given parameters.
The backup will immediately be created using a randomly-generated
source tree on the local filesystem, and will be stored in a random
temporary directory using the 'local' storage mode. All generated data
files will be automatically removed during `tearDown()`, though
implementations are responsible for cleaning up any additional copies
or restores created via other methods.
:param hostname: if set, set `--hostname` to the given value
:param compression: if set, set `--compression` to the given value
:param consistency_check: if True, set `--consistency_check`
:param incremental: if False, set `--no-incremental`
:param always_level: sets `--always-level` to the given value
:param restart_always_level: sets `--restart-always-level`
:param max_level: sets `--max-level` to the given value
:return: the path to the stored backup metadata
"""
metadata_path = os.path.join(
self.storage.path, 'metadata-{}.json'.format(self.backup_count))
self.backup_count += 1
tree = Temp_Tree()
tree.add_random_data()
self.source_trees.append(tree)
backup_args = [
'freezer-agent',
'--path-to-backup',
tree.path,
'--container',
self.storage.path,
'--backup-name',
self.backup_name,
'--storage',
'local',
'--metadata-out',
metadata_path,
]
if hostname:
backup_args += ['--hostname', hostname]
if compression:
backup_args += ['--compression', compression]
if consistency_check:
backup_args += ['--consistency-check']
if incremental:
if always_level is not None:
backup_args += ['--always-level', str(always_level)]
if max_level is not None:
backup_args += ['--max-level', str(max_level)]
if restart_always_level:
backup_args += [
'--restart-always-level',
str(restart_always_level)
]
else:
backup_args += ['--no-incremental', 'NO_INCREMENTAL']
self.run_subprocess(backup_args, 'Test backup to local storage.')
return metadata_path
def create_mutated_backup(self, days_old=30, **kwargs):
"""Create a local backup with a mutated timestamp
This creates a new backup using `create_local_backup()`, modifies it
using `mutate_timestamp()`, and then returns the resulting (loaded)
metadata dict.
:param days_old: the age of the backup to create
:param kwargs: arguments to pass to `create_local_backup()`
:return: the loaded metadata
"""
metadata_path = self.create_local_backup(**kwargs)
metadata = load_metadata(metadata_path)
mutate_timestamp(metadata, days_old)
save_metadata(metadata, metadata_path)
return metadata
class TestFreezerMetadataChecksum(base.BaseFreezerTest):
def __init__(self, *args, **kwargs):
super(TestFreezerMetadataChecksum, self).__init__(*args, **kwargs)
# noinspection PyAttributeOutsideInit
def setUp(self):
super(TestFreezerMetadataChecksum, self).setUp()
# create a source tree to backup with a few empty files
# (files must be empty to avoid encoding errors with pure random data)
self.source_tree = Temp_Tree()
self.source_tree.add_random_data(size=0)
self.storage_tree = Temp_Tree()
self.dest_tree = Temp_Tree()
self.freezer_backup_name = 'freezer-test-backup-checksum-0'
self.metadata_out = os.path.join(self.storage_tree.path,
self.freezer_backup_name + '.json')
self.environ = super(TestFreezerMetadataChecksum, self).get_environ()
def tearDown(self):
super(TestFreezerMetadataChecksum, self).tearDown()
self.source_tree.cleanup()
self.dest_tree.cleanup()
self.storage_tree.cleanup()
@test.attr(type="gate")
def test_freezer_fs_backup_valid_checksum(self):
# perform a normal backup, but enable consistency checks and save the
# metadata to disk
backup_args = [
'freezer-agent', '--path-to-backup', self.source_tree.path,
'--container', self.storage_tree.path, '--backup-name',
self.freezer_backup_name, '--storage', 'local',
'--consistency_check', '--metadata-out', self.metadata_out
]
self.run_subprocess(backup_args, 'Test backup to local storage.')
# load the stored metadata to retrieve the computed checksum
with open(self.metadata_out) as f:
metadata = json.load(f)
self.assertIn('consistency_checksum', metadata,
'Checksum must exist in stored metadata.')
checksum = metadata['consistency_checksum']
restore_args = [
'freezer-agent', '--action', 'restore', '--restore-abs-path',
self.dest_tree.path, '--container', self.storage_tree.path,
'--backup-name', self.freezer_backup_name, '--storage',
'local', '--consistency_checksum', checksum
]
self.run_subprocess(
restore_args, 'Test restore from local storage with '
'computed checksum.')
@test.attr(type="gate")
def test_freezer_fs_backup_bad_checksum(self):
# as above, but we'll ignore the computed checksum
backup_args = [
'freezer-agent', '--path-to-backup', self.source_tree.path,
'--container', self.storage_tree.path, '--backup-name',
self.freezer_backup_name, '--storage', 'local',
'--consistency_check', '--metadata-out', self.metadata_out
]
self.run_subprocess(backup_args, 'Test backup to local storage.')
# make a failing sha256 checksum (assuming no added path string)
bad_checksum = '0' * 64
# attempt to restore using the bad checksum
restore_args = [
'freezer-agent', '--action', 'restore', '--restore-abs-path',
self.dest_tree.path, '--container', self.storage_tree.path,
'--backup-name', self.freezer_backup_name, '--storage', 'local',
'--consistency_checksum', bad_checksum
]
process = subprocess.Popen(restore_args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=self.environ,
shell=False)
out, err = process.communicate()
# make sure the subprocess exist with an error due to checksum mismatch
message = '{0} Output: {1} Error: {2}'.format(
'Restore process should fail with checksum error.', out, err)
self.assertEqual(1, process.returncode, message)
self.assertEqual('', out, message)
self.assertNotEqual('', err, message)
def setUp(self):
super(TestFreezerMetadataChecksum, self).setUp()
self.environ = super(TestFreezerMetadataChecksum, self).get_environ()
self.dest_tree = Temp_Tree()
self.backup_name = 'backup_checksum_test'
class TestFreezerMetadataChecksum(base.BaseFreezerTest):
def __init__(self, *args, **kwargs):
super(TestFreezerMetadataChecksum, self).__init__(*args, **kwargs)
# noinspection PyAttributeOutsideInit
def setUp(self):
super(TestFreezerMetadataChecksum, self).setUp()
self.environ = super(TestFreezerMetadataChecksum, self).get_environ()
self.dest_tree = Temp_Tree()
self.backup_name = 'backup_checksum_test'
def tearDown(self):
super(TestFreezerMetadataChecksum, self).tearDown()
self.dest_tree.cleanup()
@test.attr(type="gate")
def test_freezer_fs_backup_valid_checksum(self):
# perform a normal backup, but enable consistency checks and save the
# metadata to disk
metadata_path = self.create_local_backup(consistency_check=True)
metadata = base.load_metadata(metadata_path)
# load the stored metadata to retrieve the computed checksum
self.assertIn('consistency_checksum', metadata,
'Checksum must exist in stored metadata.')
checksum = metadata['consistency_checksum']
restore_args = ['freezer-agent',
'--action', 'restore',
'--restore-abs-path', self.dest_tree.path,
'--container', metadata['container'],
'--backup-name', self.backup_name,
'--storage', 'local',
'--consistency-checksum', checksum]
self.run_subprocess(restore_args,
'Test restore from local storage with '
'computed checksum.')
@test.attr(type="gate")
def test_freezer_fs_backup_bad_checksum(self):
# as above, but we'll ignore the computed checksum
metadata_path = self.create_local_backup(consistency_check=True)
metadata = base.load_metadata(metadata_path)
# make a failing sha256 checksum (assuming no added path string)
bad_checksum = '0' * 64
# attempt to restore using the bad checksum
restore_args = ['freezer-agent',
'--action', 'restore',
'--restore-abs-path', self.dest_tree.path,
'--container', metadata['container'],
'--backup-name', self.backup_name,
'--storage', 'local',
'--consistency-checksum', bad_checksum]
process = subprocess.Popen(restore_args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=self.environ, shell=False)
out, err = process.communicate()
# make sure the subprocess exist with an error due to checksum mismatch
message = '{0} Output: {1} Error: {2}'.format(
'Restore process should fail with checksum error.',
out, err)
self.assertEqual(1, process.returncode, message)
self.assertEqual('', out, message)
self.assertNotEqual('', err, message)
class TestFreezerCompressGzip(base.BaseFreezerTest):
def __init__(self, *args, **kwargs):
super(TestFreezerCompressGzip, self).__init__(*args, **kwargs)
# noinspection PyAttributeOutsideInit
def setUp(self):
super(TestFreezerCompressGzip, self).setUp()
# create a source tree to backup with a few empty files
# (files must be empty to avoid encoding errors with pure random data)
self.source_tree = Temp_Tree()
self.source_tree.add_random_data(size=0)
self.storage_tree = Temp_Tree()
self.dest_tree = Temp_Tree()
self.environ = super(TestFreezerCompressGzip, self).get_environ()
def tearDown(self):
super(TestFreezerCompressGzip, self).tearDown()
self.source_tree.cleanup()
self.dest_tree.cleanup()
self.storage_tree.cleanup()
def _backup(self, name, method):
# perform a normal backup, with gzip specified
backup_args = ['freezer-agent',
'--path-to-backup',
self.source_tree.path,
'--container',
self.storage_tree.path,
'--backup-name',
name,
'--storage',
'local',
'--compress',
method,
'--metadata-out',
os.path.join(self.storage_tree.path, 'metadata.json')]
self.run_subprocess(backup_args, 'Test gzip backup to local storage.')
def _restore(self, name, method):
restore_args = ['freezer-agent',
'--action',
'restore',
'--restore-abs-path',
self.dest_tree.path,
'--container',
self.storage_tree.path,
'--backup-name',
name,
'--storage',
'local',
'--compress',
method]
self.run_subprocess(restore_args, 'Test restore from local storage.')
def _metadata(self):
path = os.path.join(self.storage_tree.path, 'metadata.json')
with open(path, 'r') as f:
return json.load(f)
def _file_get_mimetype(self, metadata):
"""Given some file metadata, find its mimetype using the file command
:param metadata: the parsed json file metadata
:return: the mimetype
"""
hostname = metadata['hostname']
backup_name = metadata['backup_name']
time_stamp = metadata['time_stamp']
dir_name = '%s_%s' % (hostname, backup_name)
file_name = '%s_%s_%s_0' % (hostname, backup_name, time_stamp)
data_file_path = os.path.join(self.storage_tree.path,
dir_name,
str(time_stamp),
file_name)
self.assertEqual(True, os.path.exists(data_file_path))
# run 'file' in brief mode to only output the values we want
proc = subprocess.Popen(['file', '-b', '--mime-type', data_file_path],
stdout=subprocess.PIPE)
out, err = proc.communicate()
self.assertEqual(0, proc.returncode)
return out.strip()
@test.attr(type="gate")
def test_freezer_backup_compress_gzip(self):
backup_name = 'freezer-test-backup-gzip-0'
self._backup(backup_name, 'gzip')
self._restore(backup_name, 'gzip')
# metadata should show the correct algorithm
metadata = self._metadata()
self.assertIn('compression', metadata)
self.assertEqual('gzip', metadata['compression'])
# file utility should detect the correct mimetype
mimetype = self._file_get_mimetype(metadata)
self.assertEqual('application/gzip', mimetype)
# actual contents should be the same
diff_args = ['diff', '-r', '-q',
self.source_tree.path,
self.dest_tree.path]
self.run_subprocess(diff_args, 'Verify restored copy is identical to '
'original.')
@test.attr(type="gate")
def test_freezer_backup_compress_bzip2(self):
backup_name = 'freezer-test-backup-bzip2-0'
self._backup(backup_name, 'bzip2')
self._restore(backup_name, 'bzip2')
metadata = self._metadata()
self.assertIn('compression', metadata)
self.assertEqual('bzip2', metadata['compression'])
mimetype = self._file_get_mimetype(metadata)
self.assertEqual('application/x-bzip2', mimetype)
diff_args = ['diff', '-r', '-q',
self.source_tree.path,
self.dest_tree.path]
self.run_subprocess(diff_args, 'Verify restored copy is identical to '
'original.')
@test.attr(type="gate")
def test_freezer_backup_compress_xz(self):
backup_name = 'freezer-test-backup-xz-0'
self._backup(backup_name, 'xz')
self._restore(backup_name, 'xz')
metadata = self._metadata()
self.assertIn('compression', metadata)
self.assertEqual('xz', metadata['compression'])
mimetype = self._file_get_mimetype(metadata)
self.assertEqual('application/x-xz', mimetype)
diff_args = ['diff', '-r', '-q',
self.source_tree.path,
self.dest_tree.path]
self.run_subprocess(diff_args, 'Verify restored copy is identical to '
'original.')
class TestFreezerCompressGzip(base.BaseFreezerTest):
def __init__(self, *args, **kwargs):
super(TestFreezerCompressGzip, self).__init__(*args, **kwargs)
# noinspection PyAttributeOutsideInit
def setUp(self):
super(TestFreezerCompressGzip, self).setUp()
# create a source tree to backup with a few empty files
# (files must be empty to avoid encoding errors with pure random data)
self.source_tree = Temp_Tree()
self.source_tree.add_random_data(size=0)
self.storage_tree = Temp_Tree()
self.dest_tree = Temp_Tree()
self.environ = super(TestFreezerCompressGzip, self).get_environ()
def tearDown(self):
super(TestFreezerCompressGzip, self).tearDown()
self.source_tree.cleanup()
self.dest_tree.cleanup()
self.storage_tree.cleanup()
def _backup(self, name, method):
# perform a normal backup, with gzip specified
backup_args = [
'freezer-agent', '--path-to-backup', self.source_tree.path,
'--container', self.storage_tree.path, '--backup-name', name,
'--storage', 'local', '--compress', method, '--metadata-out',
os.path.join(self.storage_tree.path, 'metadata.json')
]
self.run_subprocess(backup_args, 'Test gzip backup to local storage.')
def _restore(self, name, method):
restore_args = [
'freezer-agent', '--action', 'restore', '--restore-abs-path',
self.dest_tree.path, '--container', self.storage_tree.path,
'--backup-name', name, '--storage', 'local', '--compress', method
]
self.run_subprocess(restore_args, 'Test restore from local storage.')
def _metadata(self):
path = os.path.join(self.storage_tree.path, 'metadata.json')
with open(path, 'r') as f:
return json.load(f)
def _file_get_mimetype(self, metadata):
"""Given some file metadata, find its mimetype using the file command
:param metadata: the parsed json file metadata
:return: the mimetype
"""
hostname = metadata['hostname']
backup_name = metadata['backup_name']
time_stamp = metadata['time_stamp']
dir_name = '%s_%s' % (hostname, backup_name)
file_name = '%s_%s_%s_0' % (hostname, backup_name, time_stamp)
data_file_path = os.path.join(self.storage_tree.path, dir_name,
str(time_stamp), file_name)
self.assertEqual(True, os.path.exists(data_file_path))
# run 'file' in brief mode to only output the values we want
proc = subprocess.Popen(['file', '-b', '--mime-type', data_file_path],
stdout=subprocess.PIPE)
out, err = proc.communicate()
self.assertEqual(0, proc.returncode)
return out.strip()
@test.attr(type="gate")
def test_freezer_backup_compress_gzip(self):
backup_name = 'freezer-test-backup-gzip-0'
self._backup(backup_name, 'gzip')
self._restore(backup_name, 'gzip')
# metadata should show the correct algorithm
metadata = self._metadata()
self.assertIn('compression', metadata)
self.assertEqual('gzip', metadata['compression'])
# file utility should detect the correct mimetype
mimetype = self._file_get_mimetype(metadata)
self.assertEqual('application/gzip', mimetype)
# actual contents should be the same
diff_args = [
'diff', '-r', '-q', self.source_tree.path, self.dest_tree.path
]
self.run_subprocess(
diff_args, 'Verify restored copy is identical to '
'original.')
@test.attr(type="gate")
def test_freezer_backup_compress_bzip2(self):
backup_name = 'freezer-test-backup-bzip2-0'
self._backup(backup_name, 'bzip2')
self._restore(backup_name, 'bzip2')
metadata = self._metadata()
self.assertIn('compression', metadata)
self.assertEqual('bzip2', metadata['compression'])
mimetype = self._file_get_mimetype(metadata)
self.assertEqual('application/x-bzip2', mimetype)
diff_args = [
'diff', '-r', '-q', self.source_tree.path, self.dest_tree.path
]
self.run_subprocess(
diff_args, 'Verify restored copy is identical to '
'original.')
@test.attr(type="gate")
def test_freezer_backup_compress_xz(self):
backup_name = 'freezer-test-backup-xz-0'
self._backup(backup_name, 'xz')
self._restore(backup_name, 'xz')
metadata = self._metadata()
self.assertIn('compression', metadata)
self.assertEqual('xz', metadata['compression'])
mimetype = self._file_get_mimetype(metadata)
self.assertEqual('application/x-xz', mimetype)
diff_args = [
'diff', '-r', '-q', self.source_tree.path, self.dest_tree.path
]
self.run_subprocess(
diff_args, 'Verify restored copy is identical to '
'original.')
class TestFreezerMetadataChecksum(base.BaseFreezerTest):
def __init__(self, *args, **kwargs):
super(TestFreezerMetadataChecksum, self).__init__(*args, **kwargs)
# noinspection PyAttributeOutsideInit
def setUp(self):
super(TestFreezerMetadataChecksum, self).setUp()
# create a source tree to backup with a few empty files
# (files must be empty to avoid encoding errors with pure random data)
self.source_tree = Temp_Tree()
self.source_tree.add_random_data(size=0)
self.storage_tree = Temp_Tree()
self.dest_tree = Temp_Tree()
self.freezer_backup_name = 'freezer-test-backup-checksum-0'
self.metadata_out = os.path.join(
self.storage_tree.path,
self.freezer_backup_name + '.json')
self.environ = super(TestFreezerMetadataChecksum, self).get_environ()
def tearDown(self):
super(TestFreezerMetadataChecksum, self).tearDown()
self.source_tree.cleanup()
self.dest_tree.cleanup()
self.storage_tree.cleanup()
@test.attr(type="gate")
def test_freezer_fs_backup_valid_checksum(self):
# perform a normal backup, but enable consistency checks and save the
# metadata to disk
backup_args = ['freezer-agent',
'--path-to-backup',
self.source_tree.path,
'--container',
self.storage_tree.path,
'--backup-name',
self.freezer_backup_name,
'--storage',
'local',
'--consistency_check',
'--metadata-out',
self.metadata_out]
self.run_subprocess(backup_args, 'Test backup to local storage.')
# load the stored metadata to retrieve the computed checksum
with open(self.metadata_out) as f:
metadata = json.load(f)
self.assertIn('consistency_checksum', metadata,
'Checksum must exist in stored metadata.')
checksum = metadata['consistency_checksum']
restore_args = ['freezer-agent',
'--action',
'restore',
'--restore-abs-path',
self.dest_tree.path,
'--container',
self.storage_tree.path,
'--backup-name',
self.freezer_backup_name,
'--storage',
'local',
'--consistency_checksum',
checksum]
self.run_subprocess(restore_args,
'Test restore from local storage with '
'computed checksum.')
@test.attr(type="gate")
def test_freezer_fs_backup_bad_checksum(self):
# as above, but we'll ignore the computed checksum
backup_args = ['freezer-agent',
'--path-to-backup',
self.source_tree.path,
'--container',
self.storage_tree.path,
'--backup-name',
self.freezer_backup_name,
'--storage',
'local',
'--consistency_check',
'--metadata-out',
self.metadata_out]
self.run_subprocess(backup_args, 'Test backup to local storage.')
# make a failing sha256 checksum (assuming no added path string)
bad_checksum = '0' * 64
# attempt to restore using the bad checksum
restore_args = ['freezer-agent',
'--action',
'restore',
'--restore-abs-path',
self.dest_tree.path,
'--container',
self.storage_tree.path,
'--backup-name',
self.freezer_backup_name,
'--storage',
'local',
'--consistency_checksum',
bad_checksum]
process = subprocess.Popen(restore_args,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=self.environ, shell=False)
out, err = process.communicate()
# make sure the subprocess exist with an error due to checksum mismatch
message = '{0} Output: {1} Error: {2}'.format(
'Restore process should fail with checksum error.',
out, err)
self.assertEqual(1, process.returncode, message)
self.assertEqual('', out, message)
self.assertNotEqual('', err, message)