def test_list_files(self):
"""
Unit test for RsyncCopyController._list_file's code
"""
# Mock rsync invocation
rsync_mock = mock.Mock(name='Rsync()')
rsync_mock.ret = 0
rsync_mock.out = 'drwxrwxrwt 69632 2015/02/09 15:01:00 tmp\n' \
'drwxrwxrwt 69612 Thu Feb 19 15:01:22 2015 tmp2'
rsync_mock.err = 'err'
# Test the _list_files internal method
rcc = RsyncCopyController()
return_values = list(rcc._list_files(rsync_mock, 'some/path'))
# Returned list must contain two elements
assert len(return_values) == 2
# Check rsync.get_output has called correctly
rsync_mock.get_output.assert_called_with('--no-human-readable',
'--list-only',
'-r',
'some/path',
check=True)
# Check the result
assert return_values[0] == _FileItem(
'drwxrwxrwt', 69632,
datetime(year=2015,
month=2,
day=9,
hour=15,
minute=1,
second=0,
tzinfo=dateutil.tz.tzlocal()), 'tmp')
assert return_values[1] == _FileItem(
'drwxrwxrwt', 69612,
datetime(year=2015,
month=2,
day=19,
hour=15,
minute=1,
second=22,
tzinfo=dateutil.tz.tzlocal()), 'tmp2')
# Test the _list_files internal method with a wrong output (added TZ)
rsync_mock.out = (
'drwxrwxrwt 69612 Thu Feb 19 15:01:22 CET 2015 tmp2\n')
rcc = RsyncCopyController()
with pytest.raises(RsyncListFilesFailure):
# The list() call is needed to consume the generator
list(rcc._list_files(rsync_mock, 'some/path'))
# Check rsync.get_output has called correctly
rsync_mock.get_output.assert_called_with('--no-human-readable',
'--list-only',
'-r',
'some/path',
check=True)
def test_fill_buckets(self):
"""
Unit test for RsyncCopyController._fill_buckets's code
"""
# Create a fake file list af about 525 GB of files
filedate = datetime(
year=2015,
month=2,
day=19,
hour=15,
minute=1,
second=22,
tzinfo=dateutil.tz.tzlocal(),
)
file_list = []
total_size = 0
for i in range(1001):
# We are using a prime number to get a non-correlatable distribution
# of file sizes in the buckets
size = 1048583 * i
file_list.append(
_FileItem("drwxrwxrwt", size, filedate, "tmp%08d" % i))
total_size += size
# Test the _fill_buckets internal method with only one worker:
# the result must be a bucket with the same list passed as argument
rcc = RsyncCopyController(workers=1)
buckets = list(rcc._fill_buckets(file_list))
assert len(buckets) == 1
assert buckets[0] == file_list
# Test the _fill_buckets internal method with multiple workers
# the result must be a bucket with the same list passed as argument
for workers in range(2, 17):
rcc = RsyncCopyController(workers=workers)
buckets = list(rcc._fill_buckets(file_list))
# There is enough buckets to contains all the files
assert len(buckets) >= int(total_size / BUCKET_SIZE)
for i, bucket in enumerate(buckets):
size = sum([f.size for f in bucket])
# The bucket is not bigger than BUCKET_SIZE
assert size < BUCKET_SIZE, "Bucket %s (%s) size %s too big" % (
i,
workers,
size,
)
# The bucket cannot be empty
assert len(bucket), "Bucket %s (%s) is empty" % (i, workers)
def analyse_func(item):
l = item.label
item.dir_file = l + '_dir_file'
item.exclude_and_protect_file = l + '_exclude_and_protect_file'
item.safe_list = [_FileItem('mode', 1, 'date', 'path')]
item.check_list = [_FileItem('mode', 1, 'date', 'path')]
def test_analyze_directory(self, list_files_mock, rsync_factory_mock,
tmpdir):
"""
Unit test for RsyncCopyController._analyze_directory's code
"""
# Build file list for ref
ref_list = [
_FileItem(
'drwxrwxrwt', 69632,
datetime(year=2015,
month=2,
day=9,
hour=15,
minute=1,
second=0,
tzinfo=dateutil.tz.tzlocal()), '.'),
_FileItem(
'drwxrwxrwt', 69612,
datetime(year=2015,
month=2,
day=19,
hour=15,
minute=1,
second=22,
tzinfo=dateutil.tz.tzlocal()), 'tmp'),
_FileItem(
'-rw-r--r--', 69632,
datetime(year=2015,
month=2,
day=20,
hour=18,
minute=15,
second=33,
tzinfo=dateutil.tz.tzlocal()), 'tmp/safe'),
_FileItem(
'-rw-r--r--', 69612,
datetime(year=2015,
month=2,
day=20,
hour=19,
minute=15,
second=33,
tzinfo=dateutil.tz.tzlocal()), 'tmp/check'),
_FileItem(
'-rw-r--r--', 69612,
datetime(year=2015,
month=2,
day=20,
hour=19,
minute=15,
second=33,
tzinfo=dateutil.tz.tzlocal()), 'tmp/diff_time'),
_FileItem(
'-rw-r--r--', 69612,
datetime(year=2015,
month=2,
day=20,
hour=19,
minute=15,
second=33,
tzinfo=dateutil.tz.tzlocal()), 'tmp/diff_size'),
]
# Build the list for source adding a new file, ...
src_list = ref_list + [
_FileItem(
'-rw-r--r--', 69612,
datetime(year=2015,
month=2,
day=20,
hour=22,
minute=15,
second=33,
tzinfo=dateutil.tz.tzlocal()), 'tmp/new'),
]
# ... changing the timestamp one old file ...
src_list[4] = _FileItem(
'-rw-r--r--', 69612,
datetime(year=2015,
month=2,
day=20,
hour=20,
minute=15,
second=33,
tzinfo=dateutil.tz.tzlocal()), 'tmp/diff_time')
# ... and changing the size of another
src_list[5] = _FileItem(
'-rw-r--r--', 77777,
datetime(year=2015,
month=2,
day=20,
hour=19,
minute=15,
second=33,
tzinfo=dateutil.tz.tzlocal()), 'tmp/diff_size')
# Apply it to _list_files calls
list_files_mock.side_effect = [ref_list, src_list]
# Build the prerequisites
server = build_real_server(
global_conf={'barman_home': tmpdir.mkdir('home').strpath})
config = server.config
executor = server.backup_manager.executor
# Create the RsyncCopyController putting the safe_horizon between
# the tmp/safe and tmp2/check timestamps
rcc = RsyncCopyController(
path=server.path,
ssh_command=executor.ssh_command,
ssh_options=executor.ssh_options,
network_compression=config.network_compression,
reuse_backup=None,
safe_horizon=datetime(year=2015,
month=2,
day=20,
hour=19,
minute=0,
second=0,
tzinfo=dateutil.tz.tzlocal()))
backup_info = build_test_backup_info(
server=server,
pgdata="/pg/data",
config_file="/etc/postgresql.conf",
hba_file="/pg/data/pg_hba.conf",
ident_file="/pg/data/pg_ident.conf",
begin_xlog="0/2000028",
begin_wal="000000010000000000000002",
begin_offset=28)
backup_info.save()
# This is to check that all the preparation is done correctly
assert os.path.exists(backup_info.filename)
# Add a temp dir (usually created by copy method
rcc.temp_dir = tmpdir.mkdir('tmp').strpath
# Create an item to inspect
item = _RsyncCopyItem(label='pgdata',
src=':/pg/data/',
dst=backup_info.get_data_directory(),
is_directory=True,
item_class=rcc.PGDATA_CLASS,
optional=False)
# Then run the _analyze_directory method
rcc._analyze_directory(item)
# Verify that _rsync_factory has been called correctly
assert rsync_factory_mock.mock_calls == [
mock.call(item),
]
# Verify that _list_files has been called correctly
assert list_files_mock.mock_calls == [
mock.call(rsync_factory_mock.return_value,
backup_info.get_data_directory() + '/'),
mock.call(rsync_factory_mock.return_value, ':/pg/data/')
]
# Check the result
# 1) The list of directories should be there and should contain all
# the directories
assert item.dir_file
assert open(item.dir_file).read() == ('.\n' 'tmp\n')
# The exclude_and_protect file should be populated correctly with all
# the files in the source
assert item.exclude_and_protect_file
assert open(
item.exclude_and_protect_file).read() == ('P tmp/safe\n'
'- tmp/safe\n'
'P tmp/check\n'
'- tmp/check\n'
'P tmp/diff_time\n'
'- tmp/diff_time\n'
'P tmp/diff_size\n'
'- tmp/diff_size\n'
'P tmp/new\n'
'- tmp/new\n')
# The check list must contain identical files after the safe_horizon
assert len(item.check_list) == 1
assert item.check_list[0].path == 'tmp/check'
# The safe list must contain every file that is not in check and is
# present in the source
assert len(item.safe_list) == 4
assert item.safe_list[0].path == 'tmp/safe'
assert item.safe_list[1].path == 'tmp/diff_time'
assert item.safe_list[2].path == 'tmp/diff_size'
assert item.safe_list[3].path == 'tmp/new'
def test_analyze_directory_empty_dst(self, list_files_mock, tmpdir):
"""
Verify that RsyncCopyController._analyze_directory produces an empty
exclude_and_protect_file when the destination directory is empty.
"""
# Only the current directory is in file list
ref_list = [
_FileItem(
"drwxrwxrwt",
69632,
datetime(
year=2015,
month=2,
day=9,
hour=15,
minute=1,
second=0,
tzinfo=dateutil.tz.tzlocal(),
),
".",
),
]
# Minimal src_list so that there is something to copy
src_list = ref_list + [
_FileItem(
"drwxrwxrwt",
69612,
datetime(
year=2015,
month=2,
day=19,
hour=15,
minute=1,
second=22,
tzinfo=dateutil.tz.tzlocal(),
),
"tmp",
),
]
# Set up prerequisites and run the analyze directory function
item, backup_info = self._run_analyze_directory(
list_files_mock, tmpdir, ref_list, src_list)
# Verify that _list_files has been called correctly
assert list_files_mock.mock_calls == [
mock.call(item,
backup_info.get_data_directory() + "/"),
mock.call(item, ":/pg/data/"),
]
# Check the result
# 1) The list of directories should be there and should contain all
# the directories
assert item.dir_file
assert open(item.dir_file).read() == (".\ntmp\n")
# The exclude_and_protect file should contain only wildcards to include
# all directories and exclude all files
assert item.exclude_and_protect_file
assert open(item.exclude_and_protect_file).read() == "+ */\n- *\n"
def test_analyze_directory(self, list_files_mock, tmpdir):
"""
Unit test for RsyncCopyController._analyze_directory's code
"""
# Build file list for ref
ref_list = [
_FileItem(
"drwxrwxrwt",
69632,
datetime(
year=2015,
month=2,
day=9,
hour=15,
minute=1,
second=0,
tzinfo=dateutil.tz.tzlocal(),
),
".",
),
_FileItem(
"drwxrwxrwt",
69612,
datetime(
year=2015,
month=2,
day=19,
hour=15,
minute=1,
second=22,
tzinfo=dateutil.tz.tzlocal(),
),
"tmp",
),
_FileItem(
"-rw-r--r--",
69632,
datetime(
year=2015,
month=2,
day=20,
hour=18,
minute=15,
second=33,
tzinfo=dateutil.tz.tzlocal(),
),
"tmp/safe",
),
_FileItem(
"-rw-r--r--",
69612,
datetime(
year=2015,
month=2,
day=20,
hour=19,
minute=15,
second=33,
tzinfo=dateutil.tz.tzlocal(),
),
"tmp/check",
),
_FileItem(
"-rw-r--r--",
69612,
datetime(
year=2015,
month=2,
day=20,
hour=19,
minute=15,
second=33,
tzinfo=dateutil.tz.tzlocal(),
),
"tmp/diff_time",
),
_FileItem(
"-rw-r--r--",
69612,
datetime(
year=2015,
month=2,
day=20,
hour=19,
minute=15,
second=33,
tzinfo=dateutil.tz.tzlocal(),
),
"tmp/diff_size",
),
]
# Build the list for source adding a new file, ...
src_list = ref_list + [
_FileItem(
"-rw-r--r--",
69612,
datetime(
year=2015,
month=2,
day=20,
hour=22,
minute=15,
second=33,
tzinfo=dateutil.tz.tzlocal(),
),
"tmp/new",
),
]
# ... changing the timestamp one old file ...
src_list[4] = _FileItem(
"-rw-r--r--",
69612,
datetime(
year=2015,
month=2,
day=20,
hour=20,
minute=15,
second=33,
tzinfo=dateutil.tz.tzlocal(),
),
"tmp/diff_time",
)
# ... and changing the size of another
src_list[5] = _FileItem(
"-rw-r--r--",
77777,
datetime(
year=2015,
month=2,
day=20,
hour=19,
minute=15,
second=33,
tzinfo=dateutil.tz.tzlocal(),
),
"tmp/diff_size",
)
item, backup_info = self._run_analyze_directory(
list_files_mock, tmpdir, ref_list, src_list)
# Verify that _list_files has been called correctly
assert list_files_mock.mock_calls == [
mock.call(item,
backup_info.get_data_directory() + "/"),
mock.call(item, ":/pg/data/"),
]
# Check the result
# 1) The list of directories should be there and should contain all
# the directories
assert item.dir_file
assert open(item.dir_file).read() == (".\ntmp\n")
# The exclude_and_protect file should be populated correctly with all
# the files in the source
assert item.exclude_and_protect_file
assert open(
item.exclude_and_protect_file).read() == ("P /tmp/safe\n"
"- /tmp/safe\n"
"P /tmp/check\n"
"- /tmp/check\n"
"P /tmp/diff_time\n"
"- /tmp/diff_time\n"
"P /tmp/diff_size\n"
"- /tmp/diff_size\n"
"P /tmp/new\n"
"- /tmp/new\n")
# The check list must contain identical files after the safe_horizon
assert len(item.check_list) == 1
assert item.check_list[0].path == "tmp/check"
# The safe list must contain every file that is not in check and is
# present in the source
assert len(item.safe_list) == 4
assert item.safe_list[0].path == "tmp/safe"
assert item.safe_list[1].path == "tmp/diff_time"
assert item.safe_list[2].path == "tmp/diff_size"
assert item.safe_list[3].path == "tmp/new"
def test_list_files(self, rsync_factory_mock):
"""
Unit test for RsyncCopyController._list_file's code
"""
# Mock rsync invocation
rsync_mock = mock.Mock(name="Rsync()")
rsync_mock.ret = 0
rsync_mock.out = (
"drwxrwxrwt 69632 2015/02/09 15:01:00 tmp\n"
"drwxrwxrwt 69612 Thu Feb 19 15:01:22 2015 tmp2")
rsync_mock.err = "err"
# Mock _rsync_factory() invocation
rsync_factory_mock.return_value = rsync_mock
# Create an item to inspect
item = _RsyncCopyItem(
label="pgdata",
src=":/pg/data/",
dst="/some/dir",
is_directory=True,
item_class=RsyncCopyController.PGDATA_CLASS,
optional=False,
)
# Test the _list_files internal method
rcc = RsyncCopyController()
return_values = list(rcc._list_files(item, "some/path"))
# Returned list must contain two elements
assert len(return_values) == 2
# Verify that _rsync_factory has been called correctly
assert rsync_factory_mock.mock_calls == [
mock.call(item),
]
# Check rsync.get_output has called correctly
rsync_mock.get_output.assert_called_with("--no-human-readable",
"--list-only",
"-r",
"some/path",
check=True)
# Check the result
assert return_values[0] == _FileItem(
"drwxrwxrwt",
69632,
datetime(
year=2015,
month=2,
day=9,
hour=15,
minute=1,
second=0,
tzinfo=dateutil.tz.tzlocal(),
),
"tmp",
)
assert return_values[1] == _FileItem(
"drwxrwxrwt",
69612,
datetime(
year=2015,
month=2,
day=19,
hour=15,
minute=1,
second=22,
tzinfo=dateutil.tz.tzlocal(),
),
"tmp2",
)
# Test the _list_files internal method with a wrong output (added TZ)
rsync_mock.out = "drwxrwxrwt 69612 Thu Feb 19 15:01:22 CET 2015 tmp2\n"
rcc = RsyncCopyController()
with pytest.raises(RsyncListFilesFailure):
# The list() call is needed to consume the generator
list(rcc._list_files(rsync_mock, "some/path"))
# Check rsync.get_output has called correctly
rsync_mock.get_output.assert_called_with("--no-human-readable",
"--list-only",
"-r",
"some/path",
check=True)
def analyse_func(item):
label = item.label
item.dir_file = label + "_dir_file"
item.exclude_and_protect_file = label + "_exclude_and_protect_file"
item.safe_list = [_FileItem("mode", 1, "date", "path")]
item.check_list = [_FileItem("mode", 1, "date", "path")]
