def test_mv_command(self):
src = "Mail"
dst = "Mail.snapshots"
snapshotter.snapshot(src, dst)
mv = ' '.join(self.mock_run_function.call_args_list[1][0][0])
rm = ' '.join(self.mock_run_function.call_args_list[2][0][0])
ln = ' '.join(self.mock_run_function.call_args_list[3][0][0])
# The absolute path to the incomplete.snapshot dir, no trailing /.
incomplete_dir = os.path.join(
os.path.abspath(dst), "incomplete.snapshot")
# The absolute path to the latest.snapshot symlink.
latest = os.path.join(
os.path.abspath(dst), "latest.snapshot")
# The absolute path to the YYYY-MM-DDTHH_MM_SS.snapshot dir,
# no trailing /.
snapshot_dir = os.path.join(
os.path.abspath(dst), self.datetime + ".snapshot")
assert mv == "mv {incomplete} {snapshot}".format(
incomplete=incomplete_dir, snapshot=snapshot_dir)
assert rm == "rm -f {latest}".format(latest=latest)
assert ln == "ln -s {snapshot} {latest}".format(
snapshot=self.datetime + ".snapshot", latest=latest)
def test_extra_args_are_passed_on_to_rsync(self):
extra_args = ["-v", "--info=progress2"]
snapshotter.snapshot("src", "dest", extra_args=extra_args)
for arg in extra_args:
assert arg in self.mock_run_function.call_args_list[0][0][0]
def test_extra_args_are_passed_on_to_rsync(self):
extra_args = ["-v", "--info=progress2"]
snapshotter.snapshot("src", "dest", extra_args=extra_args)
for arg in extra_args:
assert arg in self.mock_run_function.call_args_list[0][0][0]
def test_removing_oldest_snapshot(self):
"""If out of space it should remove oldest snapshot and rerun rsync."""
snapshots = [
"2015-03-05T16_23_12.snapshot",
"2015-03-05T16_24_15.snapshot",
"2015-03-05T16_25_09.snapshot",
"2015-03-05T16_27_09.snapshot",
"2015-03-05T16_28_09.snapshot",
"2015-03-05T16_29_09.snapshot",
]
self.mock_ls_snapshots_function.return_value = snapshots
rsync_returns = [snapshotter.NoSpaceLeftOnDeviceError(), None]
def rsync(*args, **kwargs):
result = rsync_returns.pop(0)
if isinstance(result, Exception):
raise result
else:
return result
self.mock_rsync_function.side_effect = rsync
snapshotter.snapshot("source", "destination")
assert self.mock_rm_function.call_count == 2
assert self.mock_rm_function.call_args_list[0] == mock.call(
'2015-03-05T16_23_12.snapshot',
None,
None,
debug=False,
directory=True)
assert self.mock_rsync_function.call_count == 2
def test_removing_oldest_snapshot(self):
"""If out of space it should remove oldest snapshot and rerun rsync."""
snapshots = [
"2015-03-05T16_23_12.snapshot",
"2015-03-05T16_24_15.snapshot",
"2015-03-05T16_25_09.snapshot",
"2015-03-05T16_27_09.snapshot",
"2015-03-05T16_28_09.snapshot",
"2015-03-05T16_29_09.snapshot",
]
self.mock_ls_snapshots_function.return_value = snapshots
rsync_returns = [snapshotter.NoSpaceLeftOnDeviceError(), None]
def rsync(*args, **kwargs):
result = rsync_returns.pop(0)
if isinstance(result, Exception):
raise result
else:
return result
self.mock_rsync_function.side_effect = rsync
snapshotter.snapshot("source", "destination")
assert self.mock_rm_function.call_count == 2
assert self.mock_rm_function.call_args_list[0] == mock.call(
'2015-03-05T16_23_12.snapshot', None, None, debug=False,
directory=True)
assert self.mock_rsync_function.call_count == 2
def test_mv_command(self):
src = "Mail"
dst = "Mail.snapshots"
snapshotter.snapshot(src, dst)
mv = ' '.join(self.mock_run_function.call_args_list[1][0][0])
rm = ' '.join(self.mock_run_function.call_args_list[2][0][0])
ln = ' '.join(self.mock_run_function.call_args_list[3][0][0])
# The absolute path to the incomplete.snapshot dir, no trailing /.
incomplete_dir = os.path.join(os.path.abspath(dst),
"incomplete.snapshot")
# The absolute path to the latest.snapshot symlink.
latest = os.path.join(os.path.abspath(dst), "latest.snapshot")
# The absolute path to the YYYY-MM-DDTHH_MM_SS.snapshot dir,
# no trailing /.
snapshot_dir = os.path.join(os.path.abspath(dst),
self.datetime + ".snapshot")
assert mv == "mv {incomplete} {snapshot}".format(
incomplete=incomplete_dir, snapshot=snapshot_dir)
assert rm == "rm -f {latest}".format(latest=latest)
assert ln == "ln -s {snapshot} {latest}".format(
snapshot=self.datetime + ".snapshot", latest=latest)
def test_it_raises_if_max_snapshots_equals_min_snapshots(self):
try:
snapshotter.snapshot("/home/fred",
"/media/backup",
min_snapshots=5,
max_snapshots=5)
assert False, "snapshot() should have raised an exception"
except snapshotter.InconsistentArgumentsError:
pass
def test_with_remote_source(self):
src = "[email protected]:Documents"
dst = "Snapshots/Documents"
snapshotter.snapshot(src, dst)
args = _get_args(self.mock_run_function.call_args_list[0])
src_arg = args[-2]
assert src_arg == "{src}/".format(src=src)
def test_with_trailing_slash(self):
"""A trailing / is given it should be left there."""
src = "/home/fred/" # Trailing slash.
dst = "/media/backup"
snapshotter.snapshot(src, dst)
args = _get_args(self.mock_run_function.call_args_list[0])
assert "/home/fred/" in args
def test_without_trailing_slash(self):
"""A trailing / should be appened to the source path if not given."""
src = "/home/fred" # No trailing slash.
dst = "/media/backup"
snapshotter.snapshot(src, dst)
args = _get_args(self.mock_run_function.call_args_list[0])
assert "/home/fred/" in args
def test_without_trailing_slash(self):
"""A trailing / should be appened to the source path if not given."""
src = "/home/fred" # No trailing slash.
dst = "/media/backup"
snapshotter.snapshot(src, dst)
args = _get_args(self.mock_run_function.call_args_list[0])
assert "/home/fred/" in args
def test_with_trailing_slash(self):
"""A trailing / is given it should be left there."""
src = "/home/fred/" # Trailing slash.
dst = "/media/backup"
snapshotter.snapshot(src, dst)
args = _get_args(self.mock_run_function.call_args_list[0])
assert "/home/fred/" in args
def test_with_remote_source(self):
src = "[email protected]:Documents"
dst = "Snapshots/Documents"
snapshotter.snapshot(src, dst)
args = _get_args(self.mock_run_function.call_args_list[0])
src_arg = args[-2]
assert src_arg == "{src}/".format(src=src)
def test_with_remote_dest(self):
src = "Documents"
dst = "[email protected]:Snapshots/Documents"
snapshotter.snapshot(src, dst)
expected_destination = os.path.join(dst, "incomplete.snapshot")
args = _get_args(self.mock_run_function.call_args_list[0])
dst_arg = args[-1]
assert dst_arg == "{dst}".format(dst=expected_destination)
def test_tilde_in_backup_source(self):
"""Test giving a source path with a ~ in it."""
src = "~"
dst = "/media/SNAPSHOTS"
snapshotter.snapshot(src, dst)
args = _get_args(self.mock_run_function.call_args_list[0])
src_arg = args[-2]
assert src_arg == "~/"
def test_tilde_in_backup_source(self):
"""Test giving a source path with a ~ in it."""
src = "~"
dst = "/media/SNAPSHOTS"
snapshotter.snapshot(src, dst)
args = _get_args(self.mock_run_function.call_args_list[0])
src_arg = args[-2]
assert src_arg == "~/"
def test_it_raises_if_max_snapshots_equals_min_snapshots(self):
try:
snapshotter.snapshot(
"/home/fred",
"/media/backup",
min_snapshots=5,
max_snapshots=5)
assert False, "snapshot() should have raised an exception"
except snapshotter.InconsistentArgumentsError:
pass
def test_with_remote_dest(self):
src = "Documents"
dst = "[email protected]:Snapshots/Documents"
snapshotter.snapshot(src, dst)
expected_destination = os.path.join(dst, "incomplete.snapshot")
args = _get_args(self.mock_run_function.call_args_list[0])
dst_arg = args[-1]
assert dst_arg == "{dst}".format(dst=expected_destination)
def test_not_passing_dry_run_to_rsync(self):
"""If --n isn't given to snapshotter it shouldn't be given to rsync."""
src = "/home/fred"
dst = "/media/backup"
snapshotter.snapshot(src, dst, debug=False)
assert self.mock_run_function.call_count == 4, (
"We expect 4 commands to be run: rsync, mv, rm, ln")
args = _get_args(self.mock_run_function.call_args_list[0])
assert "--dry-run" not in args
def test_not_passing_dry_run_to_rsync(self):
"""If --n isn't given to snapshotter it shouldn't be given to rsync."""
src = "/home/fred"
dst = "/media/backup"
snapshotter.snapshot(src, dst, debug=False)
assert self.mock_run_function.call_count == 4, (
"We expect 4 commands to be run: rsync, mv, rm, ln")
args = _get_args(self.mock_run_function.call_args_list[0])
assert "--dry-run" not in args
def test_relative_local_to_relative_local(self):
"""Test backing up a relative local dir to a relative local dir."""
src = "Mail"
dst = "Mail.snapshots"
snapshotter.snapshot(src, dst)
args = _get_args(self.mock_run_function.call_args_list[0])
src_arg = args[-2]
dst_arg = args[-1]
assert src_arg == "Mail/"
assert dst_arg == os.path.join(os.getcwd(), dst, "incomplete.snapshot")
def test_relative_local_to_absolute_local(self):
"""Test backing up a relative local dir to an absolute local dir."""
src = "Music"
dst = "/media/backup/Music.snapshots"
snapshotter.snapshot(src, dst)
args = _get_args(self.mock_run_function.call_args_list[0])
src_arg = args[-2]
dst_arg = args[-1]
assert src_arg == "Music/"
assert dst_arg == "/media/backup/Music.snapshots/incomplete.snapshot"
def test_rsync_error(self):
"""Should raise CalledProcessError if rsync exits with non-zero."""
self.mock_run_function.side_effect = snapshotter.CalledProcessError(
"command", "output", 11)
src = "/home/fred"
dst = "/media/backup"
try:
snapshotter.snapshot(src, dst, debug=True)
assert False, "snapshot() should have raised an exception"
except snapshotter.CalledProcessError as err:
assert err.output == "output 11"
def test_passing_dry_run_to_rsync(self):
"""snapshot() should pass -n/--dry-run on to rsync."""
src = "/home/fred"
dst = "/media/backup"
snapshotter.snapshot(src, dst, debug=True)
args = _get_args(self.mock_run_function.call_args_list[0])
assert "--dry-run" in args, (
"snapshot() should pass the -n/--dry-run argument on to rsync")
for call in self.mock_run_function.call_args_list[1:]:
assert call[1].get("debug") is True
def test_root_as_source(self):
"""Test giving / as the source path."""
src = "/"
dst = "/media/SNAPSHOTS"
snapshotter.snapshot(src, dst)
args = _get_args(self.mock_run_function.call_args_list[0])
src_arg = args[-2]
dst_arg = args[-1]
assert src_arg == "/"
assert dst_arg == "/media/SNAPSHOTS/incomplete.snapshot"
def test_relative_local_to_relative_local(self):
"""Test backing up a relative local dir to a relative local dir."""
src = "Mail"
dst = "Mail.snapshots"
snapshotter.snapshot(src, dst)
args = _get_args(self.mock_run_function.call_args_list[0])
src_arg = args[-2]
dst_arg = args[-1]
assert src_arg == "Mail/"
assert dst_arg == os.path.join(os.getcwd(), dst, "incomplete.snapshot")
def test_relative_local_to_absolute_local(self):
"""Test backing up a relative local dir to an absolute local dir."""
src = "Music"
dst = "/media/backup/Music.snapshots"
snapshotter.snapshot(src, dst)
args = _get_args(self.mock_run_function.call_args_list[0])
src_arg = args[-2]
dst_arg = args[-1]
assert src_arg == "Music/"
assert dst_arg == "/media/backup/Music.snapshots/incomplete.snapshot"
def test_rsync_error(self):
"""Should raise CalledProcessError if rsync exits with non-zero."""
self.mock_run_function.side_effect = snapshotter.CalledProcessError(
"command", "output", 11)
src = "/home/fred"
dst = "/media/backup"
try:
snapshotter.snapshot(src, dst, debug=True)
assert False, "snapshot() should have raised an exception"
except snapshotter.CalledProcessError as err:
assert err.output == "output 11"
def test_passing_dry_run_to_rsync(self):
"""snapshot() should pass -n/--dry-run on to rsync."""
src = "/home/fred"
dst = "/media/backup"
snapshotter.snapshot(src, dst, debug=True)
args = _get_args(self.mock_run_function.call_args_list[0])
assert "--dry-run" in args, (
"snapshot() should pass the -n/--dry-run argument on to rsync")
for call in self.mock_run_function.call_args_list[1:]:
assert call[1].get("debug") is True
def test_root_as_source(self):
"""Test giving / as the source path."""
src = "/"
dst = "/media/SNAPSHOTS"
snapshotter.snapshot(src, dst)
args = _get_args(self.mock_run_function.call_args_list[0])
src_arg = args[-2]
dst_arg = args[-1]
assert src_arg == "/"
assert dst_arg == "/media/SNAPSHOTS/incomplete.snapshot"
def test_mv_command_with_remote_dest(self):
src = "Mail"
dst = "[email protected]:/path/to/snapshots"
snapshotter.snapshot(src, dst)
incomplete_dir = "/path/to/snapshots/incomplete.snapshot"
snapshot_dir = "/path/to/snapshots/" + self.datetime + ".snapshot"
mv = ' '.join(self.mock_run_function.call_args_list[1][0][0])
expected_call = (
'ssh [email protected] mv {incomplete} {snapshot}'.format(
incomplete=incomplete_dir, snapshot=snapshot_dir))
assert mv == expected_call
def test_link_dest(self):
"""The right --link-dest=... arg should be given to rsync."""
src = "/home/fred"
dst = "/media/backup"
snapshotter.snapshot(src, dst)
args = _get_args(self.mock_run_function.call_args_list[0])
link_dest_args = [a for a in args if a.startswith("--link-dest")]
assert len(link_dest_args) == 1
link_dest_arg = link_dest_args[0]
_, value = link_dest_arg.split("=")
assert value == "../latest.snapshot"
def test_mv_command_fails(self):
"""snapshot() should raise if the mv command exits with non-zero."""
src = "Mail"
dst = "Mail.snapshots"
self.mock_run_function.side_effect = snapshotter.CalledProcessError(
"command", "output", 25)
try:
snapshotter.snapshot(src, dst)
assert False, "snapshot() should have raised an exception"
except snapshotter.CalledProcessError as err:
assert err.output == "output 25"
def test_mv_command_fails(self):
"""snapshot() should raise if the mv command exits with non-zero."""
src = "Mail"
dst = "Mail.snapshots"
self.mock_run_function.side_effect = snapshotter.CalledProcessError(
"command", "output", 25)
try:
snapshotter.snapshot(src, dst)
assert False, "snapshot() should have raised an exception"
except snapshotter.CalledProcessError as err:
assert err.output == "output 25"
def test_it_removes_oldest_snapshots_when_more_than_max_snapshots(self):
snapshots = ["first", "second", "third", "fourth", "fifth"]
self.mock_ls_snapshots_function.return_value = snapshots
def remove_oldest_snapshot(*args, **kwargs):
snapshots.pop()
self.mock_remove_oldest_snapshot_function.side_effect = (
remove_oldest_snapshot)
snapshotter.snapshot("/home/fred", "/media/backup", max_snapshots=4)
assert self.mock_remove_oldest_snapshot_function.call_count == 2
def test_mv_command_with_remote_dest_with_no_user(self):
src = "Mail"
dst = "yourdomain.org:/path/to/snapshots"
snapshotter.snapshot(src, dst)
incomplete_dir = "/path/to/snapshots/incomplete.snapshot"
snapshot_dir = "/path/to/snapshots/" + self.datetime + ".snapshot"
mv = ' '.join(self.mock_run_function.call_args_list[1][0][0])
expected_call = ('ssh yourdomain.org '
'mv {incomplete} {snapshot}'.format(
incomplete=incomplete_dir, snapshot=snapshot_dir))
assert mv == expected_call
def test_link_dest(self):
"""The right --link-dest=... arg should be given to rsync."""
src = "/home/fred"
dst = "/media/backup"
snapshotter.snapshot(src, dst)
args = _get_args(self.mock_run_function.call_args_list[0])
link_dest_args = [
a for a in args if a.startswith("--link-dest")]
assert len(link_dest_args) == 1
link_dest_arg = link_dest_args[0]
_, value = link_dest_arg.split("=")
assert value == "../latest.snapshot"
def test_it_removes_oldest_snapshots_when_more_than_max_snapshots(self):
snapshots = ["first", "second", "third", "fourth", "fifth"]
self.mock_ls_snapshots_function.return_value = snapshots
def remove_oldest_snapshot(*args, **kwargs):
snapshots.pop()
self.mock_remove_oldest_snapshot_function.side_effect = (
remove_oldest_snapshot)
snapshotter.snapshot(
"/home/fred",
"/media/backup",
max_snapshots=4)
assert self.mock_remove_oldest_snapshot_function.call_count == 2
def snapshot(env, min_snapshots, max_snapshots, unless_absent, dbname, dest):
""" Create an Odoo database snapshot from an existing one.
This script dumps the database using pg_dump.
It also copies the filestore.
Unlike Odoo, this script allows you to make a backup of a
database without going through the web interface. This
avoids timeout and file size limitation problems when
databases are too large.
It also creates and maintains the specified amount of
snapshots in the target directory. The snapshot
implementaiton uses rsync. It hardlinks files that did not
change. This can save a lot of disk space and bandwith
especially for filestore items.
(TBD) You can define a ssh destination.
"""
if not shutil.which("rsync"):
msg = "rsync binary not found in path."
raise click.ClickException(msg)
if not db_exists(dbname):
msg = "Database does not exist: {}".format(dbname)
raise click.ClickException(msg)
if not os.path.exists(dest) and unless_absent:
msg = "Destination does not exist: {}".format(dest)
raise click.ClickException(msg)
elif not os.path.exists(dest):
os.makedirs(dest)
if max_snapshots <= min_snapshots:
msg = "--max-snapshots must be greater than --min-snapshots"
raise click.ClickException(msg)
db = odoo.sql_db.db_connect(dbname)
try:
with odoo.tools.osutil.tempdir() as temp_dir:
with do_backup("folder", temp_dir,
"w") as _backup, db.cursor() as cr:
_create_manifest(cr, dbname, _backup)
_backup_filestore(dbname, _backup)
_dump_db(dbname, _backup)
snapshotter.snapshot(temp_dir, dest, False, min_snapshots,
max_snapshots)
finally:
odoo.sql_db.close_db(dbname)
def test_functional(self, mock_datetime_function):
"""One functional test that actually calls rsync and copies files."""
datetime = "2015-02-23T18_58_02"
mock_datetime_function.return_value = datetime
try:
dest = tempfile.mkdtemp()
src = os.path.join(_this_directory(), "test_data")
snapshotter.snapshot(src, dest)
snapshot_dir = os.path.join(dest, datetime + ".snapshot")
assert os.path.isdir(snapshot_dir)
for file_ in os.listdir(src):
assert os.path.isfile(os.path.join(snapshot_dir, file_))
latest_symlink = os.path.join(dest, "latest.snapshot")
assert os.path.islink(latest_symlink)
assert os.path.realpath(latest_symlink) == snapshot_dir
finally:
shutil.rmtree(dest)
def test_functional(self, mock_datetime_function):
"""One functional test that actually calls rsync and copies files."""
datetime = "2015-02-23T18_58_02"
mock_datetime_function.return_value = datetime
try:
dest = tempfile.mkdtemp()
src = os.path.join(_this_directory(), "test_data")
snapshotter.snapshot(src, dest)
snapshot_dir = os.path.join(dest, datetime + ".snapshot")
assert os.path.isdir(snapshot_dir)
for file_ in os.listdir(src):
assert os.path.isfile(os.path.join(snapshot_dir, file_))
latest_symlink = os.path.join(dest, "latest.snapshot")
assert os.path.islink(latest_symlink)
assert os.path.realpath(latest_symlink) == snapshot_dir
finally:
shutil.rmtree(dest)
def test_it_does_not_raise_if_min_snapshots_less_than_max_snapshots(self):
snapshotter.snapshot(
"/home/fred",
"/media/backup",
min_snapshots=4,
max_snapshots=5)
def _snapshot(self, path):
snapshotter.snapshot(path, '/tmp/snap')
def test_it_does_not_raise_if_min_snapshots_less_than_max_snapshots(self):
snapshotter.snapshot("/home/fred",
"/media/backup",
min_snapshots=4,
max_snapshots=5)
