def test_will_not_run_with_missing_name(reset_mockTime):
conf = {
"frequency": "* * *",
}
with pytest.raises(KeyError) as excinfo:
BackupConfig.from_dict(conf)
assert "'name'" in str(excinfo.value)
def test_view_backups(
reset_mockTime,
patch_datetime_now,
populate_conf_dir,
populate_data_dir,
tmp_path,
capsys,
):
expected = "full/TEST_APP_FULL_2020-12-25T170555.tgz\n"
# Set the backup directory.
backup_dir = tmp_path / BASE_BACKUP_DIR
# Create the click context that backup_manager expects to deal with
ctx = create_click_ctx(
Path(populate_conf_dir), Path(populate_data_dir), Path(backup_dir)
)
# Create our configuration
conf = {
"name": "full",
"backup_limit": 0,
}
# Create a backup to view.
with capsys.disabled():
backup_config = BackupConfig.from_dict(conf)
backup_config.backup(ctx)
backup_manager = BackupManager(conf)
backup_manager.view_backups(ctx)
captured = capsys.readouterr()
output = captured.out
assert expected == output
def test_simple_S3_remote_backup_parsing(reset_mockTime):
conf = {
"name": "full",
"remote_backups": [
{
"name": "s3_sunday",
"strategy_type": "S3",
"frequency": "* * 0",
"configuration": {
"bucket_name": "name",
"access_key": "asdf123",
"secret_key": "qwer456",
"bucket_path": "home/weekly",
"tags": {"frequency": "weekley", "type": "data"},
},
}
],
}
backup_config = BackupConfig.from_dict(conf)
assert backup_config.remote_backups[0].name == "s3_sunday"
assert backup_config.remote_backups[0].strategy_type == "S3"
assert backup_config.remote_backups[0].frequency == "* * 0"
assert backup_config.remote_backups[0].strategy.bucket_name == "name"
assert backup_config.remote_backups[0].strategy.access_key == "asdf123"
assert backup_config.remote_backups[0].strategy.secret_key == "qwer456"
assert backup_config.remote_backups[0].strategy.bucket_path == "home/weekly"
assert backup_config.remote_backups[0].strategy.tags["frequency"] == "weekley"
assert backup_config.remote_backups[0].strategy.tags["type"] == "data"
def test_frequency_always_run():
conf = {"name": "full", "frequency": "* * *"}
backup_config = BackupConfig.from_dict(conf)
result = backup_config.should_run()
assert result
def test_backup_with_limit_keep_last_5(
reset_mockTime, patch_datetime_now, populate_conf_dir, populate_data_dir, tmp_path
):
# Set the backup directory.
backup_dir = tmp_path / BASE_BACKUP_DIR
# Create the click context that backup_manager expects to deal with.
ctx = create_click_ctx(
Path(populate_conf_dir), Path(populate_data_dir), Path(backup_dir)
)
# Create our configuration.
conf = {
"name": "full",
"backup_limit": 5,
}
expected_result = set(
{
"TEST_APP_FULL_2020-12-25T170600.tgz",
"TEST_APP_FULL_2020-12-25T170601.tgz",
"TEST_APP_FULL_2020-12-25T170602.tgz",
"TEST_APP_FULL_2020-12-25T170603.tgz",
"TEST_APP_FULL_2020-12-25T170604.tgz",
}
)
backup_config = BackupConfig.from_dict(conf)
for x in range(10):
backup = backup_config.backup(ctx)
mock_time.increment()
# Compare as a set so list order does not matter.
assert set(os.listdir(os.path.dirname(backup))) == expected_result
def test_data_dir_include_and_exclude_list(
reset_mockTime, populate_conf_dir, populate_data_dir, tmp_path
):
# Set the backup directory.
backup_dir = tmp_path / BASE_BACKUP_DIR
# Create the click context that backup_manager expects to deal with.
ctx = create_click_ctx(
Path(populate_conf_dir), Path(populate_data_dir), Path(backup_dir)
)
# Build our list of expected files to find in the backup
expected_files = get_expected_files(populate_data_dir, DATA_FILES_ALL).union(
get_expected_files(populate_conf_dir, CONF_FILES_ALL)
)
expected_files = get_expected_files(populate_conf_dir, set([".hidden/11.log"]))
expected_files = expected_files.union(
get_expected_files(populate_data_dir, (set(["populated_folder/third.log"])))
)
# Create our configuration.
conf = {
"name": "full",
"file_filter": {
"data_dir": {"include_list": ["**/*.log"], "exclude_list": ["**/?.log"]},
"conf_dir": {
"include_list": [".hidden/**/*"],
"exclude_list": ["**/*.txt"],
},
},
}
backup_config = BackupConfig.from_dict(conf)
backup = backup_config.backup(ctx)
assert Path(backup).exists()
assert get_tar_contents(backup) == expected_files
def test_minimum_config_required_for_local_backup(
reset_mockTime, populate_conf_dir, populate_data_dir, tmp_path
):
"""
Try to create a backup file with the minimum of config set.
Check that the file was created and that its contents were expected.
"""
# Set the backup directory.
backup_dir = tmp_path / BASE_BACKUP_DIR
# Create the click context that backup_manager expects to deal with
ctx = create_click_ctx(
Path(populate_conf_dir), Path(populate_data_dir), Path(backup_dir)
)
# Build our list of expected files to find in the backup
expected_files = get_expected_files(populate_data_dir, DATA_FILES_ALL).union(
get_expected_files(populate_conf_dir, CONF_FILES_ALL)
)
# Create our configuration
conf = {
"name": "full",
}
backup_config = BackupConfig.from_dict(conf)
backup = backup_config.backup(ctx)
assert Path(backup).exists()
assert get_tar_contents(backup) == expected_files
def test_view_backups_multiple_backup_strategies(
reset_mockTime,
patch_datetime_now,
populate_conf_dir,
populate_data_dir,
tmp_path,
capsys,
):
expected = (
"logs/TEST_APP_LOGS_2020-12-25T170556.tgz\n"
"full/TEST_APP_FULL_2020-12-25T170555.tgz\n"
)
# Set the backup directory.
backup_dir = tmp_path / BASE_BACKUP_DIR
# Create the click context that backup_manager expects to deal with
ctx = create_click_ctx(
Path(populate_conf_dir), Path(populate_data_dir), Path(backup_dir)
)
# Create our configuration
conf_full = {
"name": "full",
"backup_limit": 0,
}
conf_logs = {
"name": "logs",
"file_filter": {
"data_dir": {"include_list": ["1.txt", "**/first.txt"]},
"conf_dir": {"include_list": ["**/*.txt"]},
},
}
# Create a backup to view.
with capsys.disabled():
backup_config = BackupConfig.from_dict(conf_full)
backup_config.backup(ctx)
mock_time.increment()
backup_config = BackupConfig.from_dict(conf_logs)
backup_config.backup(ctx)
mock_time.increment()
backup_manager = BackupManager([])
backup_manager.view_backups(ctx)
captured = capsys.readouterr()
output = captured.out
assert expected == output
def test_frequency_first_of_month_correct_date(reset_mockTime, monkeypatch):
monkeypatch.setattr(
"time.time",
lambda: 1614568341, # 1st of march
)
conf = {"name": "full", "frequency": "1 * *"}
backup_config = BackupConfig.from_dict(conf)
result = backup_config.should_run()
assert result
def test_defaults_set_correctly_empty_filter_dirs(reset_mockTime):
conf = {
"name": "full",
"frequency": "",
"backup_limit": "",
"remote_backups": "",
"file_filter": {"data_dir": {}, "conf_dir": {}},
}
backup_config = BackupConfig.from_dict(conf)
assert backup_config.file_filter.data_dir.include_list == ["**/*"]
assert backup_config.file_filter.data_dir.exclude_list == ["[]"]
assert backup_config.file_filter.conf_dir.include_list == ["**/*"]
assert backup_config.file_filter.conf_dir.exclude_list == ["[]"]
def test_defaults_set_correctly_when_missing(reset_mockTime):
conf = {
"name": "full",
}
backup_config = BackupConfig.from_dict(conf)
assert backup_config.name == "full"
assert backup_config.backup_limit == 0
assert backup_config.file_filter.data_dir.include_list == ["**/*"]
assert backup_config.file_filter.data_dir.exclude_list == ["[]"]
assert backup_config.file_filter.conf_dir.include_list == ["**/*"]
assert backup_config.file_filter.conf_dir.exclude_list == ["[]"]
assert backup_config.remote_backups == []
assert backup_config.frequency == "* * *"
def test_view_multiple_backups_descending_order(
reset_mockTime,
patch_datetime_now,
populate_conf_dir,
populate_data_dir,
tmp_path,
capsys,
):
expected = (
"full/TEST_APP_FULL_2020-12-25T170604.tgz\n"
+ "full/TEST_APP_FULL_2020-12-25T170603.tgz\n"
+ "full/TEST_APP_FULL_2020-12-25T170602.tgz\n"
+ "full/TEST_APP_FULL_2020-12-25T170601.tgz\n"
+ "full/TEST_APP_FULL_2020-12-25T170600.tgz\n"
+ "full/TEST_APP_FULL_2020-12-25T170559.tgz\n"
+ "full/TEST_APP_FULL_2020-12-25T170558.tgz\n"
+ "full/TEST_APP_FULL_2020-12-25T170557.tgz\n"
+ "full/TEST_APP_FULL_2020-12-25T170556.tgz\n"
+ "full/TEST_APP_FULL_2020-12-25T170555.tgz\n"
)
# Set the backup directory.
backup_dir = tmp_path / BASE_BACKUP_DIR
# Create the click context that backup_manager expects to deal with
ctx = create_click_ctx(
Path(populate_conf_dir), Path(populate_data_dir), Path(backup_dir)
)
# Create our configuration
conf = {
"name": "full",
"backup_limit": 0,
}
# Create backups to view.
with capsys.disabled():
backup_config = BackupConfig.from_dict(conf)
for x in range(10):
backup_config.backup(ctx)
mock_time.increment()
backup_manager = BackupManager(conf)
backup_manager.view_backups(ctx)
captured = capsys.readouterr()
output = captured.out
assert expected == output
def test_data_dir_missing_exclude_list(
reset_mockTime, populate_conf_dir, populate_data_dir, tmp_path
):
# Set the backup directory.
backup_dir = tmp_path / BASE_BACKUP_DIR
# Create the click context that backup_manager expects to deal with.
ctx = create_click_ctx(
Path(populate_conf_dir), Path(populate_data_dir), Path(backup_dir)
)
# Build our list of expected files to find in the backup
expected_files = get_expected_files(populate_data_dir, DATA_FILES_ALL).union(
get_expected_files(populate_conf_dir, CONF_FILES_ALL)
)
# Create our configuration.
conf = {"name": "full", "file_filter": {"data_dir": {"exclude_list": ""}}}
backup_config = BackupConfig.from_dict(conf)
backup = backup_config.backup(ctx)
assert Path(backup).exists()
assert get_tar_contents(backup) == expected_files
def test_backup_with_limit_keep_5(
reset_mockTime, patch_datetime_now, populate_conf_dir, populate_data_dir, tmp_path
):
# Set the backup directory.
backup_dir = tmp_path / BASE_BACKUP_DIR
# Create the click context that backup_manager expects to deal with.
ctx = create_click_ctx(
Path(populate_conf_dir), Path(populate_data_dir), Path(backup_dir)
)
# Create our configuration.
conf = {
"name": "full",
"backup_limit": 5,
}
backup_config = BackupConfig.from_dict(conf)
for x in range(10):
backup = backup_config.backup(ctx)
mock_time.increment()
assert len(os.listdir(os.path.dirname(backup))) == 5
def test_backup_with_unsafe_name_for_files(
reset_mockTime, patch_datetime_now, populate_conf_dir, populate_data_dir, tmp_path
):
# Set the backup directory.
backup_dir = tmp_path / BASE_BACKUP_DIR
# Create the click context that backup_manager expects to deal with.
ctx = create_click_ctx(
Path(populate_conf_dir), Path(populate_data_dir), Path(backup_dir)
)
# Create our configuration.
conf = {
"name": "Backup: Weekly > Sunday",
}
expected_result = [
"TEST_APP_BACKUP-WEEKLY-SUNDAY_2020-12-25T170555.tgz",
]
backup_config = BackupConfig.from_dict(conf)
backup = backup_config.backup(ctx)
# Compare as a set so list order does not matter.
assert os.listdir(os.path.dirname(backup)) == expected_result
def backup_tgz(populate_conf_dir, populate_data_dir, tmpdir_factory) -> Path:
"""
Populate a common backup directory and create a backup to be used by all tests.
As this is a session fixture any changes to this will effect all tests.
Returns:
backup: (Path). The backup created.
"""
backup_dir = tmpdir_factory.mktemp(BASE_BACKUP_DIR)
# Create the click context that backup_manager expects to deal with
ctx = create_click_ctx(
Path(populate_conf_dir), Path(populate_data_dir), Path(backup_dir)
)
# Create our configuration
conf = {
"name": "full",
}
backup_config = BackupConfig.from_dict(conf)
backup = backup_config.backup(ctx)
return backup