def test_missing_files():
top_level_directory = TEMP_DIR / "2020-07-04_12_04_489"
directories = [
top_level_directory / "config",
top_level_directory / "database",
top_level_directory / "elasticsearch",
top_level_directory / "files",
top_level_directory / "hbase" / "archive" / "data" / "tamr",
top_level_directory / "files" / "dataset",
top_level_directory / "files" / "dedup",
top_level_directory / "files" / "job",
top_level_directory / "files" / "procurify",
]
bash.create_directories(directories)
files = [
top_level_directory / "config" / "config.json",
top_level_directory / "database" / "pg_dump.out",
top_level_directory / "_SUCCEEDED",
]
for file in files:
with open(file, "w"):
pass
valid = backup.validate_backup(Path(top_level_directory))
assert not valid
backups = [top_level_directory]
bash.remove_directories(backups, allow_recursive_deletes=True)
def test_keeping_more_failed_backups_than_present():
failed_backups = [
"2020-07-05_13-33-42-871",
"2020-07-05_14-33-42-871",
"2020-07-05_15-33-42-871",
]
for backup_name in failed_backups:
_make_backup(backup_name, "_FAILED")
valid = backup.validate_backup(TEMP_DIR / backup_name)
assert valid
result = backup.classify_backups(TEMP_DIR)
assert len(result["not_succeeded"]) == 3
deleted_backups = backup.delete_old_backups(
TEMP_DIR,
num_successful_backups_to_keep=3,
num_failed_backups_to_keep=5)
assert len(deleted_backups) == 0
result = backup.classify_backups(TEMP_DIR)
assert len(result["not_succeeded"]) == 3
backup.delete_backups(backups=failed_backups, backup_directory=TEMP_DIR)
def test_keeping_more_failed_backups_than_present():
failed_backups = [
"2020-07-05_13-33-42-871",
"2020-07-05_14-33-42-871",
"2020-07-05_15-33-42-871",
]
with tempfile.TemporaryDirectory() as tempdir:
for backup_name in failed_backups:
_make_backup(backup_name, "_FAILED", Path(tempdir))
valid = backup.validate_backup(Path(tempdir) / backup_name)
assert valid
result = backup.classify_backups(Path(tempdir))
assert len(result["not_succeeded"]) == 3
deleted_backups = backup.delete_old_backups(
Path(tempdir), num_successful_backups_to_keep=3, num_failed_backups_to_keep=5
)
assert len(deleted_backups) == 0
result = backup.classify_backups(Path(tempdir))
assert len(result["not_succeeded"]) == 3
backup.delete_backups(backups=failed_backups, backup_directory=Path(tempdir))
def test_keeping_backups():
completed_backups = [
"2020-07-04_13-33-42-871",
"2020-07-04_14-33-42-871",
"2020-07-04_15-33-42-871",
"2020-07-04_16-33-42-871",
"2020-07-04_17-33-42-871",
]
failed_backups = [
"2020-07-05_13-33-42-871",
"2020-07-05_14-33-42-871",
"2020-07-05_15-33-42-871",
]
completed_backups_keep = [
"2020-07-04_15-33-42-871",
"2020-07-04_16-33-42-871",
"2020-07-04_17-33-42-871",
]
failed_backups_keep = ["2020-07-05_15-33-42-871"]
for backup_name in completed_backups:
_make_backup(backup_name, "_SUCCEEDED")
valid = backup.validate_backup(TEMP_DIR / backup_name)
assert valid
for backup_name in failed_backups:
_make_backup(backup_name, "_FAILED")
valid = backup.validate_backup(TEMP_DIR / backup_name)
assert valid
result = backup.classify_backups(TEMP_DIR)
assert len(result["succeeded"]) == 5
assert len(result["not_succeeded"]) == 3
deleted_backups = backup.delete_old_backups(
TEMP_DIR,
num_successful_backups_to_keep=3,
num_failed_backups_to_keep=1)
assert len(deleted_backups) == 4
result = backup.classify_backups(TEMP_DIR)
assert len(result["succeeded"]) == 3
assert len(result["not_succeeded"]) == 1
all_backups = completed_backups_keep + failed_backups_keep
backup.delete_backups(backups=all_backups, backup_directory=TEMP_DIR)
def test_keeping_backups():
completed_backups = [
"2020-07-04_13-33-42-871",
"2020-07-04_14-33-42-871",
"2020-07-04_15-33-42-871",
"2020-07-04_16-33-42-871",
"2020-07-04_17-33-42-871",
]
failed_backups = [
"2020-07-05_13-33-42-871",
"2020-07-05_14-33-42-871",
"2020-07-05_15-33-42-871",
]
completed_backups_keep = [
"2020-07-04_15-33-42-871",
"2020-07-04_16-33-42-871",
"2020-07-04_17-33-42-871",
]
failed_backups_keep = ["2020-07-05_15-33-42-871"]
with tempfile.TemporaryDirectory() as tempdir:
for backup_name in completed_backups:
_make_backup(backup_name, "_SUCCEEDED", Path(tempdir))
valid = backup.validate_backup(Path(tempdir) / backup_name)
assert valid
for backup_name in failed_backups:
_make_backup(backup_name, "_FAILED", Path(tempdir))
valid = backup.validate_backup(Path(tempdir) / backup_name)
assert valid
result = backup.classify_backups(Path(tempdir))
assert len(result["succeeded"]) == 5
assert len(result["not_succeeded"]) == 3
deleted_backups = backup.delete_old_backups(
Path(tempdir), num_successful_backups_to_keep=3, num_failed_backups_to_keep=1
)
assert len(deleted_backups) == 4
result = backup.classify_backups(Path(tempdir))
assert len(result["succeeded"]) == 3
assert len(result["not_succeeded"]) == 1
all_backups = completed_backups_keep + failed_backups_keep
backup.delete_backups(backups=all_backups, backup_directory=Path(tempdir))
def test_manifest_invalid_extension():
backup_name = "2020-12-16_19-33-03-982"
_make_backup(backup_name, "_SUCCEEDED", manifest_extension="txt")
valid = backup.validate_backup(TEMP_DIR / backup_name)
assert not valid
backups = [TEMP_DIR / backup_name]
bash.remove_directories(backups, allow_recursive_deletes=True)
def test_invalid_date():
backup_name = "2020-07_12_04_489"
_make_backup(backup_name, "_SUCCEEDED")
valid = backup.validate_backup(TEMP_DIR / backup_name)
assert not valid
backups = [TEMP_DIR / backup_name]
bash.remove_directories(backups, allow_recursive_deletes=True)
def test_manifest_invalid_extension():
backup_name = "2020-12-16_19-33-03-982"
with tempfile.TemporaryDirectory() as tempdir:
_make_backup(backup_name, "_SUCCEEDED", Path(tempdir), manifest_extension="txt")
valid = backup.validate_backup(Path(tempdir) / backup_name)
assert not valid
backups = [Path(tempdir) / backup_name]
bash.remove_directories(backups, allow_recursive_deletes=True)
def test_invalid_date():
backup_name = "2020-07_12_04_489"
with tempfile.TemporaryDirectory() as tempdir:
_make_backup(backup_name, "_SUCCEEDED", Path(tempdir))
valid = backup.validate_backup(Path(tempdir) / backup_name)
assert not valid
backups = [Path(tempdir) / backup_name]
bash.remove_directories(backups, allow_recursive_deletes=True)
def test_validate_and_remove_backup(backups: List[str], completion: str):
all_directories = []
all_files = []
for backup_name in backups:
directories, files = _make_backup(backup_name, completion)
all_directories.extend(directories)
all_files.extend(files)
valid = backup.validate_backup(TEMP_DIR / backup_name)
assert valid
backup.delete_backups(backups=backups, backup_directory=TEMP_DIR)
for directory in all_directories:
assert not path.exists(
directory), f"Removed directory {directory} still exists"
for file in all_files:
assert not path.exists(file), f"Removed file {file} still exists"
def test_validate_and_remove_backup(backups: List[str], completion: str):
all_directories = []
all_files = []
with tempfile.TemporaryDirectory() as tempdir:
for backup_name in backups:
directories, files = _make_backup(backup_name, completion, Path(tempdir))
all_directories.extend(directories)
all_files.extend(files)
valid = backup.validate_backup(Path(tempdir) / backup_name)
assert valid
backup.delete_backups(backups=backups, backup_directory=Path(tempdir))
for directory in all_directories:
assert not path.exists(directory), f"Removed directory {directory} still exists"
for file in all_files:
assert not path.exists(file), f"Removed file {file} still exists"