def test_missing_files():
top_level_directory = TEMP_DIR / "2020-07-04_12_04_489"
directories = [
top_level_directory / "config",
top_level_directory / "database",
top_level_directory / "elasticsearch",
top_level_directory / "files",
top_level_directory / "hbase" / "archive" / "data" / "tamr",
top_level_directory / "files" / "dataset",
top_level_directory / "files" / "dedup",
top_level_directory / "files" / "job",
top_level_directory / "files" / "procurify",
]
bash.create_directories(directories)
files = [
top_level_directory / "config" / "config.json",
top_level_directory / "database" / "pg_dump.out",
top_level_directory / "_SUCCEEDED",
]
for file in files:
with open(file, "w"):
pass
valid = backup.validate_backup(Path(top_level_directory))
assert not valid
backups = [top_level_directory]
bash.remove_directories(backups, allow_recursive_deletes=True)
def test_failure_to_create_directory_that_already_exists():
with tempfile.TemporaryDirectory() as tempdir:
top_level_directory = (
Path(tempdir) /
"test_failure_to_create_directory_that_already_exists")
directories = [
top_level_directory / "test1",
]
for directory in directories:
assert not path.exists(
directory), f"Directory {directory} already exists"
# Setup
bash.create_directories(directories)
# Test that cannot create again
with pytest.raises(OSError):
bash.create_directories(directories, failure_if_exists=True)
# Clean up
bash.remove_directories(directories)
bash.remove_directories([top_level_directory])
for directory in directories:
assert not path.exists(
directory), f"Removed directory {directory} still exists"
assert not path.exists(
top_level_directory
), f"Directory {top_level_directory} still exists after removal"
def delete_backups(
*,
backups: List[str],
backup_directory: Union[Path, str],
backup_datetime_format: str = "%Y-%m-%d_%H-%M-%S-%f",
) -> List[str]:
"""Deletes backup folders recursively.
Args:
backups: list of backups to delete
backup_directory: Path to backup directory
backup_datetime_format: String datetime format in backup folder name
Returns:
list of deleted backup names
"""
if not isinstance(backup_directory, Path):
backup_directory = Path(backup_directory)
backups_to_delete = sorted(backups, reverse=True)
LOGGER.info(
f"Deleting {len(backups_to_delete)} backup(s): '{backups_to_delete}'")
deleted_backups = []
for backup in backups:
backup_path = os.path.join(backup_directory, backup)
if validate_backup(Path(backup_path),
backup_datetime_format=backup_datetime_format):
bash.remove_directories([backup_path],
allow_recursive_deletes=True)
deleted_backups.append(backup)
else:
LOGGER.error(f"{backup} is not a valid backup to delete.")
return deleted_backups
def test_create_and_remove_directories_with_relative_path():
top_level_directory = TEMP_DIR / "test_create_and_remove_directories_with_relative_path"
directories = [
top_level_directory / "test1",
top_level_directory / "test2",
top_level_directory / "test3",
]
for directory in directories:
assert not path.exists(directory), f"Directory {directory} already exists"
bash.create_directories(directories, require_absolute_path=False)
for directory in directories:
assert path.exists(directory), f"Created directory {directory} does not exist"
bash.remove_directories(directories, require_absolute_path=False)
for directory in directories:
assert not path.exists(directory), f"Removed directory {directory} still exists"
assert path.exists(top_level_directory), (
f"Directory {top_level_directory} does not exist " f"after non-recurisve removal"
)
bash.remove_directories([top_level_directory], require_absolute_path=False)
assert not path.exists(
top_level_directory
), f"Directory {top_level_directory} still exists after removal"
def test_manifest_invalid_extension():
backup_name = "2020-12-16_19-33-03-982"
_make_backup(backup_name, "_SUCCEEDED", manifest_extension="txt")
valid = backup.validate_backup(TEMP_DIR / backup_name)
assert not valid
backups = [TEMP_DIR / backup_name]
bash.remove_directories(backups, allow_recursive_deletes=True)
def test_invalid_date():
backup_name = "2020-07_12_04_489"
_make_backup(backup_name, "_SUCCEEDED")
valid = backup.validate_backup(TEMP_DIR / backup_name)
assert not valid
backups = [TEMP_DIR / backup_name]
bash.remove_directories(backups, allow_recursive_deletes=True)
def test_manifest_invalid_extension():
backup_name = "2020-12-16_19-33-03-982"
with tempfile.TemporaryDirectory() as tempdir:
_make_backup(backup_name, "_SUCCEEDED", Path(tempdir), manifest_extension="txt")
valid = backup.validate_backup(Path(tempdir) / backup_name)
assert not valid
backups = [Path(tempdir) / backup_name]
bash.remove_directories(backups, allow_recursive_deletes=True)
def test_invalid_date():
backup_name = "2020-07_12_04_489"
with tempfile.TemporaryDirectory() as tempdir:
_make_backup(backup_name, "_SUCCEEDED", Path(tempdir))
valid = backup.validate_backup(Path(tempdir) / backup_name)
assert not valid
backups = [Path(tempdir) / backup_name]
bash.remove_directories(backups, allow_recursive_deletes=True)
def test_failure_to_remove_directory_that_does_not_exist():
top_level_directory = TEMP_DIR / "test_failure_to_remove_directory_that_does_not_exist"
directories = [
top_level_directory / "test1",
]
for directory in directories:
assert not path.exists(directory), f"Directory {directory} already exists"
with pytest.raises(OSError):
bash.remove_directories(directories, failure_if_not_exists=True)
def test_failure_to_remove_directory_that_does_not_exist():
with tempfile.TemporaryDirectory() as tempdir:
top_level_directory = (
Path(tempdir) /
"test_failure_to_remove_directory_that_does_not_exist")
directories = [
top_level_directory / "test1",
]
for directory in directories:
assert not path.exists(
directory), f"Directory {directory} already exists"
with pytest.raises(OSError):
bash.remove_directories(directories, failure_if_not_exists=True)
def test_create_and_remove_directories_with_absolute_path():
with tempfile.TemporaryDirectory() as tempdir:
top_level_directory = (
Path(tempdir) /
"test_create_and_remove_directories_with_absolute_path")
directories = [
top_level_directory / "test1",
top_level_directory / "test2",
top_level_directory / "test3",
]
for directory in directories:
assert not path.exists(
directory), f"Directory {directory} already exists"
bash.create_directories(directories)
for directory in directories:
assert path.exists(
directory), f"Created directory {directory} does not exist"
bash.remove_directories(directories)
for directory in directories:
assert not path.exists(
directory), f"Removed directory {directory} still exists"
assert path.exists(top_level_directory), (
f"Directory {top_level_directory} does not exist "
f"after non-recurisve removal")
bash.remove_directories([top_level_directory])
assert not path.exists(
top_level_directory
), f"Directory {top_level_directory} still exists after removal"
def test_failure_to_remove_non_empty_directory():
with tempfile.TemporaryDirectory() as tempdir:
top_level_directory = Path(
tempdir) / "test_failure_to_remove_non_empty_directory"
directories = [
top_level_directory / "test1",
top_level_directory / "test2",
top_level_directory / "test3",
]
for directory in directories:
assert not path.exists(
directory), f"Directory {directory} already exists"
# Setup
bash.create_directories(directories)
for directory in directories:
assert path.exists(
directory), f"Created directory {directory} does not exist"
# The test to remove the top level directory first, which should fail
with pytest.raises(OSError):
bash.remove_directories([top_level_directory])
assert path.exists(
top_level_directory
), f"Directory {top_level_directory} removed even though not empty"
for directory in directories:
assert path.exists(
directory
), f"Directory {directory} removed due to top-level directory removal"
# Clean up
bash.remove_directories(directories)
bash.remove_directories([top_level_directory])
for directory in directories:
assert not path.exists(
directory), f"Removed directory {directory} still exists"
assert not path.exists(
top_level_directory
), f"Directory {top_level_directory} still exists after removal"
def test_failure_to_remove_directory_with_relative_path():
directories = ["test1" "test2" "test3"]
with pytest.raises(ValueError):
bash.remove_directories(directories)
