def test_missing_files():
top_level_directory = TEMP_DIR / "2020-07-04_12_04_489"
directories = [
top_level_directory / "config",
top_level_directory / "database",
top_level_directory / "elasticsearch",
top_level_directory / "files",
top_level_directory / "hbase" / "archive" / "data" / "tamr",
top_level_directory / "files" / "dataset",
top_level_directory / "files" / "dedup",
top_level_directory / "files" / "job",
top_level_directory / "files" / "procurify",
]
bash.create_directories(directories)
files = [
top_level_directory / "config" / "config.json",
top_level_directory / "database" / "pg_dump.out",
top_level_directory / "_SUCCEEDED",
]
for file in files:
with open(file, "w"):
pass
valid = backup.validate_backup(Path(top_level_directory))
assert not valid
backups = [top_level_directory]
bash.remove_directories(backups, allow_recursive_deletes=True)
def test_failure_to_create_directory_that_already_exists():
with tempfile.TemporaryDirectory() as tempdir:
top_level_directory = (
Path(tempdir) /
"test_failure_to_create_directory_that_already_exists")
directories = [
top_level_directory / "test1",
]
for directory in directories:
assert not path.exists(
directory), f"Directory {directory} already exists"
# Setup
bash.create_directories(directories)
# Test that cannot create again
with pytest.raises(OSError):
bash.create_directories(directories, failure_if_exists=True)
# Clean up
bash.remove_directories(directories)
bash.remove_directories([top_level_directory])
for directory in directories:
assert not path.exists(
directory), f"Removed directory {directory} still exists"
assert not path.exists(
top_level_directory
), f"Directory {top_level_directory} still exists after removal"
def _make_old_files(directory_path: str, num_days_old: int,
temporary_directory: Path) -> Tuple[List, List]:
"""
Creates a simulated directory structure
with files having modification times set to specified number of days in the past.
The files are created in a temporary directory which must be supplied
Args:
directory_path: Path within temporary directory to create file structure
num_days_old: number of days in the past to set the file modification times
temporary_directory: The temporary directory to create backups within
Returns:
A tuple of two lists with the first containing the list of paths to the created directories
and the second containing the list of paths to the created files
"""
top_level_directory = temporary_directory / directory_path
directories = [top_level_directory]
bash.create_directories(directories)
files = [
top_level_directory / f"temp_{num_days_old}_days_1",
top_level_directory / f"temp_{num_days_old}_days_2",
]
# create a timestamp in the past
mod_timestamp = (datetime.now() - timedelta(days=num_days_old)).timestamp()
for file in files:
with open(file, "w"):
pass
os.utime(file, (mod_timestamp, mod_timestamp))
return directories, files
def test_create_and_remove_directories_with_relative_path():
top_level_directory = TEMP_DIR / "test_create_and_remove_directories_with_relative_path"
directories = [
top_level_directory / "test1",
top_level_directory / "test2",
top_level_directory / "test3",
]
for directory in directories:
assert not path.exists(directory), f"Directory {directory} already exists"
bash.create_directories(directories, require_absolute_path=False)
for directory in directories:
assert path.exists(directory), f"Created directory {directory} does not exist"
bash.remove_directories(directories, require_absolute_path=False)
for directory in directories:
assert not path.exists(directory), f"Removed directory {directory} still exists"
assert path.exists(top_level_directory), (
f"Directory {top_level_directory} does not exist " f"after non-recurisve removal"
)
bash.remove_directories([top_level_directory], require_absolute_path=False)
assert not path.exists(
top_level_directory
), f"Directory {top_level_directory} still exists after removal"
def _make_backup(
backup_name: str,
completion_status: str,
temporary_directory: Path,
manifest_extension: str = "json",
) -> Tuple[List, List]:
"""
Creates a simulated backup with the primary files in place necessary for identification.
The files are created in a temporary directory that must be supplied
Args:
backup_name: The name of the backup to be used as the backup path
completion_status: The status of the backup to put in the simulated manifest file
temporary_directory: The temporary directory to create backups within
manifest_extension: Either "json" or "yaml"
Returns:
A tuple of two lists with the first containing the list of paths to the created directories
and the second containing the list of paths to the created files
"""
top_level_directory = temporary_directory / backup_name
directories = [
top_level_directory / "config",
top_level_directory / "database",
top_level_directory / "elasticsearch",
top_level_directory / "files",
top_level_directory / "hbase" / "archive" / "data" / "tamr",
top_level_directory / "files" / "dataset",
top_level_directory / "files" / "dedup",
top_level_directory / "files" / "job",
top_level_directory / "files" / "procurify",
]
bash.create_directories(directories)
files = [
top_level_directory / f"manifest.{manifest_extension}",
top_level_directory / "config" / "config.json",
top_level_directory / "database" / "pg_dump.out",
top_level_directory / completion_status,
]
data = {"finished": backup_name, "errorMessage": None}
for file in files:
if file == (top_level_directory / "_SUCCEEDED"):
with open(file, "w") as f:
json.dump(data, f)
else:
with open(file, "w"):
pass
return directories, files
def test_failure_to_remove_non_empty_directory():
with tempfile.TemporaryDirectory() as tempdir:
top_level_directory = Path(
tempdir) / "test_failure_to_remove_non_empty_directory"
directories = [
top_level_directory / "test1",
top_level_directory / "test2",
top_level_directory / "test3",
]
for directory in directories:
assert not path.exists(
directory), f"Directory {directory} already exists"
# Setup
bash.create_directories(directories)
for directory in directories:
assert path.exists(
directory), f"Created directory {directory} does not exist"
# The test to remove the top level directory first, which should fail
with pytest.raises(OSError):
bash.remove_directories([top_level_directory])
assert path.exists(
top_level_directory
), f"Directory {top_level_directory} removed even though not empty"
for directory in directories:
assert path.exists(
directory
), f"Directory {directory} removed due to top-level directory removal"
# Clean up
bash.remove_directories(directories)
bash.remove_directories([top_level_directory])
for directory in directories:
assert not path.exists(
directory), f"Removed directory {directory} still exists"
assert not path.exists(
top_level_directory
), f"Directory {top_level_directory} still exists after removal"
def test_create_and_remove_directories_with_absolute_path():
with tempfile.TemporaryDirectory() as tempdir:
top_level_directory = (
Path(tempdir) /
"test_create_and_remove_directories_with_absolute_path")
directories = [
top_level_directory / "test1",
top_level_directory / "test2",
top_level_directory / "test3",
]
for directory in directories:
assert not path.exists(
directory), f"Directory {directory} already exists"
bash.create_directories(directories)
for directory in directories:
assert path.exists(
directory), f"Created directory {directory} does not exist"
bash.remove_directories(directories)
for directory in directories:
assert not path.exists(
directory), f"Removed directory {directory} still exists"
assert path.exists(top_level_directory), (
f"Directory {top_level_directory} does not exist "
f"after non-recurisve removal")
bash.remove_directories([top_level_directory])
assert not path.exists(
top_level_directory
), f"Directory {top_level_directory} still exists after removal"
def test_failure_to_create_directory_with_relative_path():
directories = ["test1" "test2" "test3"]
with pytest.raises(ValueError):
bash.create_directories(directories)
