def test_missing_files(): top_level_directory = TEMP_DIR / "2020-07-04_12_04_489" directories = [ top_level_directory / "config", top_level_directory / "database", top_level_directory / "elasticsearch", top_level_directory / "files", top_level_directory / "hbase" / "archive" / "data" / "tamr", top_level_directory / "files" / "dataset", top_level_directory / "files" / "dedup", top_level_directory / "files" / "job", top_level_directory / "files" / "procurify", ] bash.create_directories(directories) files = [ top_level_directory / "config" / "config.json", top_level_directory / "database" / "pg_dump.out", top_level_directory / "_SUCCEEDED", ] for file in files: with open(file, "w"): pass valid = backup.validate_backup(Path(top_level_directory)) assert not valid backups = [top_level_directory] bash.remove_directories(backups, allow_recursive_deletes=True)
def test_failure_to_create_directory_that_already_exists(): with tempfile.TemporaryDirectory() as tempdir: top_level_directory = ( Path(tempdir) / "test_failure_to_create_directory_that_already_exists") directories = [ top_level_directory / "test1", ] for directory in directories: assert not path.exists( directory), f"Directory {directory} already exists" # Setup bash.create_directories(directories) # Test that cannot create again with pytest.raises(OSError): bash.create_directories(directories, failure_if_exists=True) # Clean up bash.remove_directories(directories) bash.remove_directories([top_level_directory]) for directory in directories: assert not path.exists( directory), f"Removed directory {directory} still exists" assert not path.exists( top_level_directory ), f"Directory {top_level_directory} still exists after removal"
def delete_backups( *, backups: List[str], backup_directory: Union[Path, str], backup_datetime_format: str = "%Y-%m-%d_%H-%M-%S-%f", ) -> List[str]: """Deletes backup folders recursively. Args: backups: list of backups to delete backup_directory: Path to backup directory backup_datetime_format: String datetime format in backup folder name Returns: list of deleted backup names """ if not isinstance(backup_directory, Path): backup_directory = Path(backup_directory) backups_to_delete = sorted(backups, reverse=True) LOGGER.info( f"Deleting {len(backups_to_delete)} backup(s): '{backups_to_delete}'") deleted_backups = [] for backup in backups: backup_path = os.path.join(backup_directory, backup) if validate_backup(Path(backup_path), backup_datetime_format=backup_datetime_format): bash.remove_directories([backup_path], allow_recursive_deletes=True) deleted_backups.append(backup) else: LOGGER.error(f"{backup} is not a valid backup to delete.") return deleted_backups
def test_create_and_remove_directories_with_relative_path(): top_level_directory = TEMP_DIR / "test_create_and_remove_directories_with_relative_path" directories = [ top_level_directory / "test1", top_level_directory / "test2", top_level_directory / "test3", ] for directory in directories: assert not path.exists(directory), f"Directory {directory} already exists" bash.create_directories(directories, require_absolute_path=False) for directory in directories: assert path.exists(directory), f"Created directory {directory} does not exist" bash.remove_directories(directories, require_absolute_path=False) for directory in directories: assert not path.exists(directory), f"Removed directory {directory} still exists" assert path.exists(top_level_directory), ( f"Directory {top_level_directory} does not exist " f"after non-recurisve removal" ) bash.remove_directories([top_level_directory], require_absolute_path=False) assert not path.exists( top_level_directory ), f"Directory {top_level_directory} still exists after removal"
def test_manifest_invalid_extension(): backup_name = "2020-12-16_19-33-03-982" _make_backup(backup_name, "_SUCCEEDED", manifest_extension="txt") valid = backup.validate_backup(TEMP_DIR / backup_name) assert not valid backups = [TEMP_DIR / backup_name] bash.remove_directories(backups, allow_recursive_deletes=True)
def test_invalid_date(): backup_name = "2020-07_12_04_489" _make_backup(backup_name, "_SUCCEEDED") valid = backup.validate_backup(TEMP_DIR / backup_name) assert not valid backups = [TEMP_DIR / backup_name] bash.remove_directories(backups, allow_recursive_deletes=True)
def test_manifest_invalid_extension(): backup_name = "2020-12-16_19-33-03-982" with tempfile.TemporaryDirectory() as tempdir: _make_backup(backup_name, "_SUCCEEDED", Path(tempdir), manifest_extension="txt") valid = backup.validate_backup(Path(tempdir) / backup_name) assert not valid backups = [Path(tempdir) / backup_name] bash.remove_directories(backups, allow_recursive_deletes=True)
def test_invalid_date(): backup_name = "2020-07_12_04_489" with tempfile.TemporaryDirectory() as tempdir: _make_backup(backup_name, "_SUCCEEDED", Path(tempdir)) valid = backup.validate_backup(Path(tempdir) / backup_name) assert not valid backups = [Path(tempdir) / backup_name] bash.remove_directories(backups, allow_recursive_deletes=True)
def test_failure_to_remove_directory_that_does_not_exist(): top_level_directory = TEMP_DIR / "test_failure_to_remove_directory_that_does_not_exist" directories = [ top_level_directory / "test1", ] for directory in directories: assert not path.exists(directory), f"Directory {directory} already exists" with pytest.raises(OSError): bash.remove_directories(directories, failure_if_not_exists=True)
def test_failure_to_remove_directory_that_does_not_exist(): with tempfile.TemporaryDirectory() as tempdir: top_level_directory = ( Path(tempdir) / "test_failure_to_remove_directory_that_does_not_exist") directories = [ top_level_directory / "test1", ] for directory in directories: assert not path.exists( directory), f"Directory {directory} already exists" with pytest.raises(OSError): bash.remove_directories(directories, failure_if_not_exists=True)
def test_create_and_remove_directories_with_absolute_path(): with tempfile.TemporaryDirectory() as tempdir: top_level_directory = ( Path(tempdir) / "test_create_and_remove_directories_with_absolute_path") directories = [ top_level_directory / "test1", top_level_directory / "test2", top_level_directory / "test3", ] for directory in directories: assert not path.exists( directory), f"Directory {directory} already exists" bash.create_directories(directories) for directory in directories: assert path.exists( directory), f"Created directory {directory} does not exist" bash.remove_directories(directories) for directory in directories: assert not path.exists( directory), f"Removed directory {directory} still exists" assert path.exists(top_level_directory), ( f"Directory {top_level_directory} does not exist " f"after non-recurisve removal") bash.remove_directories([top_level_directory]) assert not path.exists( top_level_directory ), f"Directory {top_level_directory} still exists after removal"
def test_failure_to_remove_non_empty_directory(): with tempfile.TemporaryDirectory() as tempdir: top_level_directory = Path( tempdir) / "test_failure_to_remove_non_empty_directory" directories = [ top_level_directory / "test1", top_level_directory / "test2", top_level_directory / "test3", ] for directory in directories: assert not path.exists( directory), f"Directory {directory} already exists" # Setup bash.create_directories(directories) for directory in directories: assert path.exists( directory), f"Created directory {directory} does not exist" # The test to remove the top level directory first, which should fail with pytest.raises(OSError): bash.remove_directories([top_level_directory]) assert path.exists( top_level_directory ), f"Directory {top_level_directory} removed even though not empty" for directory in directories: assert path.exists( directory ), f"Directory {directory} removed due to top-level directory removal" # Clean up bash.remove_directories(directories) bash.remove_directories([top_level_directory]) for directory in directories: assert not path.exists( directory), f"Removed directory {directory} still exists" assert not path.exists( top_level_directory ), f"Directory {top_level_directory} still exists after removal"
def test_failure_to_remove_directory_with_relative_path(): directories = ["test1" "test2" "test3"] with pytest.raises(ValueError): bash.remove_directories(directories)