def test_missing_files(): top_level_directory = TEMP_DIR / "2020-07-04_12_04_489" directories = [ top_level_directory / "config", top_level_directory / "database", top_level_directory / "elasticsearch", top_level_directory / "files", top_level_directory / "hbase" / "archive" / "data" / "tamr", top_level_directory / "files" / "dataset", top_level_directory / "files" / "dedup", top_level_directory / "files" / "job", top_level_directory / "files" / "procurify", ] bash.create_directories(directories) files = [ top_level_directory / "config" / "config.json", top_level_directory / "database" / "pg_dump.out", top_level_directory / "_SUCCEEDED", ] for file in files: with open(file, "w"): pass valid = backup.validate_backup(Path(top_level_directory)) assert not valid backups = [top_level_directory] bash.remove_directories(backups, allow_recursive_deletes=True)
def test_failure_to_create_directory_that_already_exists(): with tempfile.TemporaryDirectory() as tempdir: top_level_directory = ( Path(tempdir) / "test_failure_to_create_directory_that_already_exists") directories = [ top_level_directory / "test1", ] for directory in directories: assert not path.exists( directory), f"Directory {directory} already exists" # Setup bash.create_directories(directories) # Test that cannot create again with pytest.raises(OSError): bash.create_directories(directories, failure_if_exists=True) # Clean up bash.remove_directories(directories) bash.remove_directories([top_level_directory]) for directory in directories: assert not path.exists( directory), f"Removed directory {directory} still exists" assert not path.exists( top_level_directory ), f"Directory {top_level_directory} still exists after removal"
def _make_old_files(directory_path: str, num_days_old: int, temporary_directory: Path) -> Tuple[List, List]: """ Creates a simulated directory structure with files having modification times set to specified number of days in the past. The files are created in a temporary directory which must be supplied Args: directory_path: Path within temporary directory to create file structure num_days_old: number of days in the past to set the file modification times temporary_directory: The temporary directory to create backups within Returns: A tuple of two lists with the first containing the list of paths to the created directories and the second containing the list of paths to the created files """ top_level_directory = temporary_directory / directory_path directories = [top_level_directory] bash.create_directories(directories) files = [ top_level_directory / f"temp_{num_days_old}_days_1", top_level_directory / f"temp_{num_days_old}_days_2", ] # create a timestamp in the past mod_timestamp = (datetime.now() - timedelta(days=num_days_old)).timestamp() for file in files: with open(file, "w"): pass os.utime(file, (mod_timestamp, mod_timestamp)) return directories, files
def test_create_and_remove_directories_with_relative_path(): top_level_directory = TEMP_DIR / "test_create_and_remove_directories_with_relative_path" directories = [ top_level_directory / "test1", top_level_directory / "test2", top_level_directory / "test3", ] for directory in directories: assert not path.exists(directory), f"Directory {directory} already exists" bash.create_directories(directories, require_absolute_path=False) for directory in directories: assert path.exists(directory), f"Created directory {directory} does not exist" bash.remove_directories(directories, require_absolute_path=False) for directory in directories: assert not path.exists(directory), f"Removed directory {directory} still exists" assert path.exists(top_level_directory), ( f"Directory {top_level_directory} does not exist " f"after non-recurisve removal" ) bash.remove_directories([top_level_directory], require_absolute_path=False) assert not path.exists( top_level_directory ), f"Directory {top_level_directory} still exists after removal"
def _make_backup( backup_name: str, completion_status: str, temporary_directory: Path, manifest_extension: str = "json", ) -> Tuple[List, List]: """ Creates a simulated backup with the primary files in place necessary for identification. The files are created in a temporary directory that must be supplied Args: backup_name: The name of the backup to be used as the backup path completion_status: The status of the backup to put in the simulated manifest file temporary_directory: The temporary directory to create backups within manifest_extension: Either "json" or "yaml" Returns: A tuple of two lists with the first containing the list of paths to the created directories and the second containing the list of paths to the created files """ top_level_directory = temporary_directory / backup_name directories = [ top_level_directory / "config", top_level_directory / "database", top_level_directory / "elasticsearch", top_level_directory / "files", top_level_directory / "hbase" / "archive" / "data" / "tamr", top_level_directory / "files" / "dataset", top_level_directory / "files" / "dedup", top_level_directory / "files" / "job", top_level_directory / "files" / "procurify", ] bash.create_directories(directories) files = [ top_level_directory / f"manifest.{manifest_extension}", top_level_directory / "config" / "config.json", top_level_directory / "database" / "pg_dump.out", top_level_directory / completion_status, ] data = {"finished": backup_name, "errorMessage": None} for file in files: if file == (top_level_directory / "_SUCCEEDED"): with open(file, "w") as f: json.dump(data, f) else: with open(file, "w"): pass return directories, files
def test_failure_to_remove_non_empty_directory(): with tempfile.TemporaryDirectory() as tempdir: top_level_directory = Path( tempdir) / "test_failure_to_remove_non_empty_directory" directories = [ top_level_directory / "test1", top_level_directory / "test2", top_level_directory / "test3", ] for directory in directories: assert not path.exists( directory), f"Directory {directory} already exists" # Setup bash.create_directories(directories) for directory in directories: assert path.exists( directory), f"Created directory {directory} does not exist" # The test to remove the top level directory first, which should fail with pytest.raises(OSError): bash.remove_directories([top_level_directory]) assert path.exists( top_level_directory ), f"Directory {top_level_directory} removed even though not empty" for directory in directories: assert path.exists( directory ), f"Directory {directory} removed due to top-level directory removal" # Clean up bash.remove_directories(directories) bash.remove_directories([top_level_directory]) for directory in directories: assert not path.exists( directory), f"Removed directory {directory} still exists" assert not path.exists( top_level_directory ), f"Directory {top_level_directory} still exists after removal"
def test_create_and_remove_directories_with_absolute_path(): with tempfile.TemporaryDirectory() as tempdir: top_level_directory = ( Path(tempdir) / "test_create_and_remove_directories_with_absolute_path") directories = [ top_level_directory / "test1", top_level_directory / "test2", top_level_directory / "test3", ] for directory in directories: assert not path.exists( directory), f"Directory {directory} already exists" bash.create_directories(directories) for directory in directories: assert path.exists( directory), f"Created directory {directory} does not exist" bash.remove_directories(directories) for directory in directories: assert not path.exists( directory), f"Removed directory {directory} still exists" assert path.exists(top_level_directory), ( f"Directory {top_level_directory} does not exist " f"after non-recurisve removal") bash.remove_directories([top_level_directory]) assert not path.exists( top_level_directory ), f"Directory {top_level_directory} still exists after removal"
def test_failure_to_create_directory_with_relative_path(): directories = ["test1" "test2" "test3"] with pytest.raises(ValueError): bash.create_directories(directories)