def test_delete_old_files_exclude():
base_path = TEMP_DIR / "old"
data_path = TEMP_DIR / "old" / "data"
_make_old_files(data_path, 10)
result = bash.delete_old_files(base_path, num_days_to_keep=0, exclude_paths=[data_path])
assert len(result) == 0
result = bash.delete_old_files(base_path, num_days_to_keep=0)
assert len(result) == 2
def test_delete_old_files_exclude():
with tempfile.TemporaryDirectory() as tempdir:
base_path = Path(tempdir) / "old"
data_path = Path(tempdir) / "old" / "data"
_make_old_files(data_path, 10, tempdir)
result = bash.delete_old_files(base_path,
num_days_to_keep=0,
exclude_paths=[data_path])
assert len(result) == 0
result = bash.delete_old_files(base_path, num_days_to_keep=0)
assert len(result) == 2
def test_delete_old_files():
base_path = TEMP_DIR / "old"
file_ages_days = [1, 10, 20]
for age in file_ages_days:
_make_old_files(base_path, age)
result = bash.delete_old_files(base_path, num_days_to_keep=30)
assert len(result) == 0
result = bash.delete_old_files(base_path, num_days_to_keep=14)
assert len(result) == 2
result = bash.delete_old_files(base_path, num_days_to_keep=0)
assert len(result) == 4
def test_delete_old_files():
with tempfile.TemporaryDirectory() as tempdir:
base_path = Path(tempdir) / "old"
file_ages_days = [1, 10, 20]
for age in file_ages_days:
_make_old_files(base_path, age, tempdir)
result = bash.delete_old_files(base_path, num_days_to_keep=30)
assert len(result) == 0
result = bash.delete_old_files(base_path, num_days_to_keep=14)
assert len(result) == 2
result = bash.delete_old_files(base_path, num_days_to_keep=0)
assert len(result) == 4
def delete_old_spark_event_logs(tamr_home_directory: Union[Path, str],
*,
num_days_to_keep: int = 14) -> List[str]:
"""
Deletes sparkEventLogs older than the specified number of days.
This assumes that Spark is running locally on the same VM as Tamr
and that the logs are on the local filesystem.
Args:
tamr_home_directory: Path to the Tamr home directory
num_days_to_keep: Number of days for which to keep logs
Returns:
A list of deleted sparkEventLogs files
Raises:
ValueError: if num_days_to_keep is less than 0
FileNotFoundError: if sparkEventLogs directory doesn't exist
"""
spark_event_log_directory = os.path.join(
tamr_home_directory, "tamr/unify-data/job/sparkEventLogs")
if not os.path.exists(spark_event_log_directory):
message = f"directory does not exist: {spark_event_log_directory}"
LOGGER.error(message)
raise FileNotFoundError(message)
return bash.delete_old_files(spark_event_log_directory,
num_days_to_keep=num_days_to_keep)
def test_delete_old_files_recursive():
base_path = TEMP_DIR / "old"
data_path = TEMP_DIR / "old" / "data"
_make_old_files(data_path, 10)
result = bash.delete_old_files(base_path, num_days_to_keep=0)
assert len(result) == 2
def test_delete_old_files_invalid_path():
with tempfile.TemporaryDirectory() as tempdir:
with pytest.raises(FileNotFoundError):
bash.delete_old_files(Path(tempdir) / "fake_path",
num_days_to_keep=14)
def test_delete_old_files_invalid_num_days():
with tempfile.TemporaryDirectory() as tempdir:
with pytest.raises(ValueError):
bash.delete_old_files(Path(tempdir), num_days_to_keep=-1)
def test_delete_old_files_invalid_path():
with pytest.raises(FileNotFoundError):
bash.delete_old_files(TEMP_DIR / "fake_path", num_days_to_keep=14)
def test_delete_old_files_invalid_num_days():
with pytest.raises(ValueError):
bash.delete_old_files(TEMP_DIR, num_days_to_keep=-1)
