def periodic_snapshot_task_snapshots(self, task):
snapshots = list_snapshots(LocalShell(), task["dataset"], task["recursive"])
zettarepl_task = PeriodicSnapshotTask.from_data(None, self.middleware.call_sync(
"zettarepl.periodic_snapshot_task_definition", task,
))
snapshot_owner = PeriodicSnapshotTaskSnapshotOwner(datetime.utcnow(), zettarepl_task)
task_snapshots = set()
for snapshot in snapshots:
if snapshot_owner.owns_dataset(snapshot.dataset):
try:
parsed_snapshot_name = parse_snapshot_name(snapshot.name, task["naming_schema"])
except ValueError:
pass
else:
if snapshot_owner.owns_snapshot(snapshot.dataset, parsed_snapshot_name):
task_snapshots.add(str(snapshot))
return task_snapshots
def test__calculate_snapshots_to_remove():
assert calculate_snapshots_to_remove([
PeriodicSnapshotTaskSnapshotOwner(
datetime(2019, 5, 30, 21, 52),
Mock(dataset="dst/work",
recursive=False,
exclude=[],
lifetime=timedelta(days=14),
naming_schema="auto-%Y-%m-%d_%H-%M")),
PeriodicSnapshotTaskSnapshotOwner(
datetime(2019, 5, 30, 21, 52),
Mock(dataset="dst/work",
recursive=False,
exclude=[],
lifetime=timedelta(hours=1),
naming_schema="snap%d%m%Y%H%M")),
], [Snapshot("dst/work", "snap300520191856")]) == [
Snapshot("dst/work", "snap300520191856")
]
def _run_local_retention(self, now: datetime):
periodic_snapshot_tasks = select_by_class(PeriodicSnapshotTask,
self.tasks)
replication_tasks = select_by_class(ReplicationTask, self.tasks)
push_replication_tasks_that_can_hold = [
replication_task for replication_task in replication_tasks
if replication_task.hold_pending_snapshots
]
pull_replications_tasks = list(
filter(self._is_pull_replication_task, replication_tasks))
local_snapshots_queries = []
local_snapshots_queries.extend([
(periodic_snapshot_task.dataset, periodic_snapshot_task.recursive)
for periodic_snapshot_task in periodic_snapshot_tasks
])
local_snapshots_queries.extend(
replication_tasks_source_datasets_queries(
push_replication_tasks_that_can_hold))
local_snapshots_queries.extend([
(replication_task.target_dataset, replication_task.recursive)
for replication_task in pull_replications_tasks
])
local_snapshots = multilist_snapshots(self.local_shell,
local_snapshots_queries)
local_snapshots_grouped = group_snapshots_by_datasets(local_snapshots)
owners = []
owners.extend([
PeriodicSnapshotTaskSnapshotOwner(now, periodic_snapshot_task)
for periodic_snapshot_task in periodic_snapshot_tasks
])
# These are always only PUSH replication tasks
for transport, replication_tasks in self._transport_for_replication_tasks(
push_replication_tasks_that_can_hold):
shell = self._get_retention_shell(transport)
owners.extend(
pending_push_replication_task_snapshot_owners(
local_snapshots_grouped, shell, replication_tasks))
for transport, replication_tasks in self._transport_for_replication_tasks(
pull_replications_tasks):
shell = self._get_retention_shell(transport)
remote_snapshots_queries = replication_tasks_source_datasets_queries(
replication_tasks)
try:
remote_snapshots = multilist_snapshots(
shell, remote_snapshots_queries)
except Exception as e:
logger.warning(
"Local retention failed: error listing snapshots on %r: %r",
transport, e)
return
remote_snapshots_grouped = group_snapshots_by_datasets(
remote_snapshots)
owners.extend([
executed_pull_replication_task_snapshot_owner(
now, replication_task, remote_snapshots_grouped,
local_snapshots_grouped)
for replication_task in replication_tasks
])
snapshots_to_destroy = calculate_snapshots_to_remove(
owners, local_snapshots)
logger.info("Retention destroying local snapshots: %r",
snapshots_to_destroy)
destroy_snapshots(self.local_shell, snapshots_to_destroy)
def annotate_snapshots(self, snapshots):
property_name = self.middleware.call_sync("pool.snapshottask.removal_date_property")
zettarepl_tasks = [
PeriodicSnapshotTask.from_data(task["id"], self.middleware.call_sync(
"zettarepl.periodic_snapshot_task_definition", task,
))
for task in self.middleware.call_sync("pool.snapshottask.query", [["enabled", "=", True]])
]
snapshot_owners = [
PeriodicSnapshotTaskSnapshotOwner(datetime.utcnow(), zettarepl_task)
for zettarepl_task in zettarepl_tasks
]
for snapshot in snapshots:
task_destroy_at = None
task_destroy_at_id = None
for snapshot_owner in snapshot_owners:
if snapshot_owner.owns_dataset(snapshot["dataset"]):
try:
parsed_snapshot_name = parse_snapshot_name(
snapshot["snapshot_name"], snapshot_owner.periodic_snapshot_task.naming_schema
)
except ValueError:
pass
else:
if snapshot_owner.owns_snapshot(snapshot["dataset"], parsed_snapshot_name):
destroy_at = parsed_snapshot_name.datetime + snapshot_owner.periodic_snapshot_task.lifetime
if task_destroy_at is None or task_destroy_at < destroy_at:
task_destroy_at = destroy_at
task_destroy_at_id = snapshot_owner.periodic_snapshot_task.id
property_destroy_at = None
if property_name in snapshot["properties"]:
try:
property_destroy_at = isodate.parse_datetime(snapshot["properties"][property_name]["value"])
except Exception as e:
self.middleware.logger.warning("Error parsing snapshot %r %s: %r", snapshot["name"], property_name,
e)
if task_destroy_at is not None and property_destroy_at is not None:
if task_destroy_at < property_destroy_at:
task_destroy_at = None
else:
property_destroy_at = None
if task_destroy_at is not None:
snapshot["retention"] = {
"datetime": task_destroy_at,
"source": "periodic_snapshot_task",
"periodic_snapshot_task_id": task_destroy_at_id,
}
elif property_destroy_at is not None:
snapshot["retention"] = {
"datetime": property_destroy_at,
"source": "property",
}
else:
snapshot["retention"] = None
return snapshots