async def target_unmatched_snapshots(self, direction, source_datasets, target_dataset, transport, ssh_credentials): fake_replication_task = types.SimpleNamespace() fake_replication_task.source_datasets = source_datasets fake_replication_task.target_dataset = target_dataset datasets = { source_dataset: get_target_dataset(fake_replication_task, source_dataset) for source_dataset in source_datasets } try: local_shell = LocalShell() async with self._get_zettarepl_shell( transport, ssh_credentials) as remote_shell: if direction == "PUSH": source_shell = local_shell target_shell = remote_shell else: source_shell = remote_shell target_shell = local_shell target_datasets = set(list_datasets(target_shell)) datasets = { source_dataset: target_dataset for source_dataset, target_dataset in datasets.items() if target_dataset in target_datasets } source_snapshots = group_snapshots_by_datasets( await self.middleware.run_in_thread( multilist_snapshots, source_shell, [(dataset, False) for dataset in datasets.keys()])) target_snapshots = group_snapshots_by_datasets( await self.middleware.run_in_thread( multilist_snapshots, target_shell, [(dataset, False) for dataset in datasets.values()])) except Exception as e: raise CallError(repr(e)) errors = {} for source_dataset, target_dataset in datasets.items(): unmatched_snapshots = list( set(target_snapshots.get(target_dataset, [])) - set(source_snapshots.get(source_dataset, []))) if unmatched_snapshots: errors[target_dataset] = unmatched_snapshots return errors
def pending_push_replication_task_snapshot_owners( src_snapshots: {str: [str]}, shell: Shell, replication_tasks: [ReplicationTask]): replication_tasks = [ replication_task for replication_task in replication_tasks if replication_task.hold_pending_snapshots ] if replication_tasks: dst_snapshots_queries = replication_tasks_target_datasets_queries( replication_tasks) try: dst_snapshots = multilist_snapshots(shell, dst_snapshots_queries) except Exception as e: logger.error( "Failed to list snapshots with %r: %r. Assuming remote has no snapshots", shell, e) dst_snapshots = {} else: dst_snapshots = group_snapshots_by_datasets(dst_snapshots) return [ PendingPushReplicationTaskSnapshotOwner(replication_task, src_snapshots, dst_snapshots) for replication_task in replication_tasks ] return []
def calculate_snapshots_to_remove(owners: [SnapshotOwner], snapshots: [Snapshot]): result = [] for dataset, dataset_snapshots in group_snapshots_by_datasets( snapshots).items(): dataset_owners = [ owner for owner in owners if owner.owns_dataset(dataset) ] result.extend([ Snapshot(dataset, snapshot) for snapshot in calculate_dataset_snapshots_to_remove( dataset_owners, dataset, dataset_snapshots) ]) return result
def pending_push_replication_task_snapshot_owners(src_snapshots: {str: [str]}, shell: Shell, replication_tasks: [ReplicationTask]): replication_tasks = [replication_task for replication_task in replication_tasks if replication_task.hold_pending_snapshots] if replication_tasks: try: dst_snapshots = group_snapshots_by_datasets( multilist_snapshots(shell, [(replication_task.target_dataset, replication_task.recursive) for replication_task in replication_tasks]) ) except Exception: logger.error("Failed to list snapshots with shell %r, assuming remote has no snapshots", shell, exc_info=True) dst_snapshots = {} return [PendingPushReplicationTaskSnapshotOwner(replication_task, src_snapshots, dst_snapshots) for replication_task in replication_tasks] return []