def test_pull_replication(): subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs destroy -r data/dst", shell=True) subprocess.check_call("zfs create data/src", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_02-00", shell=True) subprocess.check_call("zfs create data/dst", shell=True) definition = Definition.from_data( yaml.load( textwrap.dedent("""\ timezone: "UTC" replication-tasks: - id: src direction: pull transport: type: local source-dataset: data/src target-dataset: data/dst recursive: true naming-schema: - "%Y-%m-%d_%H-%M" auto: true retention-policy: none """))) local_shell = LocalShell() zettarepl = Zettarepl(Mock(), local_shell) zettarepl.set_tasks(definition.tasks) zettarepl._run_replication_tasks( select_by_class(ReplicationTask, definition.tasks)) assert len(list_snapshots(local_shell, "data/dst", False)) == 2
class ZettareplProcess: def __init__(self, definition, debug_level, log_handler, command_queue, observer_queue): self.definition = definition self.debug_level = debug_level self.log_handler = log_handler self.command_queue = command_queue self.observer_queue = observer_queue self.zettarepl = None self.vmware_contexts = {} def __call__(self): setproctitle.setproctitle('middlewared (zettarepl)') osc.die_with_parent() if logging.getLevelName(self.debug_level) == logging.TRACE: # If we want TRACE then we want all debug from zettarepl default_level = logging.DEBUG elif logging.getLevelName(self.debug_level) == logging.DEBUG: # Regular development level. We don't need verbose debug from zettarepl default_level = logging.INFO else: default_level = logging.getLevelName(self.debug_level) setup_logging("", "DEBUG", self.log_handler) oqlh = ObserverQueueLoggingHandler(self.observer_queue) oqlh.setFormatter( logging.Formatter( '[%(asctime)s] %(levelname)-8s [%(threadName)s] [%(name)s] %(message)s', '%Y/%m/%d %H:%M:%S')) logging.getLogger("zettarepl").addHandler(oqlh) for handler in logging.getLogger("zettarepl").handlers: handler.addFilter(LongStringsFilter()) handler.addFilter(ReplicationTaskLoggingLevelFilter(default_level)) c = Client('ws+unix:///var/run/middlewared-internal.sock', py_exceptions=True) c.subscribe('core.reconfigure_logging', lambda *args, **kwargs: reconfigure_logging()) definition = Definition.from_data(self.definition, raise_on_error=False) self.observer_queue.put(DefinitionErrors(definition.errors)) clock = Clock() tz_clock = TzClock(definition.timezone, clock.now) scheduler = Scheduler(clock, tz_clock) local_shell = LocalShell() self.zettarepl = Zettarepl(scheduler, local_shell) self.zettarepl.set_observer(self._observer) self.zettarepl.set_tasks(definition.tasks) start_daemon_thread(target=self._process_command_queue) while True: try: self.zettarepl.run() except Exception: logging.getLogger("zettarepl").error("Unhandled exception", exc_info=True) time.sleep(10) def _observer(self, message): self.observer_queue.put(message) logger = logging.getLogger("middlewared.plugins.zettarepl") try: if isinstance( message, (PeriodicSnapshotTaskStart, PeriodicSnapshotTaskSuccess, PeriodicSnapshotTaskError)): task_id = int(message.task_id.split("_")[-1]) if isinstance(message, PeriodicSnapshotTaskStart): with Client() as c: context = c.call("vmware.periodic_snapshot_task_begin", task_id, job=True) self.vmware_contexts[task_id] = context if context and context["vmsynced"]: # If there were no failures and we successfully took some VMWare snapshots # set the ZFS property to show the snapshot has consistent VM snapshots # inside it. return message.response( properties={"freenas:vmsynced": "Y"}) if isinstance( message, (PeriodicSnapshotTaskSuccess, PeriodicSnapshotTaskError)): context = self.vmware_contexts.pop(task_id, None) if context: with Client() as c: c.call("vmware.periodic_snapshot_task_end", context, job=True) except ClientException as e: if e.error: logger.error( "Unhandled exception in ZettareplProcess._observer: %r", e.error) if e.trace: logger.error( "Unhandled exception in ZettareplProcess._observer:\n%s", e.trace["formatted"]) except Exception: logger.error("Unhandled exception in ZettareplProcess._observer", exc_info=True) def _process_command_queue(self): logger = logging.getLogger("middlewared.plugins.zettarepl") while self.zettarepl is not None: command, args = self.command_queue.get() if command == "timezone": self.zettarepl.scheduler.tz_clock.timezone = pytz.timezone( args) if command == "tasks": definition = Definition.from_data(args, raise_on_error=False) self.observer_queue.put(DefinitionErrors(definition.errors)) self.zettarepl.set_tasks(definition.tasks) if command == "run_task": class_name, task_id = args for task in self.zettarepl.tasks: if task.__class__.__name__ == class_name and task.id == task_id: logger.debug("Running task %r", task) self.zettarepl.scheduler.interrupt([task]) break else: logger.warning("Task %s(%r) not found", class_name, task_id) self.observer_queue.put( ReplicationTaskError(task_id, "Task not found"))
def test_push_replication(dst_parent_is_readonly, dst_exists, transport, properties, compression): if transport["type"] != "ssh" and compression: return subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs destroy -r data/dst_parent", shell=True) subprocess.check_call("zfs create data/src", shell=True) subprocess.check_call("zfs set test:property=test-value data/src", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_02-00", shell=True) subprocess.check_call("zfs create data/dst_parent", shell=True) if dst_exists: subprocess.check_call("zfs create data/dst_parent/dst", shell=True) if dst_parent_is_readonly: subprocess.check_call("zfs set readonly=on data/dst_parent", shell=True) definition = yaml.safe_load( textwrap.dedent("""\ timezone: "UTC" periodic-snapshot-tasks: src: dataset: data/src recursive: true lifetime: PT1H naming-schema: "%Y-%m-%d_%H-%M" schedule: minute: "0" replication-tasks: src: direction: push source-dataset: data/src target-dataset: data/dst_parent/dst recursive: true periodic-snapshot-tasks: - src auto: true retention-policy: none retries: 1 """)) definition["replication-tasks"]["src"]["transport"] = transport definition["replication-tasks"]["src"]["properties"] = properties if compression: definition["replication-tasks"]["src"]["compression"] = compression definition = Definition.from_data(definition) local_shell = LocalShell() zettarepl = Zettarepl(Mock(), local_shell) zettarepl._spawn_retention = Mock() observer = Mock() zettarepl.set_observer(observer) zettarepl.set_tasks(definition.tasks) zettarepl._spawn_replication_tasks( select_by_class(ReplicationTask, definition.tasks)) wait_replication_tasks_to_complete(zettarepl) error = observer.call_args_list[-1][0][0] assert isinstance(error, ReplicationTaskSuccess), error assert len(list_snapshots(local_shell, "data/dst_parent/dst", False)) == 2 assert (("test-value" in subprocess.check_output( "zfs get test:property data/dst_parent/dst", shell=True, encoding="utf-8")) == properties) subprocess.check_call("zfs snapshot data/src@2018-10-01_03-00", shell=True) zettarepl._spawn_replication_tasks( select_by_class(ReplicationTask, definition.tasks)) wait_replication_tasks_to_complete(zettarepl) error = observer.call_args_list[-1][0][0] assert isinstance(error, ReplicationTaskSuccess), error assert len(list_snapshots(local_shell, "data/dst_parent/dst", False)) == 3
class ZettareplProcess: def __init__(self, definition, debug_level, log_handler, command_queue, observer_queue): self.definition = definition self.debug_level = debug_level self.log_handler = log_handler self.command_queue = command_queue self.observer_queue = observer_queue self.zettarepl = None self.vmware_contexts = {} def __call__(self): setproctitle.setproctitle('middlewared (zettarepl)') start_daemon_thread(target=watch_parent) if logging.getLevelName(self.debug_level) == logging.TRACE: # If we want TRACE then we want all debug from zettarepl debug_level = "DEBUG" elif logging.getLevelName(self.debug_level) == logging.DEBUG: # Regular development level. We don't need verbose debug from zettarepl debug_level = "INFO" else: debug_level = self.debug_level setup_logging("", debug_level, self.log_handler) for handler in logging.getLogger().handlers: handler.addFilter(LongStringsFilter()) definition = Definition.from_data(self.definition) clock = Clock() tz_clock = TzClock(definition.timezone, clock.now) scheduler = Scheduler(clock, tz_clock) local_shell = LocalShell() self.zettarepl = Zettarepl(scheduler, local_shell) self.zettarepl.set_observer(self._observer) self.zettarepl.set_tasks(definition.tasks) start_daemon_thread(target=self._process_command_queue) while True: try: self.zettarepl.run() except Exception: logging.getLogger("zettarepl").error("Unhandled exception", exc_info=True) time.sleep(10) def _observer(self, message): self.observer_queue.put(message) logger = logging.getLogger("middlewared.plugins.zettarepl") try: if isinstance( message, (PeriodicSnapshotTaskStart, PeriodicSnapshotTaskSuccess, PeriodicSnapshotTaskError)): task_id = int(message.task_id.split("_")[-1]) if isinstance(message, PeriodicSnapshotTaskStart): with Client(py_exceptions=True) as c: context = c.call("vmware.periodic_snapshot_task_begin", task_id) self.vmware_contexts[task_id] = context if context and context["vmsynced"]: # If there were no failures and we successfully took some VMWare snapshots # set the ZFS property to show the snapshot has consistent VM snapshots # inside it. return message.response( properties={"freenas:vmsynced": "Y"}) if isinstance( message, (PeriodicSnapshotTaskSuccess, PeriodicSnapshotTaskError)): context = self.vmware_contexts.pop(task_id, None) if context: with Client(py_exceptions=True) as c: c.call("vmware.periodic_snapshot_task_end", context) except Exception: logger.error("Unhandled exception in ZettareplProcess._observer", exc_info=True) def _process_command_queue(self): logger = logging.getLogger("middlewared.plugins.zettarepl") while self.zettarepl is not None: command, args = self.command_queue.get() if command == "timezone": self.zettarepl.scheduler.tz_clock.timezone = pytz.timezone( args) if command == "tasks": self.zettarepl.set_tasks(Definition.from_data(args).tasks) if command == "run_task": class_name, task_id = args for task in self.zettarepl.tasks: if task.__class__.__name__ == class_name and task.id == task_id: logger.debug("Running task %r", task) self.zettarepl.scheduler.interrupt([task]) break else: logger.warning("Task %s(%r) not found", class_name, task_id)
def test_snapshot_gone(transport): subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs destroy -r data/dst", shell=True) subprocess.check_call("zfs create data/src", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_02-00", shell=True) definition = yaml.safe_load( textwrap.dedent("""\ timezone: "UTC" periodic-snapshot-tasks: src: dataset: data/src recursive: true lifetime: PT1H naming-schema: "%Y-%m-%d_%H-%M" schedule: minute: "0" replication-tasks: src: direction: push source-dataset: data/src target-dataset: data/dst recursive: true periodic-snapshot-tasks: - src auto: true retention-policy: none retries: 2 """)) definition["replication-tasks"]["src"]["transport"] = transport definition = Definition.from_data(definition) local_shell = LocalShell() zettarepl = Zettarepl(Mock(), local_shell) zettarepl._spawn_retention = Mock() observer = Mock() zettarepl.set_observer(observer) zettarepl.set_tasks(definition.tasks) deleted = False def resume_replications_mock(*args, **kwargs): nonlocal deleted if not deleted: # Snapshots are already listed, and now we remove one of them to simulate PULL replication # from remote system that has `allow_empty_snapshots: false`. Only do this once. subprocess.check_call("zfs destroy data/src@2018-10-01_01-00", shell=True) deleted = True return resume_replications(*args, **kwargs) with patch("zettarepl.replication.run.resume_replications", resume_replications_mock): zettarepl._spawn_replication_tasks( select_by_class(ReplicationTask, definition.tasks)) wait_replication_tasks_to_complete(zettarepl) error = observer.call_args_list[-1][0][0] assert isinstance(error, ReplicationTaskSuccess), error assert len(list_snapshots(local_shell, "data/dst", False)) == 1
def test_parallel_replication(): subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs receive -A data/dst", shell=True) subprocess.call("zfs destroy -r data/dst", shell=True) subprocess.check_call("zfs create data/src", shell=True) subprocess.check_call("zfs create data/src/a", shell=True) subprocess.check_call( "dd if=/dev/urandom of=/mnt/data/src/a/blob bs=1M count=1", shell=True) subprocess.check_call("zfs snapshot data/src/a@2018-10-01_01-00", shell=True) subprocess.check_call("zfs create data/src/b", shell=True) subprocess.check_call( "dd if=/dev/urandom of=/mnt/data/src/b/blob bs=1M count=1", shell=True) subprocess.check_call("zfs snapshot data/src/b@2018-10-01_01-00", shell=True) subprocess.check_call("zfs create data/dst", shell=True) subprocess.check_call("zfs create data/dst/a", shell=True) subprocess.check_call("zfs create data/dst/b", shell=True) definition = Definition.from_data( yaml.load( textwrap.dedent("""\ timezone: "UTC" periodic-snapshot-tasks: src-a: dataset: data/src/a recursive: true lifetime: PT1H naming-schema: "%Y-%m-%d_%H-%M" schedule: minute: "0" src-b: dataset: data/src/b recursive: true lifetime: PT1H naming-schema: "%Y-%m-%d_%H-%M" schedule: minute: "0" replication-tasks: src-a: direction: push transport: type: ssh hostname: localhost private-key: | -----BEGIN RSA PRIVATE KEY----- MIIEowIBAAKCAQEA6+D7AWdNnM8T5f2P1j5VIwVABjugtL252iEhhGWTNaR2duCK kuZmG3+o55b0vo2mUpjWt+CKsLkVL/e/JgZqzlHYm+MhRs4q7zODo/IEtx/uAVKM zqWS0Zs9NRdU1UnhsETjrhhQhNFwx7MUYlB2s8+mAduLbeqRuVIKsXzg5Rz+m3VL Wl4R82A10oZl0UPyIcEHAtHMgMVyGzQcNUsxsp/oN40nnkXiXHaXtSMxjEtOPLno t21bJ0ZV8RFmXtJqHXgyTTM5maJM3JwqhMHD2tcHodqCcnxvuuWv31pAB+HKQ8IM dORYnhZqqs/Bt80gRLQuJBpNeX2/cKPDDMCRnQIDAQABAoIBAQCil6+N9R5rw9Ys iA85GDhpbnoGkd2iGNHeiU3oTHgf1uEN6pO61PR3ahUMpmLIYy3N66q+jxoq3Tm8 meL6HBxNYd+U/Qh4HS89OV45iV80t97ArJ2A6GL+9ypGyXFhoI7giWwEGqCOHSzH iyq25k4cfjspNqOyval7fBEA7Vq8smAMDJQE7WIJWzqrTbVAmVf9ho4r5dYxYBNW fXWo84DU8K+p0mE0BTokqqMWhKiA5JJG7OZB/iyeW2BWFOdASXvQmh1hRwMzpU4q BcZ7cJHz248SNSGMe5R3w7SmLO7PRr1/QkktJNdFmT7o/RGmQh8+KHql6r/vIzMM ci60OAxlAoGBAPYsZJZF3HK70fK3kARSzOD1LEVBDTCLnpVVzMSp6thG8cQqfCI5 pCfT/NcUsCAP6J+yl6dqdtonXISmGolI1s1KCBihs5D4jEdjbg9KbKh68AsHXaD3 v5L3POJ9hQnI6zJdvCfxniHdUArfyYhqsp1bnCn+85g4ed7BzDqMX2IDAoGBAPVL Y45rALw7lsjxJndyFdffJtyAeuwxgJNwWGuY21xhwqPbuwsgLHsGerHNKB5QAJT8 JOlrcrfC13s6Tt4wmIy/o2h1p9tMaitmVR6pJzEfHyJhSRTbeFybQ9yqlKHuk2tI jcUZV/59cyRrjhPKWoVym3Fh/P7D1t1kfdTvBrvfAoGAUH0rVkb5UTo/5xBFsmQw QM1o8CvY2CqOa11mWlcERjrMCcuqUrZuCeeyH9DP1WveL3kBROf2fFWqVmTJAGIk eXLfOs6EG75of17vOWioJl4r5i8+WccniDH2YkeQHCbpX8puHtFNVt05spSBHG1m gTTW1pRZqUet8TuEPxBuj2kCgYAVjCrRruqgnmdvfWeQpI/wp6SlSBAEQZD24q6R vRq/8cKEXGAA6TGfGQGcLtZwWzzB2ahwbMTmCZKeO5AECqbL7mWvXm6BYCQPbeza Raews/grL/qYf3MCR41djAqEcw22Jeh2QPSu4VxE/cG8UVFEWb335tCvnIp6ZkJ7 ewfPZwKBgEnc8HH1aq8IJ6vRBePNu6M9ON6PB9qW+ZHHcy47bcGogvYRQk1Ng77G LdZpyjWzzmb0Z4kjEYcrlGdbNQf9iaT0r+SJPzwBDG15+fRqK7EJI00UhjB0T67M otrkElxOBGqHSOl0jfUBrpSkSHiy0kDc3/cTAWKn0gowaznSlR9N -----END RSA PRIVATE KEY----- host-key: "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKwIWcodi3rHl0zS3VaddXnwfgIcpp1ECR9KhaB1cyIspgcHOA98wxY8onw+zHxpMUB4pve+t416FFLSkmlJ2f4=" source-dataset: data/src/a target-dataset: data/dst/a recursive: true periodic-snapshot-tasks: - src-a auto: true retention-policy: none speed-limit: 100000 src-b: direction: push transport: type: ssh hostname: localhost private-key: | -----BEGIN RSA PRIVATE KEY----- MIIEowIBAAKCAQEA6+D7AWdNnM8T5f2P1j5VIwVABjugtL252iEhhGWTNaR2duCK kuZmG3+o55b0vo2mUpjWt+CKsLkVL/e/JgZqzlHYm+MhRs4q7zODo/IEtx/uAVKM zqWS0Zs9NRdU1UnhsETjrhhQhNFwx7MUYlB2s8+mAduLbeqRuVIKsXzg5Rz+m3VL Wl4R82A10oZl0UPyIcEHAtHMgMVyGzQcNUsxsp/oN40nnkXiXHaXtSMxjEtOPLno t21bJ0ZV8RFmXtJqHXgyTTM5maJM3JwqhMHD2tcHodqCcnxvuuWv31pAB+HKQ8IM dORYnhZqqs/Bt80gRLQuJBpNeX2/cKPDDMCRnQIDAQABAoIBAQCil6+N9R5rw9Ys iA85GDhpbnoGkd2iGNHeiU3oTHgf1uEN6pO61PR3ahUMpmLIYy3N66q+jxoq3Tm8 meL6HBxNYd+U/Qh4HS89OV45iV80t97ArJ2A6GL+9ypGyXFhoI7giWwEGqCOHSzH iyq25k4cfjspNqOyval7fBEA7Vq8smAMDJQE7WIJWzqrTbVAmVf9ho4r5dYxYBNW fXWo84DU8K+p0mE0BTokqqMWhKiA5JJG7OZB/iyeW2BWFOdASXvQmh1hRwMzpU4q BcZ7cJHz248SNSGMe5R3w7SmLO7PRr1/QkktJNdFmT7o/RGmQh8+KHql6r/vIzMM ci60OAxlAoGBAPYsZJZF3HK70fK3kARSzOD1LEVBDTCLnpVVzMSp6thG8cQqfCI5 pCfT/NcUsCAP6J+yl6dqdtonXISmGolI1s1KCBihs5D4jEdjbg9KbKh68AsHXaD3 v5L3POJ9hQnI6zJdvCfxniHdUArfyYhqsp1bnCn+85g4ed7BzDqMX2IDAoGBAPVL Y45rALw7lsjxJndyFdffJtyAeuwxgJNwWGuY21xhwqPbuwsgLHsGerHNKB5QAJT8 JOlrcrfC13s6Tt4wmIy/o2h1p9tMaitmVR6pJzEfHyJhSRTbeFybQ9yqlKHuk2tI jcUZV/59cyRrjhPKWoVym3Fh/P7D1t1kfdTvBrvfAoGAUH0rVkb5UTo/5xBFsmQw QM1o8CvY2CqOa11mWlcERjrMCcuqUrZuCeeyH9DP1WveL3kBROf2fFWqVmTJAGIk eXLfOs6EG75of17vOWioJl4r5i8+WccniDH2YkeQHCbpX8puHtFNVt05spSBHG1m gTTW1pRZqUet8TuEPxBuj2kCgYAVjCrRruqgnmdvfWeQpI/wp6SlSBAEQZD24q6R vRq/8cKEXGAA6TGfGQGcLtZwWzzB2ahwbMTmCZKeO5AECqbL7mWvXm6BYCQPbeza Raews/grL/qYf3MCR41djAqEcw22Jeh2QPSu4VxE/cG8UVFEWb335tCvnIp6ZkJ7 ewfPZwKBgEnc8HH1aq8IJ6vRBePNu6M9ON6PB9qW+ZHHcy47bcGogvYRQk1Ng77G LdZpyjWzzmb0Z4kjEYcrlGdbNQf9iaT0r+SJPzwBDG15+fRqK7EJI00UhjB0T67M otrkElxOBGqHSOl0jfUBrpSkSHiy0kDc3/cTAWKn0gowaznSlR9N -----END RSA PRIVATE KEY----- host-key: "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKwIWcodi3rHl0zS3VaddXnwfgIcpp1ECR9KhaB1cyIspgcHOA98wxY8onw+zHxpMUB4pve+t416FFLSkmlJ2f4=" source-dataset: data/src/b target-dataset: data/dst/b recursive: true periodic-snapshot-tasks: - src-b auto: true retention-policy: none speed-limit: 100000 """))) local_shell = LocalShell() zettarepl = Zettarepl(Mock(), local_shell) zettarepl._spawn_retention = Mock() zettarepl.set_tasks(definition.tasks) zettarepl._spawn_replication_tasks( select_by_class(ReplicationTask, definition.tasks)) start = time.monotonic() wait_replication_tasks_to_complete(zettarepl) end = time.monotonic() assert 10 <= end - start <= 15 zettarepl._spawn_retention.assert_called_once() assert len(list_snapshots(local_shell, "data/dst/a", False)) == 1 assert len(list_snapshots(local_shell, "data/dst/b", False)) == 1 subprocess.call("zfs destroy -r data/dst", shell=True) subprocess.check_call("zfs create data/dst", shell=True) subprocess.check_call("zfs create data/dst/a", shell=True) subprocess.check_call("zfs create data/dst/b", shell=True) zettarepl._replication_tasks_can_run_in_parallel = Mock(return_value=False) zettarepl._spawn_replication_tasks( select_by_class(ReplicationTask, definition.tasks)) start = time.monotonic() wait_replication_tasks_to_complete(zettarepl) end = time.monotonic() assert 20 <= end - start <= 25 assert len(list_snapshots(local_shell, "data/dst/a", False)) == 1 assert len(list_snapshots(local_shell, "data/dst/b", False)) == 1
def test_parallel_replication(): subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs receive -A data/dst", shell=True) subprocess.call("zfs destroy -r data/dst", shell=True) subprocess.check_call("zfs create data/src", shell=True) subprocess.check_call("zfs create data/src/a", shell=True) subprocess.check_call("dd if=/dev/urandom of=/mnt/data/src/a/blob bs=1M count=1", shell=True) subprocess.check_call("zfs snapshot data/src/a@2018-10-01_01-00", shell=True) subprocess.check_call("zfs create data/src/b", shell=True) subprocess.check_call("dd if=/dev/urandom of=/mnt/data/src/b/blob bs=1M count=1", shell=True) subprocess.check_call("zfs snapshot data/src/b@2018-10-01_01-00", shell=True) subprocess.check_call("zfs create data/dst", shell=True) subprocess.check_call("zfs create data/dst/a", shell=True) subprocess.check_call("zfs create data/dst/b", shell=True) definition = yaml.safe_load(textwrap.dedent("""\ timezone: "UTC" periodic-snapshot-tasks: src-a: dataset: data/src/a recursive: true lifetime: PT1H naming-schema: "%Y-%m-%d_%H-%M" schedule: minute: "0" src-b: dataset: data/src/b recursive: true lifetime: PT1H naming-schema: "%Y-%m-%d_%H-%M" schedule: minute: "0" replication-tasks: src-a: direction: push transport: type: ssh hostname: localhost source-dataset: data/src/a target-dataset: data/dst/a recursive: true periodic-snapshot-tasks: - src-a auto: true retention-policy: none speed-limit: 100000 src-b: direction: push transport: type: ssh hostname: localhost source-dataset: data/src/b target-dataset: data/dst/b recursive: true periodic-snapshot-tasks: - src-b auto: true retention-policy: none speed-limit: 100000 """)) set_localhost_transport_options(definition["replication-tasks"]["src-a"]["transport"]) set_localhost_transport_options(definition["replication-tasks"]["src-b"]["transport"]) definition = Definition.from_data(definition) local_shell = LocalShell() zettarepl = Zettarepl(Mock(), local_shell) zettarepl._spawn_retention = Mock() zettarepl.set_tasks(definition.tasks) zettarepl._spawn_replication_tasks(select_by_class(ReplicationTask, definition.tasks)) start = time.monotonic() wait_replication_tasks_to_complete(zettarepl) end = time.monotonic() assert 10 <= end - start <= 15 zettarepl._spawn_retention.assert_called_once() assert len(list_snapshots(local_shell, "data/dst/a", False)) == 1 assert len(list_snapshots(local_shell, "data/dst/b", False)) == 1 subprocess.call("zfs destroy -r data/dst", shell=True) subprocess.check_call("zfs create data/dst", shell=True) subprocess.check_call("zfs create data/dst/a", shell=True) subprocess.check_call("zfs create data/dst/b", shell=True) zettarepl._replication_tasks_can_run_in_parallel = Mock(return_value=False) zettarepl._spawn_replication_tasks(select_by_class(ReplicationTask, definition.tasks)) start = time.monotonic() wait_replication_tasks_to_complete(zettarepl) end = time.monotonic() assert 20 <= end - start <= 25 assert len(list_snapshots(local_shell, "data/dst/a", False)) == 1 assert len(list_snapshots(local_shell, "data/dst/b", False)) == 1
def test_push_replication(dst_parent_is_readonly, dst_exists, transport, replicate, properties, encrypted, has_encrypted_child, dst_parent_encrypted): if replicate and not properties: return if encrypted and has_encrypted_child: # If parent is encrypted, child is also encrypted return subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs destroy -r data/dst_parent", shell=True) create_dataset("data/src", encrypted) subprocess.check_call("zfs set test:property=test-value data/src", shell=True) if has_encrypted_child: create_dataset("data/src/child", True) subprocess.check_call("zfs snapshot -r data/src@2018-10-01_01-00", shell=True) subprocess.check_call("zfs snapshot -r data/src@2018-10-01_02-00", shell=True) create_dataset("data/dst_parent", dst_parent_encrypted) if dst_exists: subprocess.check_call("zfs create data/dst_parent/dst", shell=True) if dst_parent_is_readonly: subprocess.check_call("zfs set readonly=on data/dst_parent", shell=True) definition = yaml.safe_load(textwrap.dedent("""\ timezone: "UTC" periodic-snapshot-tasks: src: dataset: data/src recursive: true lifetime: PT1H naming-schema: "%Y-%m-%d_%H-%M" schedule: minute: "0" replication-tasks: src: direction: push source-dataset: data/src target-dataset: data/dst_parent/dst recursive: true periodic-snapshot-tasks: - src auto: true retention-policy: none retries: 1 """)) definition["replication-tasks"]["src"]["transport"] = transport definition["replication-tasks"]["src"]["replicate"] = replicate definition["replication-tasks"]["src"]["properties"] = properties definition = Definition.from_data(definition) local_shell = LocalShell() zettarepl = Zettarepl(Mock(), local_shell) zettarepl._spawn_retention = Mock() observer = Mock() zettarepl.set_observer(observer) zettarepl.set_tasks(definition.tasks) zettarepl._spawn_replication_tasks(select_by_class(ReplicationTask, definition.tasks)) wait_replication_tasks_to_complete(zettarepl) if dst_exists and properties and encrypted and not dst_parent_encrypted: error = observer.call_args_list[-1][0][0] assert isinstance(error, ReplicationTaskError), error assert error.error == ("Unable to send encrypted dataset 'data/src' to existing unencrypted or unrelated " "dataset 'data/dst_parent/dst'") return error = observer.call_args_list[-1][0][0] assert isinstance(error, ReplicationTaskSuccess), error assert len(list_snapshots(local_shell, "data/dst_parent/dst", False)) == 2 assert ( ("test-value" in subprocess.check_output("zfs get test:property data/dst_parent/dst", shell=True, encoding="utf-8")) == properties ) subprocess.check_call("zfs snapshot -r data/src@2018-10-01_03-00", shell=True) zettarepl._spawn_replication_tasks(select_by_class(ReplicationTask, definition.tasks)) wait_replication_tasks_to_complete(zettarepl) error = observer.call_args_list[-1][0][0] assert isinstance(error, ReplicationTaskSuccess), error assert len(list_snapshots(local_shell, "data/dst_parent/dst", False)) == 3
def test_replication_resume(caplog): subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs receive -A data/dst", shell=True) subprocess.call("zfs destroy -r data/dst", shell=True) subprocess.check_call("zfs create data/src", shell=True) subprocess.check_call( "dd if=/dev/zero of=/mnt/data/src/blob bs=1M count=1", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True) subprocess.check_call("zfs create data/dst", shell=True) subprocess.check_call( "(zfs send data/src@2018-10-01_01-00 | throttle -b 102400 | zfs recv -s -F data/dst) & " "sleep 1; killall zfs", shell=True) assert "receive_resume_token\t1-" in subprocess.check_output( "zfs get -H receive_resume_token data/dst", shell=True, encoding="utf-8") definition = Definition.from_data( yaml.load( textwrap.dedent("""\ timezone: "UTC" periodic-snapshot-tasks: src: dataset: data/src recursive: true lifetime: PT1H naming-schema: "%Y-%m-%d_%H-%M" schedule: minute: "0" replication-tasks: src: direction: push transport: type: local source-dataset: data/src target-dataset: data/dst recursive: true periodic-snapshot-tasks: - src auto: true retention-policy: none """))) caplog.set_level(logging.INFO) local_shell = LocalShell() zettarepl = Zettarepl(Mock(), local_shell) zettarepl._spawn_retention = Mock() zettarepl.set_tasks(definition.tasks) zettarepl._spawn_replication_tasks( select_by_class(ReplicationTask, definition.tasks)) wait_replication_tasks_to_complete(zettarepl) assert any("Resuming replication for dst_dataset" in record.message for record in caplog.get_records("call")) assert len(list_snapshots(local_shell, "data/dst", False)) == 1
def test_preserves_clone_origin(): subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs receive -A data/dst", shell=True) subprocess.call("zfs destroy -r data/dst", shell=True) subprocess.check_call("zfs create data/src", shell=True) subprocess.check_call("zfs create data/src/iocage", shell=True) subprocess.check_call("zfs create data/src/iocage/child", shell=True) subprocess.check_call("zfs create data/src/iocage/child/dataset", shell=True) subprocess.check_call( "dd if=/dev/urandom of=/mnt/data/src/iocage/child/dataset/blob bs=1M count=1", shell=True) subprocess.check_call("zfs snapshot -r data/src@2019-11-08_14-00", shell=True) subprocess.check_call("zfs create data/src/iocage/another", shell=True) subprocess.check_call("zfs create data/src/iocage/another/child", shell=True) subprocess.check_call( "zfs clone data/src/iocage/child/dataset@2019-11-08_14-00 " "data/src/iocage/another/child/clone", shell=True) subprocess.check_call("zfs snapshot -r data/src@2019-11-08_15-00", shell=True) assert (subprocess.check_output( "zfs get -H origin data/src/iocage/another/child/clone", encoding="utf-8", shell=True).split("\n")[0].split("\t")[2] == "data/src/iocage/child/dataset@2019-11-08_14-00") assert int( subprocess.check_output( "zfs get -H -p used data/src/iocage/another/child/clone", encoding="utf-8", shell=True).split("\n")[0].split("\t")[2]) < 2e6 definition = yaml.safe_load( textwrap.dedent("""\ timezone: "UTC" replication-tasks: src: direction: push transport: type: local source-dataset: data/src target-dataset: data/dst recursive: true properties: true replicate: true also-include-naming-schema: - "%Y-%m-%d_%H-%M" auto: false retention-policy: none retries: 1 """)) definition = Definition.from_data(definition) local_shell = LocalShell() zettarepl = Zettarepl(Mock(), local_shell) zettarepl._spawn_retention = Mock() zettarepl.set_tasks(definition.tasks) zettarepl._spawn_replication_tasks( select_by_class(ReplicationTask, definition.tasks)) wait_replication_tasks_to_complete(zettarepl) assert (subprocess.check_output( "zfs get -H origin data/dst/iocage/another/child/clone", encoding="utf-8", shell=True).split("\n")[0].split("\t")[2] == "data/dst/iocage/child/dataset@2019-11-08_14-00") assert int( subprocess.check_output( "zfs get -H -p used data/dst/iocage/another/child/clone", encoding="utf-8", shell=True).split("\n")[0].split("\t")[2]) < 2e6
def test_hold_pending_snapshots__does_not_delete_orphan_snapshots(): subprocess.call("zfs destroy -r data/src", shell=True) subprocess.call("zfs destroy -r data/dst", shell=True) subprocess.check_call("zfs create data/src", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_00-00", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_01-00", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_02-00", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-01_03-00", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-02_00-00", shell=True) subprocess.check_call("zfs snapshot data/src@2018-10-03_00-00", shell=True) subprocess.check_call("zfs create data/dst", shell=True) subprocess.check_call("zfs snapshot data/dst@2018-10-01_00-00", shell=True) subprocess.check_call("zfs snapshot data/dst@2018-10-01_01-00", shell=True) subprocess.check_call("zfs snapshot data/dst@2018-10-01_02-00", shell=True) subprocess.check_call("zfs snapshot data/dst@2018-10-01_03-00", shell=True) subprocess.check_call("zfs snapshot data/dst@2018-10-02_00-00", shell=True) subprocess.check_call("zfs snapshot data/dst@2018-10-03_00-00", shell=True) definition = Definition.from_data( yaml.safe_load( textwrap.dedent("""\ timezone: "UTC" periodic-snapshot-tasks: src: dataset: data/src recursive: true lifetime: PT48H naming-schema: "%Y-%m-%d_%H-%M" schedule: minute: "0" hour: "0" replication-tasks: src: direction: push transport: type: local source-dataset: data/src target-dataset: data/dst recursive: true periodic-snapshot-tasks: - src auto: true retention-policy: source hold-pending-snapshots: true """))) local_shell = LocalShell() zettarepl = Zettarepl(Mock(), local_shell) zettarepl.set_tasks(definition.tasks) zettarepl._run_local_retention(datetime(2018, 10, 4, 0, 0)) assert list_snapshots(local_shell, "data/src", False) == [ Snapshot("data/src", "2018-10-01_01-00"), Snapshot("data/src", "2018-10-01_02-00"), Snapshot("data/src", "2018-10-01_03-00"), Snapshot("data/src", "2018-10-02_00-00"), Snapshot("data/src", "2018-10-03_00-00"), ]