def get_transport(definition_path, transport): definition = load_definition_raw(definition_path) if transport: try: transport = definition.get("transports", {})[transport] except KeyError: sys.stderr.write(f"Invalid transport {transport!r}\n") sys.exit(1) return create_transport(transport) else: return create_transport({"type": "local"})
async def _get_zettarepl_shell(self, transport, ssh_credentials): transport_definition = await self._define_transport(transport, ssh_credentials) transport = create_transport(transport_definition) shell = transport.shell(transport) try: yield shell finally: await self.middleware.run_in_thread(shell.close)
async def _get_zettarepl_shell(self, transport, ssh_credentials): if transport != "LOCAL": await self.middleware.call("network.general.will_perform_activity", "replication") transport_definition = await self._define_transport(transport, ssh_credentials) transport = create_transport(transport_definition) shell = transport.shell(transport) try: yield shell finally: await self.middleware.run_in_thread(shell.close)
async def _get_zettarepl_shell(self, transport, ssh_credentials): if transport != "LOCAL": await self.middleware.call("network.general.will_perform_activity", "replication") if transport == "SSH+NETCAT": # There is no difference shell-wise, but `_define_transport` for `SSH+NETCAT` will fail if we don't # supply `netcat_active_side` and other parameters which are totally unrelated here. transport = "SSH" transport_definition = await self._define_transport(transport, ssh_credentials) transport = create_transport(transport_definition) shell = transport.shell(transport) try: yield shell finally: await self.middleware.run_in_thread(shell.close)
def test__async_exec_timeout(transport, stdout): if transport["type"] == "local": expected_timeout = 5 else: expected_timeout = 10 transport_inst = create_transport(copy.deepcopy(transport)) shell = transport_inst.shell(transport_inst) start = time.monotonic() with pytest.raises(TimeoutError): shell.exec(["python", "-c", "'ZETTAREPL_TEST_MARKER_1'; import time; time.sleep(15)"], timeout=5) end = time.monotonic() assert expected_timeout * 0.9 < end - start < expected_timeout * 1.1 if transport["type"] == "local": assert int(subprocess.check_output( "ps axw | grep ZETTAREPL_TEST_MARKER_1 | grep -v grep | wc -l", shell=True, encoding="utf-8", ).strip()) == 0
def from_data(cls, id, data: dict, periodic_snapshot_tasks: [PeriodicSnapshotTask]): replication_task_validator.validate(data) for k in [ "source-dataset", "naming-schema", "also-include-naming-schema" ]: if k in data and isinstance(data[k], str): data[k] = [data[k]] data.setdefault("exclude", []) data.setdefault("properties", True) data.setdefault("replicate", False) data.setdefault("periodic-snapshot-tasks", []) data.setdefault("only-matching-schedule", False) data.setdefault("allow-from-scratch", False) data.setdefault("hold-pending-snapshots", False) data.setdefault("compression", None) data.setdefault("speed-limit", None) data.setdefault("dedup", False) data.setdefault("large-block", False) data.setdefault("embed", False) data.setdefault("compressed", False) data.setdefault("retries", 5) data.setdefault("logging-level", "notset") resolved_periodic_snapshot_tasks = [] for periodic_snapshot_task_id in data["periodic-snapshot-tasks"]: for periodic_snapshot_task in periodic_snapshot_tasks: if periodic_snapshot_task.id == periodic_snapshot_task_id: resolved_periodic_snapshot_tasks.append( periodic_snapshot_task) break else: raise ValueError( f"Periodic snapshot task {periodic_snapshot_task_id!r} does not exist" ) if data["recursive"]: for source_dataset in data["source-dataset"]: for periodic_snapshot_task in resolved_periodic_snapshot_tasks: if is_child(source_dataset, periodic_snapshot_task.dataset): for exclude in periodic_snapshot_task.exclude: if exclude not in data["exclude"]: raise ValueError( "Replication tasks should exclude everything their periodic snapshot tasks exclude " f"(task does not exclude {exclude!r} from periodic snapshot task " f"{periodic_snapshot_task.id!r})") if data["replicate"]: if not data["recursive"]: raise ValueError( "Replication tasks that replicate entire filesystem should be recursive" ) if data["exclude"]: raise ValueError( "Replication tasks that replicate entire filesystem can't exclude datasets" ) if not data["properties"]: raise ValueError( "Replication tasks that replicate entire filesystem can't exclude properties" ) data["direction"] = ReplicationDirection(data["direction"]) if data["direction"] == ReplicationDirection.PUSH: if "naming-schema" in data: raise ValueError( "Push replication task can't have naming-schema") data.setdefault("also-include-naming-schema", []) if not resolved_periodic_snapshot_tasks and not data[ "also-include-naming-schema"]: raise ValueError( "You must at least provide either periodic-snapshot-tasks or also-include-naming-schema " "for push replication task") elif data["direction"] == ReplicationDirection.PULL: if "naming-schema" not in data: raise ValueError( "You must provide naming-schema for pull replication task") if "also-include-naming-schema" in data: raise ValueError( "Pull replication task can't have also-include-naming-schema" ) data.setdefault("also-include-naming-schema", data.pop("naming-schema")) schedule, restrict_schedule = cls._parse_schedules(data) if data["direction"] == ReplicationDirection.PULL: if data["hold-pending-snapshots"]: raise ValueError( "Pull replication tasks can't hold pending snapshots because they don't do source " "retention") retention_policy = TargetSnapshotRetentionPolicy.from_data(data) compression = replication_compressions[ data["compression"]] if data["compression"] else None return cls(id, data["direction"], create_transport(data["transport"]), data["source-dataset"], data["target-dataset"], data["recursive"], data["exclude"], data["properties"], data["replicate"], resolved_periodic_snapshot_tasks, data["also-include-naming-schema"], data["auto"], schedule, restrict_schedule, data["only-matching-schedule"], data["allow-from-scratch"], data["hold-pending-snapshots"], retention_policy, compression, data["speed-limit"], data["dedup"], data["large-block"], data["embed"], data["compressed"], data["retries"], logging._nameToLevel[data["logging-level"].upper()])
def from_data(cls, id, data: dict, periodic_snapshot_tasks: [PeriodicSnapshotTask]): replication_task_validator.validate(data) for k in [ "source-dataset", "naming-schema", "also-include-naming-schema" ]: if k in data and isinstance(data[k], str): data[k] = [data[k]] data.setdefault("exclude", []) data.setdefault("properties", True) data.setdefault("properties-exclude", []) data.setdefault("properties-override", {}) data.setdefault("replicate", False) data.setdefault("encryption", None) data.setdefault("periodic-snapshot-tasks", []) data.setdefault("name-regex", None) data.setdefault("only-matching-schedule", False) data.setdefault("readonly", "ignore") data.setdefault("allow-from-scratch", False) data.setdefault("hold-pending-snapshots", False) data.setdefault("compression", None) data.setdefault("speed-limit", None) data.setdefault("dedup", False) data.setdefault("large-block", False) data.setdefault("embed", False) data.setdefault("compressed", False) data.setdefault("retries", 5) data.setdefault("logging-level", "notset") resolved_periodic_snapshot_tasks = [] for periodic_snapshot_task_id in data["periodic-snapshot-tasks"]: for periodic_snapshot_task in periodic_snapshot_tasks: if periodic_snapshot_task.id == periodic_snapshot_task_id: resolved_periodic_snapshot_tasks.append( periodic_snapshot_task) break else: raise ValueError( f"Periodic snapshot task {periodic_snapshot_task_id!r} does not exist" ) if data["recursive"]: cls._validate_exclude(data, resolved_periodic_snapshot_tasks) if data["replicate"]: if not data["recursive"]: raise ValueError( "Replication tasks that replicate entire filesystem should be recursive" ) if data["exclude"]: raise ValueError( "Replication tasks that replicate entire filesystem can't exclude datasets" ) if not data["properties"]: raise ValueError( "Replication tasks that replicate entire filesystem can't exclude properties" ) if data["encryption"]: encryption = ReplicationEncryption( data["encryption"]["key"], KeyFormat(data["encryption"]["key-format"]), data["encryption"]["key-location"], ) else: encryption = None data["direction"] = ReplicationDirection(data["direction"]) if data["direction"] == ReplicationDirection.PUSH: if "naming-schema" in data: raise ValueError( "Push replication task can't have naming-schema") data.setdefault("also-include-naming-schema", []) if not resolved_periodic_snapshot_tasks and not data[ "also-include-naming-schema"] and not data["name-regex"]: raise ValueError( "You must at least provide either periodic-snapshot-tasks or also-include-naming-schema or " "name-regex for push replication task") elif data["direction"] == ReplicationDirection.PULL: if "naming-schema" not in data and not data["name-regex"]: raise ValueError( "You must provide naming-schema or name-regex for pull replication task" ) if "also-include-naming-schema" in data: raise ValueError( "Pull replication task can't have also-include-naming-schema" ) data.setdefault("also-include-naming-schema", data.pop("naming-schema")) schedule, restrict_schedule = cls._parse_schedules(data) if data["direction"] == ReplicationDirection.PULL: if data["hold-pending-snapshots"]: raise ValueError( "Pull replication tasks can't hold pending snapshots because they don't do source " "retention") if data["name-regex"]: try: name_pattern = re.compile(f"({data['name-regex']})$") except Exception as e: raise ValueError(f"Invalid name-regex: {e}") if data["also-include-naming-schema"]: raise ValueError( "naming-schema/also-include-naming-schema can't be used with name-regex" ) else: name_pattern = None retention_policy = TargetSnapshotRetentionPolicy.from_data(data) compression = replication_compressions[ data["compression"]] if data["compression"] else None return cls( id, data["direction"], create_transport(data["transport"]), data["source-dataset"], data["target-dataset"], data["recursive"], data["exclude"], data["properties"], data["properties-exclude"], {k: str(v) for k, v in data["properties-override"].items()}, data["replicate"], encryption, resolved_periodic_snapshot_tasks, data["also-include-naming-schema"], name_pattern, data["auto"], schedule, restrict_schedule, data["only-matching-schedule"], ReadOnlyBehavior(data["readonly"]), data["allow-from-scratch"], data["hold-pending-snapshots"], retention_policy, compression, data["speed-limit"], data["dedup"], data["large-block"], data["embed"], data["compressed"], data["retries"], logging._nameToLevel[data["logging-level"].upper()])
async def _get_zettarepl_shell(self, transport, ssh_credentials): transport_definition = await self._define_transport( transport, ssh_credentials) transport = create_transport(transport_definition) return transport.shell(transport)