def run(self):
if self.compression is not None:
raise ReplicationConfigurationError(
"compression is not supported for local replication (it has no sense)"
)
if self.speed_limit is not None:
raise ReplicationConfigurationError(
"speed-limit is not supported for local replication (it has no sense)"
)
report_progress = self._zfs_send_can_report_progress()
send = zfs_send(self.source_dataset, self.snapshot, self.properties,
self.replicate, self.incremental_base,
self.receive_resume_token, self.dedup,
self.large_block, self.embed, self.compressed,
self.raw, report_progress)
if self.encryption:
self.encryption_context = EncryptionContext(self, self.local_shell)
properties_override = get_properties_override(self,
self.encryption_context)
recv = zfs_recv(self.target_dataset, self.properties_exclude,
properties_override)
send = self._wrap_send(send)
self.async_exec = AsyncExecTee(self.local_shell, pipe(send, recv))
self.async_exec.run()
if report_progress:
self._start_progress_observer()
def test__local_async_exec_stop():
def assert_marker_count(count):
assert int(
subprocess.check_output(
"ps axw | grep ZETTAREPL_TEST_MARKER_1 | grep -v grep | wc -l",
shell=True,
encoding="utf-8",
).strip()) == count
assert_marker_count(0)
local_shell = LocalShell()
exec = local_shell.exec_async(
pipe([
"python", "-c",
"'ZETTAREPL_TEST_MARKER_1'; import time; time.sleep(60)"
], [
"python", "-c",
"'ZETTAREPL_TEST_MARKER_1'; import time; time.sleep(60)"
]))
time.sleep(2)
assert_marker_count(6)
exec.stop()
time.sleep(1)
assert_marker_count(0)
def run(self):
if self.compression is not None:
raise ReplicationConfigurationError(
"compression is not supported for local replication (it has no sense)"
)
if self.speed_limit is not None:
raise ReplicationConfigurationError(
"speed-limit is not supported for local replication (it has no sense)"
)
self.async_exec = self.local_shell.exec_async(
pipe(
zfs_send(self.source_dataset, self.snapshot, self.recursive,
self.incremental_base, self.receive_resume_token,
self.dedup, self.large_block, self.embed,
self.compressed), zfs_recv(self.target_dataset)))
def test__pipe(bad_command):
commands = [
"echo a",
"sed 's/a/)/'",
"sed 's/)/\"/'",
"sed 's/\"/d/'",
]
if bad_command is not None:
commands[bad_command] += f"; echo ERROR 1>&2; exit {bad_command * 10 + 1}"
piped = pipe(*[["sh", "-c", cmd] for cmd in commands])
print(piped[2])
cp = subprocess.run(piped, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf-8")
assert cp.returncode == (0 if bad_command is None else bad_command * 10 + 1)
assert cp.stdout == "d\n"
if bad_command is not None:
assert cp.stderr == "ERROR\n"
def run(self):
report_progress = self._zfs_send_can_report_progress()
self.private_key_file = tempfile.NamedTemporaryFile("w")
os.chmod(self.private_key_file.name, 0o600)
self.private_key_file.write(self.transport.private_key)
self.private_key_file.flush()
self.host_key_file = tempfile.NamedTemporaryFile("w")
os.chmod(self.host_key_file.name, 0o600)
self.host_key_file.write(self.transport.get_host_key_entry())
self.host_key_file.flush()
try:
cmd = [self.transport.client_capabilities.executable]
cmd.extend({
SshTransportCipher.STANDARD: [],
SshTransportCipher.FAST:
["-c", "aes128-ctr,aes192-ctr,aes256-ctr"],
SshTransportCipher.DISABLED:
["-ononeenabled=yes", "-ononeswitch=yes"],
}[self.transport.cipher])
cmd.extend(["-i", self.private_key_file.name])
cmd.extend(["-o", f"UserKnownHostsFile={self.host_key_file.name}"])
cmd.extend(["-o", "StrictHostKeyChecking=yes"])
cmd.extend(["-o", "BatchMode=yes"])
cmd.extend(
["-o", f"ConnectTimeout={self.transport.connect_timeout}"])
cmd.extend([f"-p{self.transport.port}"])
cmd.extend(
[f"{self.transport.username}@{self.transport.hostname}"])
send = zfs_send(self.source_dataset, self.snapshot,
self.properties, self.replicate,
self.incremental_base, self.receive_resume_token,
self.dedup, self.large_block, self.embed,
self.compressed, self.raw, report_progress)
recv_properties = {}
if self.encryption:
self.encryption_context = EncryptionContext(
self, self._get_recv_shell())
recv_properties = self.encryption_context.enter()
recv = zfs_recv(self.target_dataset, recv_properties)
send = self._wrap_send(send)
if self.compression is not None:
send = pipe(send, self.compression.compress)
recv = pipe(self.compression.decompress, recv)
if self.speed_limit is not None:
send = pipe(send, ["throttle", "-B", str(self.speed_limit)])
if self.direction == ReplicationDirection.PUSH:
commands = [send, cmd + [implode(recv)]]
elif self.direction == ReplicationDirection.PULL:
commands = [cmd + [implode(send)], recv]
else:
raise ValueError(
f"Invalid replication direction: {self.direction!r}")
self.async_exec = AsyncExecTee(self.local_shell, pipe(*commands))
self.async_exec.run()
if report_progress:
self._start_progress_observer()
except Exception:
self.private_key_file.close()
self.host_key_file.close()
raise
def test__pipe():
assert (
pipe(["zfs", "send", "my dataset@snap"], ["zfs", "recv", "my'dataset"]) ==
["sh", "-c", "zfs send 'my dataset@snap' | zfs recv 'my'\"'\"'dataset'"]
)
def run(self):
self.report_progress = self._zfs_send_can_report_progress()
self.private_key_file = tempfile.NamedTemporaryFile("w")
os.chmod(self.private_key_file.name, 0o600)
self.private_key_file.write(self.transport.private_key)
self.private_key_file.flush()
self.host_key_file = tempfile.NamedTemporaryFile("w")
os.chmod(self.host_key_file.name, 0o600)
self.host_key_file.write(f"{self.transport.hostname} {self.transport.host_key}")
self.host_key_file.flush()
try:
cmd = [self.transport.client_capabilities.executable]
cmd.extend({
SshTransportCipher.STANDARD: [],
SshTransportCipher.FAST: ["-c", "aes128-ctr,aes192-ctr,aes256-ctr"],
SshTransportCipher.DISABLED: ["-ononeenabled=yes", "-ononeswitch=yes"],
}[self.transport.cipher])
cmd.extend(["-i", self.private_key_file.name])
cmd.extend(["-o", f"UserKnownHostsFile={self.host_key_file.name}"])
cmd.extend(["-o", "StrictHostKeyChecking=yes"])
cmd.extend(["-o", "BatchMode=yes"])
cmd.extend(["-o", f"ConnectTimeout={self.transport.connect_timeout}"])
cmd.extend([f"-p{self.transport.port}"])
cmd.extend([f"{self.transport.username}@{self.transport.hostname}"])
send = zfs_send(self.source_dataset, self.snapshot, self.recursive, self.incremental_base,
self.receive_resume_token,
self.dedup, self.large_block, self.embed, self.compressed,
self.report_progress)
recv = zfs_recv(self.target_dataset)
send = ["sh", "-c", "(" + implode(send) + " & PID=$!; echo \"zettarepl: zfs send PID is $PID\" 1>&2; "
"wait $PID)"]
if self.compression is not None:
send = pipe(send, self.compression.compress)
recv = pipe(self.compression.decompress, recv)
if self.speed_limit is not None:
send = pipe(send, ["throttle", "-B", str(self.speed_limit)])
if self.direction == ReplicationDirection.PUSH:
commands = [send, cmd + [implode(recv)]]
elif self.direction == ReplicationDirection.PULL:
commands = [cmd + [implode(send)], recv]
else:
raise ValueError(f"Invalid replication direction: {self.direction!r}")
self.async_exec = AsyncExecTee(self.local_shell, pipe(*commands))
self.async_exec.run()
if self.report_progress:
self.stop_progress_observer = threading.Event()
pid = self.async_exec.head(self._get_zettarepl_pid, 10)
threading.Thread(daemon=True, name=f"{threading.current_thread().name}.ssh.progress_observer",
target=self._progress_observer, args=(pid,)).start()
except Exception:
self.private_key_file.close()
self.host_key_file.close()
raise
def run(self):
self.private_key_file = tempfile.NamedTemporaryFile("w")
os.chmod(self.private_key_file.name, 0o600)
self.private_key_file.write(self.transport.private_key)
self.private_key_file.flush()
self.host_key_file = tempfile.NamedTemporaryFile("w")
os.chmod(self.host_key_file.name, 0o600)
self.host_key_file.write(
f"{self.transport.hostname} {self.transport.host_key}")
self.host_key_file.flush()
try:
cmd = [self.transport.client_capabilities.executable]
cmd.extend({
SshTransportCipher.STANDARD: [],
SshTransportCipher.FAST: [
"-c",
"arcfour256,arcfour128,blowfish-cbc,aes128-ctr,aes192-ctr,aes256-ctr"
],
SshTransportCipher.DISABLED:
["-ononeenabled=yes", "-ononeswitch=yes"],
}[self.transport.cipher])
cmd.extend(["-i", self.private_key_file.name])
cmd.extend(["-o", f"UserKnownHostsFile={self.host_key_file.name}"])
cmd.extend(["-o", "StrictHostKeyChecking=yes"])
cmd.extend(["-o", "BatchMode=yes"])
cmd.extend(
["-o", f"ConnectTimeout={self.transport.connect_timeout}"])
cmd.extend([f"-p{self.transport.port}"])
cmd.extend(
[f"{self.transport.username}@{self.transport.hostname}"])
send = zfs_send(self.source_dataset, self.snapshot, self.recursive,
self.incremental_base, self.receive_resume_token,
self.dedup, self.large_block, self.embed,
self.compressed)
recv = zfs_recv(self.target_dataset)
if self.compression is not None:
send = pipe(send, self.compression.compress)
recv = pipe(self.compression.decompress, recv)
if self.speed_limit is not None:
send = pipe(send, ["throttle", "-B", str(self.speed_limit)])
if self.direction == ReplicationDirection.PUSH:
commands = [send, cmd + [implode(recv)]]
elif self.direction == ReplicationDirection.PULL:
commands = [cmd + [implode(send)], recv]
else:
raise ValueError(
f"Invalid replication direction: {self.direction!r}")
self.async_exec = self.local_shell.exec_async(pipe(*commands))
except Exception:
self.private_key_file.close()
self.host_key_file.close()
raise
