def run(self): if self.compression is not None: raise ReplicationConfigurationError( "compression is not supported for local replication (it has no sense)" ) if self.speed_limit is not None: raise ReplicationConfigurationError( "speed-limit is not supported for local replication (it has no sense)" ) report_progress = self._zfs_send_can_report_progress() send = zfs_send(self.source_dataset, self.snapshot, self.properties, self.replicate, self.incremental_base, self.receive_resume_token, self.dedup, self.large_block, self.embed, self.compressed, self.raw, report_progress) if self.encryption: self.encryption_context = EncryptionContext(self, self.local_shell) properties_override = get_properties_override(self, self.encryption_context) recv = zfs_recv(self.target_dataset, self.properties_exclude, properties_override) send = self._wrap_send(send) self.async_exec = AsyncExecTee(self.local_shell, pipe(send, recv)) self.async_exec.run() if report_progress: self._start_progress_observer()
def test__local_async_exec_stop(): def assert_marker_count(count): assert int( subprocess.check_output( "ps axw | grep ZETTAREPL_TEST_MARKER_1 | grep -v grep | wc -l", shell=True, encoding="utf-8", ).strip()) == count assert_marker_count(0) local_shell = LocalShell() exec = local_shell.exec_async( pipe([ "python", "-c", "'ZETTAREPL_TEST_MARKER_1'; import time; time.sleep(60)" ], [ "python", "-c", "'ZETTAREPL_TEST_MARKER_1'; import time; time.sleep(60)" ])) time.sleep(2) assert_marker_count(6) exec.stop() time.sleep(1) assert_marker_count(0)
def run(self): if self.compression is not None: raise ReplicationConfigurationError( "compression is not supported for local replication (it has no sense)" ) if self.speed_limit is not None: raise ReplicationConfigurationError( "speed-limit is not supported for local replication (it has no sense)" ) self.async_exec = self.local_shell.exec_async( pipe( zfs_send(self.source_dataset, self.snapshot, self.recursive, self.incremental_base, self.receive_resume_token, self.dedup, self.large_block, self.embed, self.compressed), zfs_recv(self.target_dataset)))
def test__pipe(bad_command): commands = [ "echo a", "sed 's/a/)/'", "sed 's/)/\"/'", "sed 's/\"/d/'", ] if bad_command is not None: commands[bad_command] += f"; echo ERROR 1>&2; exit {bad_command * 10 + 1}" piped = pipe(*[["sh", "-c", cmd] for cmd in commands]) print(piped[2]) cp = subprocess.run(piped, stdout=subprocess.PIPE, stderr=subprocess.PIPE, encoding="utf-8") assert cp.returncode == (0 if bad_command is None else bad_command * 10 + 1) assert cp.stdout == "d\n" if bad_command is not None: assert cp.stderr == "ERROR\n"
def run(self): report_progress = self._zfs_send_can_report_progress() self.private_key_file = tempfile.NamedTemporaryFile("w") os.chmod(self.private_key_file.name, 0o600) self.private_key_file.write(self.transport.private_key) self.private_key_file.flush() self.host_key_file = tempfile.NamedTemporaryFile("w") os.chmod(self.host_key_file.name, 0o600) self.host_key_file.write(self.transport.get_host_key_entry()) self.host_key_file.flush() try: cmd = [self.transport.client_capabilities.executable] cmd.extend({ SshTransportCipher.STANDARD: [], SshTransportCipher.FAST: ["-c", "aes128-ctr,aes192-ctr,aes256-ctr"], SshTransportCipher.DISABLED: ["-ononeenabled=yes", "-ononeswitch=yes"], }[self.transport.cipher]) cmd.extend(["-i", self.private_key_file.name]) cmd.extend(["-o", f"UserKnownHostsFile={self.host_key_file.name}"]) cmd.extend(["-o", "StrictHostKeyChecking=yes"]) cmd.extend(["-o", "BatchMode=yes"]) cmd.extend( ["-o", f"ConnectTimeout={self.transport.connect_timeout}"]) cmd.extend([f"-p{self.transport.port}"]) cmd.extend( [f"{self.transport.username}@{self.transport.hostname}"]) send = zfs_send(self.source_dataset, self.snapshot, self.properties, self.replicate, self.incremental_base, self.receive_resume_token, self.dedup, self.large_block, self.embed, self.compressed, self.raw, report_progress) recv_properties = {} if self.encryption: self.encryption_context = EncryptionContext( self, self._get_recv_shell()) recv_properties = self.encryption_context.enter() recv = zfs_recv(self.target_dataset, recv_properties) send = self._wrap_send(send) if self.compression is not None: send = pipe(send, self.compression.compress) recv = pipe(self.compression.decompress, recv) if self.speed_limit is not None: send = pipe(send, ["throttle", "-B", str(self.speed_limit)]) if self.direction == ReplicationDirection.PUSH: commands = [send, cmd + [implode(recv)]] elif self.direction == ReplicationDirection.PULL: commands = [cmd + [implode(send)], recv] else: raise ValueError( f"Invalid replication direction: {self.direction!r}") self.async_exec = AsyncExecTee(self.local_shell, pipe(*commands)) self.async_exec.run() if report_progress: self._start_progress_observer() except Exception: self.private_key_file.close() self.host_key_file.close() raise
def test__pipe(): assert ( pipe(["zfs", "send", "my dataset@snap"], ["zfs", "recv", "my'dataset"]) == ["sh", "-c", "zfs send 'my dataset@snap' | zfs recv 'my'\"'\"'dataset'"] )
def run(self): self.report_progress = self._zfs_send_can_report_progress() self.private_key_file = tempfile.NamedTemporaryFile("w") os.chmod(self.private_key_file.name, 0o600) self.private_key_file.write(self.transport.private_key) self.private_key_file.flush() self.host_key_file = tempfile.NamedTemporaryFile("w") os.chmod(self.host_key_file.name, 0o600) self.host_key_file.write(f"{self.transport.hostname} {self.transport.host_key}") self.host_key_file.flush() try: cmd = [self.transport.client_capabilities.executable] cmd.extend({ SshTransportCipher.STANDARD: [], SshTransportCipher.FAST: ["-c", "aes128-ctr,aes192-ctr,aes256-ctr"], SshTransportCipher.DISABLED: ["-ononeenabled=yes", "-ononeswitch=yes"], }[self.transport.cipher]) cmd.extend(["-i", self.private_key_file.name]) cmd.extend(["-o", f"UserKnownHostsFile={self.host_key_file.name}"]) cmd.extend(["-o", "StrictHostKeyChecking=yes"]) cmd.extend(["-o", "BatchMode=yes"]) cmd.extend(["-o", f"ConnectTimeout={self.transport.connect_timeout}"]) cmd.extend([f"-p{self.transport.port}"]) cmd.extend([f"{self.transport.username}@{self.transport.hostname}"]) send = zfs_send(self.source_dataset, self.snapshot, self.recursive, self.incremental_base, self.receive_resume_token, self.dedup, self.large_block, self.embed, self.compressed, self.report_progress) recv = zfs_recv(self.target_dataset) send = ["sh", "-c", "(" + implode(send) + " & PID=$!; echo \"zettarepl: zfs send PID is $PID\" 1>&2; " "wait $PID)"] if self.compression is not None: send = pipe(send, self.compression.compress) recv = pipe(self.compression.decompress, recv) if self.speed_limit is not None: send = pipe(send, ["throttle", "-B", str(self.speed_limit)]) if self.direction == ReplicationDirection.PUSH: commands = [send, cmd + [implode(recv)]] elif self.direction == ReplicationDirection.PULL: commands = [cmd + [implode(send)], recv] else: raise ValueError(f"Invalid replication direction: {self.direction!r}") self.async_exec = AsyncExecTee(self.local_shell, pipe(*commands)) self.async_exec.run() if self.report_progress: self.stop_progress_observer = threading.Event() pid = self.async_exec.head(self._get_zettarepl_pid, 10) threading.Thread(daemon=True, name=f"{threading.current_thread().name}.ssh.progress_observer", target=self._progress_observer, args=(pid,)).start() except Exception: self.private_key_file.close() self.host_key_file.close() raise
def run(self): self.private_key_file = tempfile.NamedTemporaryFile("w") os.chmod(self.private_key_file.name, 0o600) self.private_key_file.write(self.transport.private_key) self.private_key_file.flush() self.host_key_file = tempfile.NamedTemporaryFile("w") os.chmod(self.host_key_file.name, 0o600) self.host_key_file.write( f"{self.transport.hostname} {self.transport.host_key}") self.host_key_file.flush() try: cmd = [self.transport.client_capabilities.executable] cmd.extend({ SshTransportCipher.STANDARD: [], SshTransportCipher.FAST: [ "-c", "arcfour256,arcfour128,blowfish-cbc,aes128-ctr,aes192-ctr,aes256-ctr" ], SshTransportCipher.DISABLED: ["-ononeenabled=yes", "-ononeswitch=yes"], }[self.transport.cipher]) cmd.extend(["-i", self.private_key_file.name]) cmd.extend(["-o", f"UserKnownHostsFile={self.host_key_file.name}"]) cmd.extend(["-o", "StrictHostKeyChecking=yes"]) cmd.extend(["-o", "BatchMode=yes"]) cmd.extend( ["-o", f"ConnectTimeout={self.transport.connect_timeout}"]) cmd.extend([f"-p{self.transport.port}"]) cmd.extend( [f"{self.transport.username}@{self.transport.hostname}"]) send = zfs_send(self.source_dataset, self.snapshot, self.recursive, self.incremental_base, self.receive_resume_token, self.dedup, self.large_block, self.embed, self.compressed) recv = zfs_recv(self.target_dataset) if self.compression is not None: send = pipe(send, self.compression.compress) recv = pipe(self.compression.decompress, recv) if self.speed_limit is not None: send = pipe(send, ["throttle", "-B", str(self.speed_limit)]) if self.direction == ReplicationDirection.PUSH: commands = [send, cmd + [implode(recv)]] elif self.direction == ReplicationDirection.PULL: commands = [cmd + [implode(send)], recv] else: raise ValueError( f"Invalid replication direction: {self.direction!r}") self.async_exec = self.local_shell.exec_async(pipe(*commands)) except Exception: self.private_key_file.close() self.host_key_file.close() raise