async def asyncSetUp(self) -> None:
self.thread = local.thread
self.tmpdir = await self.thread.mkdtemp("test_mail")
self.path = self.tmpdir.path
await update_symlink(
self.thread, await
self.thread.ram.ptr(self.tmpdir.parent / "test_mail.current"),
self.path)
smtp_sock = await self.thread.task.socket(AF.INET, SOCK.STREAM)
await smtp_sock.bind(await
self.thread.ram.ptr(SockaddrIn(3000,
"127.0.0.1")))
await smtp_sock.listen(10)
self.smtpd = await start_smtpd(
self.nursery, self.thread, await
self.thread.mkdir(self.path / "smtpd"), smtp_sock)
self.maildir = await Maildir.make(self.thread, self.path / "mail")
self.dovecot = await start_dovecot(
self.nursery, self.thread, await
self.thread.mkdir(self.path / "dovecot"), self.smtpd.lmtp_listener,
self.maildir)
smtpctl = await self.thread.environ.which("smtpctl")
self.sendmail = Command(smtpctl.executable_path, [b'sendmail'],
{'SMTPD_CONFIG_FILE': self.smtpd.config_file})
self.inty = await Inotify.make(self.thread)
async def start_recursor(nursery, parent: Thread, path: Path,
ipv4_sockets: t.List[t.Tuple[handle.FileDescriptor, handle.FileDescriptor]],
ipv6_sockets: t.List[t.Tuple[handle.FileDescriptor, handle.FileDescriptor]],
root_hints: dns.zone.Zone=None) -> Powerdns:
pdns_recursor = Command(Path("/home/sbaugh/.local/src/pdns/pdns/recursordist/pdns_recursor"), ['pdns_recursor'], {})
thread = await parent.fork()
ipv4s = {str(i+1): (udp.move(thread.task), tcp.move(thread.task))
for i, (udp, tcp) in enumerate(ipv4_sockets)}
ipv6s = {"::"+str(i+1): (udp.move(thread.task), tcp.move(thread.task))
for i, (udp, tcp) in enumerate(ipv6_sockets)}
addresses = {**ipv4s, **ipv6s}
await thread.unshare_files(going_to_exec=True)
config = {
"config-dir": os.fsdecode(path),
"socket-dir": os.fsdecode(path),
# more logging
"loglevel": "9",
"log-common-errors": "yes",
"quiet": "no",
"trace": "yes",
"dont-query": "",
"logging-facility": "0",
# relevant stuff
"local-address": ",".join(addresses),
"allow-from": "127.0.0.0/8",
"local-address-udp-fds": ",".join([f"{i}={await fd.as_argument()}" for i, (fd, _) in addresses.items()]),
"local-address-tcp-fds": ",".join([f"{i}={await fd.as_argument()}" for i, (_, fd) in addresses.items()]),
}
if root_hints is not None:
config["hint-file"] = os.fsdecode(await thread.spit(path/'root.hints', root_hints.to_text()))
child = await thread.exec(pdns_recursor.args(*[f"--{name}={value}" for name, value in config.items()]))
nursery.start_soon(child.check)
return Powerdns()
async def nix_deploy(src: Store, dest: Store, path: StorePath) -> None:
"Deploy a StorePath from the src Store to the dest Store"
[(local_fd, dest_fd)] = await dest.thread.open_channels(1)
src_fd = local_fd.move(src.thread.task)
await _exec_nix_store_import_export(
await src.thread.clone(),
Command(src.nix.path / 'bin/nix-store', ['nix-store'], {}), src_fd,
path.closure, await dest.thread.clone(),
Command(dest.nix.path / 'bin/nix-store', ['nix-store'], {}), dest_fd)
async def from_store(cls, store: nix.Store) -> MiredoExecutables:
miredo_path = await store.realise(miredo_nixdep)
return MiredoExecutables(
run_client=Command(
miredo_path / "libexec" / "miredo" / "miredo-run-client",
["miredo-run-client"], {}),
privproc=Command(
miredo_path / "libexec" / "miredo" / "miredo-privproc",
["miredo-privproc"], {}),
)
async def _exec_nix_store_transfer_db(
src: ChildProcess, src_nix_store: Command, src_fd: FileDescriptor, closure: t.Sequence[t.Union[str, os.PathLike]],
dest: ChildProcess, dest_nix_store: Command, dest_fd: FileDescriptor,
) -> None:
"Exec nix-store to copy the Nix database for a closure between two stores"
await dest.task.inherit_fd(dest_fd).dup2(dest.stdin)
await dest_fd.close()
dest_child = await dest.exec(dest_nix_store.args("--load-db").env({'NIX_REMOTE': ''}))
await src.task.inherit_fd(src_fd).dup2(src.stdout)
await src_fd.close()
src_child = await src.exec(src_nix_store.args("--dump-db", *closure))
await src_child.check()
await dest_child.check()
async def _exec_nix_store_import_export(
src: ChildProcess, src_nix_store: Command, src_fd: FileDescriptor,
closure: t.Sequence[t.Union[str, os.PathLike]],
dest: ChildProcess, dest_nix_store: Command, dest_fd: FileDescriptor,
) -> None:
"Exec nix-store to copy a closure of paths between two stores"
await dest.task.inherit_fd(dest_fd).dup2(dest.stdin)
await dest_fd.close()
dest_child = await dest.exec(dest_nix_store.args("--import").env({'NIX_REMOTE': ''}))
await src.task.inherit_fd(src_fd).dup2(src.stdout)
await src_fd.close()
src_child = await src.exec(src_nix_store.args("--export", *closure))
await src_child.check()
await dest_child.check()
def __init__(self, task: Task, ram: RAM,
environment: t.Dict[bytes, bytes]) -> None:
self.data = environment
self.sh = Command(Path("/bin/sh"), ['sh'], {})
self.tmpdir = Path(self.get("TMPDIR", "/tmp"))
self.path = ExecutablePathCache(task, ram,
self.get("PATH", "").split(":"))
async def enter_nix_container(
src: Process,
nix: PackageClosure,
dest: Process,
dest_dir: Path,
) -> None:
"""Move `dest` into a container in `dest_dir`, deploying Nix inside.
We can then use `deploy` to deploy other things into this container,
which we can use from the `dest` process or any of its children.
"""
# we want to use our own container Nix store, not the global one on the system
if 'NIX_REMOTE' in dest.environ:
del dest.environ['NIX_REMOTE']
# copy the binaries over
await copy_tree(src, nix.closure, dest, dest_dir)
# enter the container
await dest.unshare(CLONE.NEWNS | CLONE.NEWUSER)
await dest.mount(dest_dir / "nix", "/nix", "none", MS.BIND, "")
# init the database
nix_store = Command(nix.path / 'bin/nix-store', ['nix-store'], {})
await bootstrap_nix_database(src, nix_store, nix.closure, dest, nix_store)
# add nix.path to PATH; TODO add a real API for this
dest.environ.path.paths.append(Path(nix.path / 'bin'))
async def deploy_nix_bin(store: Store, dest: Thread) -> Store:
"Deploy the Nix binaries from `store` to /nix through `dest`"
# copy the binaries over
await copy_tree(store.thread, store.nix.closure, dest, Path("/nix"))
# init the database
nix_store = Command(store.nix.path / 'bin/nix-store', ['nix-store'], {})
await bootstrap_nix_database(store.thread, nix_store, store.nix.closure,
dest, nix_store)
return Store(dest, store.nix)
async def start_powerdns(nursery, parent: Thread, path: Path, zone: dns.zone.Zone,
# tuple is (udpfd, listening tcpfd)
ipv4_sockets: t.List[t.Tuple[handle.FileDescriptor, handle.FileDescriptor]],
ipv6_sockets: t.List[t.Tuple[handle.FileDescriptor, handle.FileDescriptor]],
) -> Powerdns:
pdns_server = Command(Path("/home/sbaugh/.local/src/pdns/pdns/pdns_server"), ['pdns_server'], {})
# pdns_server = await parent.environ.which("pdns_server")
thread = await parent.fork()
# we pretend to pass addresses like 0.0.0.1 etc
# we add one so we don't pass 0.0.0.0 and make powerdns think it's bound to everything
ipv4s = {str(i+1): (udp.move(thread.task), tcp.move(thread.task))
for i, (udp, tcp) in enumerate(ipv4_sockets)}
ipv6s = {str(i+1): (udp.move(thread.task), tcp.move(thread.task))
for i, (udp, tcp) in enumerate(ipv6_sockets)}
await thread.unshare_files(going_to_exec=True)
config = {
"config-dir": os.fsdecode(path),
# TODO control-console seems to be a feature where it will listen on stdin or something?
# we should use that instead of this socketdir
"socket-dir": os.fsdecode(path),
# more logging
"loglevel": "9",
"log-dns-queries": "yes",
# backend
"launch": "bind",
"bind-config": os.fsdecode(await thread.spit(path/"named.conf",
'zone "%s" { file "%s"; };' % (
zone.origin.to_text(),
os.fsdecode(await thread.spit(path/"zone", zone.to_text()))))),
"enable-lua-records": "yes",
# relevant stuff
"local-address": ",".join(ipv4s),
"local-address-udp-fds": ",".join([f"{i}={await fd.as_argument()}" for i, (fd, _) in ipv4s.items()]),
"local-address-tcp-fds": ",".join([f"{i}={await fd.as_argument()}" for i, (_, fd) in ipv4s.items()]),
"local-ipv6": ",".join(ipv6s),
"local-ipv6-udp-fds": ",".join([f"{i}={await fd.as_argument()}" for i, (fd, _) in ipv6s.items()]),
"local-ipv6-tcp-fds": ",".join([f"{i}={await fd.as_argument()}" for i, (_, fd) in ipv6s.items()]),
}
print(config['local-address-udp-fds'])
child = await thread.exec(pdns_server.args(*[f"--{name}={value}" for name, value in config.items()]))
nursery.start_soon(child.check)
return Powerdns()
def __init__(
self,
environment: t.Dict[str, str],
path: ExecutablePathCache,
arglist_ptr: t.Optional[WrittenPointer[ArgList]] = None,
) -> None:
self.data = environment
self.sh = Command(Path("/bin/sh"), ['sh'], {})
"The POSIX-required `/bin/sh`, as a `rsyscall.Command`"
self.tmpdir = Path(self.get("TMPDIR", "/tmp"))
"`TMPDIR`, or `/tmp` if it's not set, as a `rsyscall.path.Path`"
self.path = path
self.arglist_ptr = arglist_ptr
async def exec_nix_store_import_export(
src: ChildThread,
src_nix_store: Command,
src_fd: FileDescriptor,
closure: t.List[Path],
dest: ChildThread,
dest_nix_store: Command,
dest_fd: FileDescriptor,
) -> None:
"Exec nix-store to copy a closure of paths between two stores"
await dest.unshare_files_and_replace({
dest.stdin: dest_fd,
})
await dest_fd.close()
dest_child = await dest.exec(
dest_nix_store.args("--import").env({'NIX_REMOTE': ''}))
await src.unshare_files_and_replace({
src.stdout: src_fd,
})
await src_fd.close()
src_child = await src.exec(src_nix_store.args("--export", *closure))
await src_child.check()
await dest_child.check()
async def exec_nix_store_transfer_db(
src: ChildThread,
src_nix_store: Command,
src_fd: FileDescriptor,
closure: t.List[Path],
dest: ChildThread,
dest_nix_store: Command,
dest_fd: FileDescriptor,
) -> None:
"Exec nix-store to copy the Nix database for a closure between two stores"
await dest.unshare_files_and_replace({
dest.stdin: dest_fd,
})
await dest_fd.close()
dest_child = await dest.exec(
dest_nix_store.args("--load-db").env({'NIX_REMOTE': ''}))
await src.unshare_files_and_replace({
src.stdout: src_fd,
})
await src_fd.close()
src_child = await src.exec(src_nix_store.args("--dump-db", *closure))
await src_child.check()
await dest_child.check()
async def exec_nginx(thread: ChildThread, nginx: Command, path: Path,
config: FileDescriptor,
listen_fds: t.List[FileDescriptor]) -> AsyncChildProcess:
nginx_fds = [fd.maybe_copy(thread.task) for fd in listen_fds]
config_fd = config.maybe_copy(thread.task)
await thread.unshare_files()
if nginx_fds:
nginx_var = ";".join([str(await fd.as_argument())
for fd in nginx_fds]) + ';'
else:
nginx_var = ""
await thread.mkdir(path / "logs")
child = await thread.exec(
nginx.env(NGINX=nginx_var).args(
"-p", path, "-c",
f"/proc/self/fd/{await config_fd.as_argument()}"))
return child
async def which(self, name: str) -> Command:
"Locate an executable with this name on PATH; throw ExecutableNotFound on failure"
try:
path = self.path_cache[name]
except KeyError:
nameptr = await self.task.ptr(name)
# do the lookup for 64 paths at a time, that seems like a good batching number
for paths in chunks(self.paths, 64):
results = await run_all(
*[self._check(path, nameptr) for path in paths])
for path, result in zip(paths, results):
if result:
# path is set as the loop variable; python has no scope
# so we can just break out and use it.
break
if result:
break
else:
raise ExecutableNotFound(name)
self.path_cache[name] = path
return Command(path / name, [name], {})
async def enter_nix_container(store: Store, dest: Thread,
dest_dir: Path) -> Store:
"""Move `dest` into a container in `dest_dir`, deploying Nix inside and returning the Store thus-created
We can then use Store.realise to deploy other things into this container,
which we can use from the `dest` thread or any of its children.
"""
# we want to use our own container Nix store, not the global one on the system
if 'NIX_REMOTE' in dest.environ:
del dest.environ['NIX_REMOTE']
# copy the binaries over
await copy_tree(store.thread, store.nix.closure, dest, dest_dir)
# enter the container
await dest.unshare(CLONE.NEWNS | CLONE.NEWUSER)
await dest.mount(dest_dir / "nix", "/nix", "none", MS.BIND, "")
# init the database
nix_store = Command(store.nix.path / 'bin/nix-store', ['nix-store'], {})
await bootstrap_nix_database(store.thread, nix_store, store.nix.closure,
dest, nix_store)
return Store(dest, store.nix)
def proxy_command(self, command: Command) -> SSHCommand:
return self.ssh_options({'ProxyCommand': command.in_shell_form()})
async def from_store(cls, store: nix.Store) -> RsyscallServerExecutable:
rsyscall_path = await store.realise(nix.rsyscall)
server = Command(
rsyscall_path / "libexec" / "rsyscall" / "rsyscall-server",
['rsyscall-server'], {})
return cls(server)
async def bin(self, store_path: StorePath, name: str) -> Command:
"Realise this StorePath, then return a Command for the binary named `name`"
path = await self.realise(store_path)
return Command(path / "bin" / name, [name], {})
async def from_store(cls, store: nix.Store) -> SSHDExecutables:
ssh_path = await store.realise(openssh)
ssh_keygen = Command(ssh_path/"bin"/"ssh-keygen", ["ssh-keygen"], {})
sshd = SSHDCommand.make(ssh_path/"bin"/"sshd")
return SSHDExecutables(ssh_keygen, sshd)
class TestMail(TrioTestCase):
async def asyncSetUp(self) -> None:
self.process = local.process
self.tmpdir = await self.process.mkdtemp("test_mail")
self.path = self.tmpdir.path
await update_symlink(
self.process, await
self.process.ram.ptr(self.tmpdir.parent / "test_mail.current"),
self.path)
smtp_sock = await self.process.task.socket(AF.INET, SOCK.STREAM)
await smtp_sock.bind(await self.process.ram.ptr(
SockaddrIn(3000, "127.0.0.1")))
await smtp_sock.listen(10)
self.smtpd = await start_smtpd(
self.nursery, self.process, await
self.process.mkdir(self.path / "smtpd"), smtp_sock)
self.maildir = await Maildir.make(self.process, self.path / "mail")
self.dovecot = await start_dovecot(
self.nursery, self.process, await
self.process.mkdir(self.path / "dovecot"),
self.smtpd.lmtp_listener, self.maildir)
smtpctl = await self.process.environ.which("smtpctl")
self.sendmail = Command(smtpctl.executable_path, [b'sendmail'],
{'SMTPD_CONFIG_FILE': self.smtpd.config_file})
self.inty = await Inotify.make(self.process)
async def asyncTearDown(self) -> None:
await self.tmpdir.cleanup()
async def send_email(self, from_: str, to: str, subject: str,
msg: str) -> None:
process = await self.process.fork()
await process.unshare_files()
fd = await process.task.memfd_create(await
process.ram.ptr(Path('message')))
msg = f'From: {from_}\nSubject: {subject}\nTo: {to}\n\n' + msg
await process.spit(fd, msg)
await fd.lseek(0, SEEK.SET)
await process.stdin.replace_with(fd)
child = await process.exec(self.sendmail.args('-t'))
await child.check()
async def test_sendmail(self) -> None:
watch = await self.inty.add(self.maildir.new(), IN.MOVED_TO)
# TODO sigh, opensmtpd isn't validating the From field...
from_ = 'from@localhost'
to = 'sbaugh@localhost'
subject = 'Subjective'
msg = 'Hello me!\n'
await self.send_email(from_, to, subject, msg)
event = await watch.wait_until_event(IN.MOVED_TO)
if event.name is None:
raise Exception("event has no name??")
mailfd = await self.process.task.open(
await self.process.ram.ptr(self.maildir.new() / event.name),
O.RDONLY)
data = await self.process.read_to_eof(mailfd)
message = email.message_from_bytes(data)
self.assertEqual(from_, message['From'])
self.assertEqual(to, message['To'])
self.assertEqual(subject, message['Subject'])
self.assertEqual(msg, message.get_payload())
# So I need to set up proper DNS stuff.
# Which... I can do by running my own DNS server :)
# aha, okay, so I could have a DNS server which,
# automatically forwards the requests to the NS server in the record
async def test_mail_tester(self) -> None:
from_ = '*****@*****.**'
to = '*****@*****.**'
subject = 'Subjective'
msg = 'Hello me!\n'
await self.send_email(from_, to, subject, msg)
await trio.sleep(9999)
async def test_async(self) -> None:
command = Command(self.path/self.stub_name, [self.stub_name], {})
child = await self.thread.exec(command)
self.nursery.start_soon(child.check)
argv, new_thread = await self.server.accept()
await do_async_things(self, new_thread.epoller, new_thread)
async def test_exit(self) -> None:
command = Command(self.path/self.stub_name, [self.stub_name], {})
child = await self.thread.exec(command)
self.nursery.start_soon(child.check)
argv, new_thread = await self.server.accept()
await new_thread.exit(0)
async def asyncSetUp(self) -> None:
self.local = local.thread
path = await stdin_bootstrap_path_from_store(local_store)
self.command = Command(path, ['rsyscall-stdin-bootstrap'], {})
self.local_child, self.remote = await stdin_bootstrap(
self.local, self.command)
def bin(self, name: str) -> Command:
return Command(self/"bin"/name, [name], {})
async def test_exit(self) -> None:
command = Command(self.tmpdir / self.stub_name, [self.stub_name], {})
child = await self.exec_process.exec(command)
self.nursery.start_soon(child.check)
argv, new_process = await self.server.accept()
await new_process.exit(0)