def test_function_call(self):
self.assert_nix(Call(RawValue("fun_call"), {"a": "b"}),
'( fun_call { a = "b"; } )')
self.assert_nix(
Call(RawValue("multiline_call"), {"a": "b"}),
'(\n multiline_call\n {\n a = "b";\n }\n)',
maxwidth=0,
)
def test_invalid(self):
self.assertRaises(ValueError, nixmerge, [123], {"a": 456})
self.assertRaises(ValueError, nixmerge, "a", "b")
self.assertRaises(ValueError, nixmerge, 123, 456)
self.assertRaises(ValueError, nixmerge, RawValue("a"), RawValue("b"))
self.assertRaises(ValueError, nixmerge, Function("aaa", {"a": 1}),
Function("ccc", {"b": 2}))
self.assertRaises(ValueError, nixmerge, Function("aaa", {"a": 1}),
{"b": 2})
def _eval_flags(self, exprs):
flags = self._nix_path_flags()
args = {key: RawValue(val) for key, val in self.args.iteritems()}
exprs_ = [RawValue(x) if x[0] == '<' else x for x in exprs]
flags.extend(
["--arg", "networkExprs", py2nix(exprs_, inline=True),
"--arg", "args", py2nix(args, inline=True),
"--argstr", "uuid", self.uuid,
"<nixops/eval-machine-info.nix>"])
return flags
def _configure_initial_nix(self,
uefi: bool,
instance_id: Optional[int] = None):
self.log_start("generating the initial configuration... ")
# 1. We generate the HW configuration and the standard configuration.
out = self.run_command("nixos-generate-config --root /mnt",
capture_stdout=True)
# 2. We will override the configuration.nix
nixos_cfg = {
"imports": [RawValue("./hardware-configuration.nix")],
("boot", "kernelParams"): ["console=ttyS0"],
("services", "openssh", "enable"):
True,
("services", "qemuGuest", "enable"):
True,
("systemd", "services", "qemu-guest-agent", "serviceConfig", "RuntimeDirectory"):
"qemu-ga",
("systemd", "services", "qemu-guest-agent", "serviceConfig", "ExecStart"):
RawValue(
"lib.mkForce \"\\${pkgs.qemu.ga}/bin/qemu-ga -t /var/run/qemu-ga\""
),
("services", "getty", "autologinUser"):
"root",
("networking", "firewall", "allowedTCPPorts"): [22],
("users", "users", "root"): {
("openssh", "authorizedKeys", "keys"): [self.public_host_key],
("initialPassword"): ""
},
("users", "mutableUsers"):
False
}
if uefi:
nixos_cfg[("boot", "loader")] = {
("efi", "canTouchEfiVariables"): True,
("systemd-boot", "enable"): True
}
else:
# Use nix2py to read self.fs_info.
nixos_cfg[("boot", "loader", "grub", "devices")] = ["/dev/sda"]
nixos_initial_postinstall_conf = py2nix(
Function("{ config, pkgs, lib, ... }", nixos_cfg))
self.run_command(
f"cat <<EOF > /mnt/etc/nixos/configuration.nix\n{nixos_initial_postinstall_conf}\nEOF"
)
self.run_command("echo preinstall > /mnt/.install_status")
self.log_end("initial configuration generated")
self.log_start("installing NixOS... ")
out = self.run_command("nixos-install --no-root-passwd",
capture_stdout=True)
self.log_end("NixOS installed")
self.run_command("echo installed > /mnt/.install_status")
def test_list_compound(self):
self.assert_nix(
[Function("123 //", 456, call=True),
RawValue("a b c")], '[ (123 // 456) (a b c) ]')
self.assert_nix([
RawValue("a b c"), {
'cde':
[RawValue("1,2,3"),
RawValue("4 5 6"),
RawValue("7\n8\n9")]
}
], '[ (a b c) { cde = [ 1,2,3 (4 5 6) (7\n8\n9) ]; } ]')
def get_physical_backup_spec(self, backupid):
val = {}
if backupid in self.backups:
for dev, snap in self.backups[backupid].items():
val[dev] = {
'snapshot': Call(RawValue("pkgs.lib.mkOverride 10"), snap)
}
val = {('deployment', 'gce', 'blockDeviceMapping'): val}
else:
val = RawValue(
"{{}} /* No backup found for id '{0}' */".format(backupid))
return Function("{ config, pkgs, ... }", val)
def get_physical_spec(self):
block_device_mapping = {}
for k, v in self.block_device_mapping.items():
if (v.get('encrypt', False)
and v.get('passphrase', "") == ""
and v.get('generatedKey', "") != ""):
block_device_mapping[k] = {
'passphrase': Call(RawValue("pkgs.lib.mkOverride 10"), v['generatedKey']),
}
return {
'imports': [
RawValue("<nixpkgs/nixos/modules/virtualisation/google-compute-config.nix>")
],
('deployment', 'gce', 'blockDeviceMapping'): block_device_mapping,
}
def _eval_flags(self, exprs):
flags = self._nix_path_flags()
args = {key: RawValue(val) for key, val in self.args.iteritems()}
flags.extend([
"--arg", "networkExprs",
py2nix(exprs, inline=True), "--arg", "args",
py2nix(args, inline=True), "--argstr", "uuid", self.uuid,
"--show-trace", "<nixops/eval-machine-info.nix>"
])
return flags
def test_functions(self):
self.assert_nix(Function("Aaa", RawValue("bbb")), "Aaa: bbb")
self.assert_nix(Function("{ ... }", [1, 2, 3]), "{ ... }: [ 1 2 3 ]")
self.assert_nix(Function("{ ... }", "a\nb\nc\n"), r'{ ... }: "a\nb\nc\n"')
self.assert_nix(
Function("{ ... }", "a\nb\nc\n"),
"{ ... }: ''\n a\n b\n c\n''",
maxwidth=0,
)
self.assert_nix(
Function("xxx", {"a": {"b": "c"}}), 'xxx: {\n a.b = "c";\n}', maxwidth=0
)
def get_physical_spec(self):
return Function(
"{ ... }", {
'imports':
[RawValue('<nixpkgs/nixos/modules/profiles/qemu-guest.nix>')],
('boot', 'loader', 'grub', 'device'):
'nodev',
('fileSystems', '/'): {
'device': '/dev/sda1',
'fsType': 'ext4'
},
('users', 'extraUsers', 'root', 'openssh', 'authorizedKeys', 'keys'):
[self.depl.active_resources.get('ssh-key').public_key],
})
def get_physical_spec(self) -> Function:
def prefix_len(netmask):
return bin(int(codecs.encode(socket.inet_aton(netmask), "hex"),
16)).count("1")
networking = {
"defaultGateway":
self.default_gateway,
"nameservers": ["67.207.67.2",
"67.207.67.3"], # default provided by DO
("interfaces", "ens3", "ipv4", "addresses"): [{
"address":
self.public_ipv4,
"prefixLength":
prefix_len(self.netmask)
}],
}
if self.public_ipv6:
networking[("interfaces", "ens3", "ipv6", "addresses")] = [{
"address":
self.public_ipv6["address"],
"prefixLength":
self.public_ipv6["prefixLength"],
}]
if self.default_gateway6:
networking["defaultGateway6"] = self.default_gateway6
return Function(
"{ ... }",
{
"imports":
[RawValue("<nixpkgs/nixos/modules/profiles/qemu-guest.nix>")],
"networking":
networking,
(
"boot",
"loader",
"grub",
"device",
):
"nodev", # keep ubuntu bootloader?
("fileSystems", "/"): {
"device": "/dev/vda1",
"fsType": "ext4"
},
("users", "extraUsers", "root", "openssh", "authorizedKeys", "keys"):
[self.get_ssh_key_resource().public_key],
},
)
def destroy_resources(self, include=[], exclude=[], wipe=False):
"""Destroy all active or obsolete resources."""
with self._get_deployment_lock():
for r in self.resources.itervalues():
r._destroyed_event = threading.Event()
r._errored = False
for rev_dep in r.destroy_before(self.resources.itervalues()):
try:
rev_dep._wait_for.append(r)
except AttributeError:
rev_dep._wait_for = [ r ]
def worker(m):
try:
if not should_do(m, include, exclude): return
try:
for dep in m._wait_for:
dep._destroyed_event.wait()
# !!! Should we print a message here?
if dep._errored:
m._errored = True
return
except AttributeError:
pass
if m.destroy(wipe=wipe): self.delete_resource(m)
except:
m._errored = True
raise
finally:
m._destroyed_event.set()
nixops.parallel.run_tasks(nr_workers=-1, tasks=self.resources.values(), worker_fun=worker)
# Remove the destroyed machines from the rollback profile.
# This way, a subsequent "nix-env --delete-generations old" or
# "nix-collect-garbage -d" will get rid of the machine
# configurations.
if self.rollback_enabled: # and len(self.active) == 0:
profile = self.create_profile()
attrs = {m.name:
Call(RawValue("builtins.storePath", m.cur_toplevel))
for m in self.active.itervalues() if m.cur_toplevel}
if subprocess.call(
["nix-env", "-p", profile, "--set", "*", "-I", "nixops=" + self.expr_path,
"-f", "<nixops/update-profile.nix>",
"--arg", "machines", py2nix(attrs, inline=True)]) != 0:
raise Exception("cannot update profile ‘{0}’".format(profile))
def get_physical_spec(self):
prefixLength = bin(int(socket.inet_aton(self.netmask).encode('hex'), 16)).count('1')
return Function("{ ... }", {
'imports': [ RawValue('<nixpkgs/nixos/modules/profiles/qemu-guest.nix>') ],
'networking': {
'defaultGateway': self.default_gateway,
'nameservers': ['8.8.8.8'], # default provided by DO
('interfaces', 'enp0s3'): {
'ip4': [{"address": self.public_ipv4, 'prefixLength': prefixLength}],
},
},
('boot', 'loader', 'grub', 'device'): 'nodev', # keep ubuntu bootloader?
('fileSystems', '/'): { 'device': '/dev/vda1', 'fsType': 'ext4'},
('users', 'extraUsers', 'root', 'openssh', 'authorizedKeys', 'keys'): [self.depl.active_resources.get('ssh-key').public_key],
})
def get_physical_spec(self):
return Function(
"{ ... }", {
'imports':
[RawValue('<nixpkgs/nixos/modules/profiles/qemu-guest.nix>')],
('config', 'boot', 'initrd', 'availableKernelModules'):
["ata_piix", "uhci_hcd", "virtio_pci", "sr_mod", "virtio_blk"],
('config', 'boot', 'loader', 'grub', 'device'):
'/dev/vda',
('config', 'fileSystems', '/'): {
'device': '/dev/vda1',
'fsType': 'btrfs'
},
('config', 'users', 'extraUsers', 'root', 'openssh', 'authorizedKeys', 'keys'):
[self._ssh_public_key]
})
def test_list_compound(self):
self.assert_nix([Call(RawValue("123 //"), 456),
RawValue("a b c")], '[ (( 123 // 456 )) (a b c) ]')
self.assert_nix([
RawValue("a b c"), {
'cde':
[RawValue("1,2,3"),
RawValue("4 5 6"),
RawValue("7\n8\n9")]
}
], '[ (a b c) { cde = [ 1,2,3 (4 5 6) (7\n8\n9) ]; } ]')
def destroy_resources(self, include=[], exclude=[], wipe=False):
"""Destroy all active and obsolete resources."""
with self._get_deployment_lock():
self._destroy_resources(include, exclude, wipe)
# Remove the destroyed machines from the rollback profile.
# This way, a subsequent "nix-env --delete-generations old" or
# "nix-collect-garbage -d" will get rid of the machine
# configurations.
if self.rollback_enabled: # and len(self.active) == 0:
profile = self.create_profile()
attrs = {m.name:
Call(RawValue("builtins.storePath"), m.cur_toplevel)
for m in self.active.itervalues() if m.cur_toplevel}
if subprocess.call(
["nix-env", "-p", profile, "--set", "*", "-I", "nixops=" + self.expr_path,
"-f", "<nixops/update-profile.nix>",
"--arg", "machines", py2nix(attrs, inline=True)]) != 0:
raise Exception("cannot update profile ‘{0}’".format(profile))
def get_physical_spec(self):
return {'imports': [RawValue('<nixops/virtualbox-image-nixops.nix>')]}
def do_machine(m):
defn = self.definitions[m.name]
attrs_list = attrs_per_resource[m.name]
# Emit configuration to realise encrypted peer-to-peer links.
for m2 in active_resources.itervalues():
ip = m.address_to(m2)
if ip:
hosts[m.name][ip] += [m2.name, m2.name + "-unencrypted"]
# Always use the encrypted/unencrypted suffixes for aliases rather
# than for the canonical name!
hosts[m.name]["127.0.0.1"].append(m.name + "-encrypted")
for m2_name in defn.encrypted_links_to:
if m2_name not in active_machines:
raise Exception("‘deployment.encryptedLinksTo’ in machine ‘{0}’ refers to an unknown machine ‘{1}’"
.format(m.name, m2_name))
m2 = active_machines[m2_name]
# Don't create two tunnels between a pair of machines.
if m.name in self.definitions[m2.name].encrypted_links_to and m.name >= m2.name:
continue
local_ipv4 = index_to_private_ip(m.index)
remote_ipv4 = index_to_private_ip(m2.index)
local_tunnel = 10000 + m2.index
remote_tunnel = 10000 + m.index
attrs_list.append({
('networking', 'p2pTunnels', 'ssh', m2.name): {
'target': '{0}-unencrypted'.format(m2.name),
'targetPort': m2.ssh_port,
'localTunnel': local_tunnel,
'remoteTunnel': remote_tunnel,
'localIPv4': local_ipv4,
'remoteIPv4': remote_ipv4,
'privateKey': '/root/.ssh/id_charon_vpn',
}
})
# FIXME: set up the authorized_key file such that ‘m’
# can do nothing more than create a tunnel.
authorized_keys[m2.name].append(m.public_vpn_key)
kernel_modules[m.name].add('tun')
kernel_modules[m2.name].add('tun')
hosts[m.name][remote_ipv4] += [m2.name, m2.name + "-encrypted"]
hosts[m2.name][local_ipv4] += [m.name, m.name + "-encrypted"]
trusted_interfaces[m.name].add('tun' + str(local_tunnel))
trusted_interfaces[m2.name].add('tun' + str(remote_tunnel))
private_ipv4 = m.private_ipv4
if private_ipv4:
attrs_list.append({
('networking', 'privateIPv4'): private_ipv4
})
public_ipv4 = m.public_ipv4
if public_ipv4:
attrs_list.append({
('networking', 'publicIPv4'): public_ipv4
})
public_vpn_key = m.public_vpn_key
if public_vpn_key:
attrs_list.append({
('networking', 'vpnPublicKey'): public_vpn_key
})
# Set system.stateVersion if the Nixpkgs version supports it.
if nixops.util.parse_nixos_version(defn.config["nixosVersion"]) >= ["15", "09"]:
attrs_list.append({
('system', 'stateVersion'): Call(RawValue("lib.mkDefault"), m.state_version or '14.12')
})
if self.nixos_version_suffix:
attrs_list.append({
('system', 'nixosVersionSuffix'): self.nixos_version_suffix
})
def emit_resource(r):
config = []
config.extend(attrs_per_resource[r.name])
if is_machine(r):
# Sort the hosts by its canonical host names.
sorted_hosts = sorted(hosts[r.name].iteritems(),
key=lambda item: item[1][0])
# Just to remember the format:
# ip_address canonical_hostname [aliases...]
extra_hosts = [
"{0} {1}".format(ip, ' '.join(names))
for ip, names in sorted_hosts
]
if authorized_keys[r.name]:
config.append({
('users', 'extraUsers', 'root'): {
('openssh', 'authorizedKeys', 'keys'):
authorized_keys[r.name]
},
('services', 'openssh'): {
'extraConfig': "PermitTunnel yes\n"
},
})
config.append({
('boot', 'kernelModules'):
list(kernel_modules[r.name]),
('networking', 'firewall'): {
'trustedInterfaces': list(trusted_interfaces[r.name])
},
('networking', 'extraHosts'):
'\n'.join(extra_hosts) + "\n"
})
# Add SSH public host keys for all machines in network
for m2 in active_machines.itervalues():
if hasattr(m2, 'public_host_key'):
# Using references to files in same tempdir for now, until NixOS has support
# for adding the keys directly as string. This way at least it is compatible
# with older versions of NixOS as well.
# TODO: after reasonable amount of time replace with string option
config.append({
('services', 'openssh', 'knownHosts', m2.name): {
'hostNames': [
m2.name + "-unencrypted",
m2.name + "-encrypted", m2.name
],
'publicKeyFile':
RawValue("./{0}.public_host_key".format(
m2.name)),
}
})
merged = reduce(nixmerge, config) if len(config) > 0 else {}
physical = r.get_physical_spec()
if len(merged) == 0 and len(physical) == 0:
return {}
else:
return r.prefix_definition({
r.name:
Function("{ config, pkgs, ... }", {
'config': merged,
'imports': [physical],
})
})
def get_physical_spec(self):
return {"imports": [RawValue("<virtualbox-image-nixops.nix>")]}
def test_raw_value(self):
self.assert_nix({'a': RawValue('import <something>')},
'{ a = import <something>; }')
self.assert_nix([RawValue("!")], '[ ! ]')
def eval(
# eval-machine-info args
networkExpr: NetworkFile, # Flake conditional
uuid: str,
deploymentName: str,
networkExprs: List[str] = [],
args: Dict[str, str] = {},
pluginNixExprs: List[str] = [],
checkConfigurationOptions: bool = True,
# Extend internal defaults
nix_path: List[str] = [],
# nix-instantiate args
nix_args: Dict[str, Any] = {},
attr: Optional[str] = None,
extra_flags: List[str] = [],
# Non-propagated args
stderr: Optional[TextIO] = None,
) -> Any:
exprs: List[str] = list(networkExprs)
if not networkExpr.is_flake:
exprs.append(networkExpr.network)
argv: List[str] = (
[
"nix-instantiate", "--eval-only", "--json", "--strict",
"--show-trace"
] + [os.path.join(get_expr_path(), "eval-machine-info.nix")] +
["-I", "nixops=" + get_expr_path()] + [
"--arg",
"networkExprs",
py2nix([RawValue(x) if x[0] == "<" else x for x in exprs]),
] + [
"--arg",
"args",
py2nix({key: RawValue(val)
for key, val in args.items()},
inline=True),
] + ["--argstr", "uuid", uuid] +
["--argstr", "deploymentName", deploymentName] +
["--arg", "pluginNixExprs",
py2nix(pluginNixExprs)] + [
"--arg", "checkConfigurationOptions",
json.dumps(checkConfigurationOptions)
] +
list(itertools.chain(*[["-I", x]
for x in (nix_path + pluginNixExprs)])) +
extra_flags)
for k, v in nix_args.items():
argv.extend(["--arg", k, py2nix(v, inline=True)])
if attr:
argv.extend(["-A", attr])
if networkExpr.is_flake:
argv.extend(["--allowed-uris", get_expr_path()])
argv.extend(["--argstr", "flakeUri", networkExpr.network])
try:
ret = subprocess.check_output(argv, stderr=stderr, text=True)
return json.loads(ret)
except OSError as e:
raise Exception("unable to run ‘nix-instantiate’: {0}".format(e))
except subprocess.CalledProcessError:
raise NixEvalError
def test_raw_value(self):
self.assert_nix({"a": RawValue("import <something>")},
"{ a = import <something>; }")
self.assert_nix([RawValue("!")], "[ ! ]")
def get_physical_spec(self) -> Dict[Any, Any]:
ipv4 = [{"address": self.public_ipv4, "prefixLength": 32}]
ipv6 = [{"address": self.public_ipv6[:-3], "prefixLength": 64}]
for addr in self.ip_addresses.values():
try:
socket.inet_pton(socket.AF_INET, addr)
ipv4.append({"address": addr, "prefixLength": 32})
except socket.error: # not a valid address ipv4
ipv6.append({"address": addr, "prefixLength": 64})
def get_interface_name(i: int) -> str:
return f"ens{10+i}" if self.legacy_if_scheme else f"enp{7+i}s0"
spec = {
"imports":
[RawValue("<nixpkgs/nixos/modules/profiles/qemu-guest.nix>")],
("boot", "loader", "grub", "device"):
"nodev",
("fileSystems", "/"): {
"device": "/dev/sda1",
"fsType": "ext4"
},
**{("fileSystems", v["mountPoint"]): {
"fsType": v["fsType"],
"device": v["device"],
}
for k, v in self.volumes.items() if v["mountPoint"]},
# Hetzner Cloud networking defaults
("networking", "defaultGateway"):
"172.31.1.1",
("networking", "nameservers"): [
"213.133.98.98",
"213.133.99.99",
"213.133.100.100",
],
(
"networking",
"interfaces",
"ens3" if self.legacy_if_scheme else "enp1s0",
): {
("ipv4", "addresses"): ipv4,
("ipv6", "addresses"): ipv6,
"useDHCP": True,
},
("users", "extraUsers", "root", "openssh", "authorizedKeys", "keys"):
[self.public_client_key],
}
for i, v in enumerate(self.server_networks.values()):
private_ipv4_addresses = [{
"address": addr,
"prefixLength": 32
} for addr in [v["privateIpAddress"]] + v["aliasIpAddresses"]]
spec[("networking", "interfaces", get_interface_name(i))] = {
("ipv4", "addresses"): private_ipv4_addresses,
"useDHCP": True,
}
for v in self.volumes.values():
if v["fsType"] == "xfs":
spec[("boot", "kernelModules")] = ["xfs"]
break
return spec