def test_nested_functions(self):
match = dedent('''
{ config, pkgs, ... }: {
a.b.c = 1;
b.c.d = 2;
d.e = [ "e" "f" ];
e = f: {
x = ''
aaa
bbb
ccc
'';
};
}
''').strip()
self.assert_nix(Function(
"{ config, pkgs, ... }", {
'a': {
'b': {
'c': 1
}
},
'b': {
'c': {
'd': 2
}
},
'd': {
'e': ['e', 'f']
},
'e': Function('f', {'x': "aaa\nbbb\nccc\n"})
}),
match,
maxwidth=26)
def test_nested_functions(self):
match = dedent(
"""
{ config, pkgs, ... }: {
a.b.c = 1;
b.c.d = 2;
d.e = [ "e" "f" ];
e = f: {
x = ''
aaa
bbb
ccc
'';
};
}
"""
).strip()
self.assert_nix(
Function(
"{ config, pkgs, ... }",
{
"a": {"b": {"c": 1}},
"b": {"c": {"d": 2}},
"d": {"e": ["e", "f"]},
"e": Function("f", {"x": "aaa\nbbb\nccc\n"}),
},
),
match,
maxwidth=26,
)
def test_invalid(self):
self.assertRaises(ValueError, nixmerge, [123], {"a": 456})
self.assertRaises(ValueError, nixmerge, "a", "b")
self.assertRaises(ValueError, nixmerge, 123, 456)
self.assertRaises(ValueError, nixmerge, RawValue("a"), RawValue("b"))
self.assertRaises(ValueError, nixmerge, Function("aaa", {"a": 1}),
Function("ccc", {"b": 2}))
self.assertRaises(ValueError, nixmerge, Function("aaa", {"a": 1}),
{"b": 2})
def test_functions(self):
self.assert_nix(Function("Aaa", RawValue("bbb")), "Aaa: bbb")
self.assert_nix(Function("{ ... }", [1, 2, 3]), "{ ... }: [ 1 2 3 ]")
self.assert_nix(Function("{ ... }", "a\nb\nc\n"), r'{ ... }: "a\nb\nc\n"')
self.assert_nix(
Function("{ ... }", "a\nb\nc\n"),
"{ ... }: ''\n a\n b\n c\n''",
maxwidth=0,
)
self.assert_nix(
Function("xxx", {"a": {"b": "c"}}), 'xxx: {\n a.b = "c";\n}', maxwidth=0
)
def destroy_resources(self, include=[], exclude=[], wipe=False):
"""Destroy all active or obsolete resources."""
with self._get_deployment_lock():
def worker(m):
if not should_do(m, include, exclude): return
if m.destroy(wipe=wipe): self.delete_resource(m)
nixops.parallel.run_tasks(nr_workers=-1,
tasks=self.resources.values(),
worker_fun=worker)
# Remove the destroyed machines from the rollback profile.
# This way, a subsequent "nix-env --delete-generations old" or
# "nix-collect-garbage -d" will get rid of the machine
# configurations.
if self.rollback_enabled: # and len(self.active) == 0:
profile = self.create_profile()
attrs = {
m.name: Function("builtins.storePath",
m.cur_toplevel,
call=True)
for m in self.active.itervalues() if m.cur_toplevel
}
if subprocess.call([
"nix-env", "-p", profile, "--set", "*", "-I",
"nixops=" + self.expr_path, "-f",
"<nixops/update-profile.nix>", "--arg", "machines",
py2nix(attrs, inline=True)
]) != 0:
raise Exception("cannot update profile ‘{0}’".format(profile))
def emit_resource(r):
config = []
config.extend(attrs_per_resource[r.name])
if is_machine(r):
# Sort the hosts by its canonical host names.
sorted_hosts = sorted(hosts[r.name].iteritems(),
key=lambda item: item[1][0])
# Just to remember the format:
# ip_address canonical_hostname [aliases...]
extra_hosts = ["{0} {1}".format(ip, ' '.join(names))
for ip, names in sorted_hosts]
if authorized_keys[r.name]:
config.append({
('users', 'extraUsers', 'root'): {
('openssh', 'authorizedKeys', 'keys'): authorized_keys[r.name]
},
('services', 'openssh'): {
'extraConfig': "PermitTunnel yes\n"
},
})
config.append({
('boot', 'kernelModules'): list(kernel_modules[r.name]),
('networking', 'firewall'): {
'trustedInterfaces': list(trusted_interfaces[r.name])
},
('networking', 'extraHosts'): '\n'.join(extra_hosts) + "\n"
})
# Add SSH public host keys for all machines in network.
for m2 in active_machines.itervalues():
if hasattr(m2, 'public_host_key') and m2.public_host_key:
# Using references to files in same tempdir for now, until NixOS has support
# for adding the keys directly as string. This way at least it is compatible
# with older versions of NixOS as well.
# TODO: after reasonable amount of time replace with string option
config.append({
('services', 'openssh', 'knownHosts', m2.name): {
'hostNames': [m2.name + "-unencrypted",
m2.name + "-encrypted",
m2.name],
'publicKey': m2.public_host_key,
}
})
merged = reduce(nixmerge, config) if len(config) > 0 else {}
physical = r.get_physical_spec()
if len(merged) == 0 and len(physical) == 0:
return {}
else:
return r.prefix_definition({
r.name: Function("{ config, lib, pkgs, ... }", {
'config': merged,
'imports': [physical],
})
})
def get_physical_backup_spec(self, backupid):
val = {}
if backupid in self.backups:
for dev, snap in self.backups[backupid].items():
val[dev] = {
'snapshot': Call(RawValue("pkgs.lib.mkOverride 10"), snap)
}
val = {('deployment', 'gce', 'blockDeviceMapping'): val}
else:
val = RawValue(
"{{}} /* No backup found for id '{0}' */".format(backupid))
return Function("{ config, pkgs, ... }", val)
def _configure_initial_nix(self,
uefi: bool,
instance_id: Optional[int] = None):
self.log_start("generating the initial configuration... ")
# 1. We generate the HW configuration and the standard configuration.
out = self.run_command("nixos-generate-config --root /mnt",
capture_stdout=True)
# 2. We will override the configuration.nix
nixos_cfg = {
"imports": [RawValue("./hardware-configuration.nix")],
("boot", "kernelParams"): ["console=ttyS0"],
("services", "openssh", "enable"):
True,
("services", "qemuGuest", "enable"):
True,
("systemd", "services", "qemu-guest-agent", "serviceConfig", "RuntimeDirectory"):
"qemu-ga",
("systemd", "services", "qemu-guest-agent", "serviceConfig", "ExecStart"):
RawValue(
"lib.mkForce \"\\${pkgs.qemu.ga}/bin/qemu-ga -t /var/run/qemu-ga\""
),
("services", "getty", "autologinUser"):
"root",
("networking", "firewall", "allowedTCPPorts"): [22],
("users", "users", "root"): {
("openssh", "authorizedKeys", "keys"): [self.public_host_key],
("initialPassword"): ""
},
("users", "mutableUsers"):
False
}
if uefi:
nixos_cfg[("boot", "loader")] = {
("efi", "canTouchEfiVariables"): True,
("systemd-boot", "enable"): True
}
else:
# Use nix2py to read self.fs_info.
nixos_cfg[("boot", "loader", "grub", "devices")] = ["/dev/sda"]
nixos_initial_postinstall_conf = py2nix(
Function("{ config, pkgs, lib, ... }", nixos_cfg))
self.run_command(
f"cat <<EOF > /mnt/etc/nixos/configuration.nix\n{nixos_initial_postinstall_conf}\nEOF"
)
self.run_command("echo preinstall > /mnt/.install_status")
self.log_end("initial configuration generated")
self.log_start("installing NixOS... ")
out = self.run_command("nixos-install --no-root-passwd",
capture_stdout=True)
self.log_end("NixOS installed")
self.run_command("echo installed > /mnt/.install_status")
def test_list_compound(self):
self.assert_nix(
[Function("123 //", 456, call=True),
RawValue("a b c")], '[ (123 // 456) (a b c) ]')
self.assert_nix([
RawValue("a b c"), {
'cde':
[RawValue("1,2,3"),
RawValue("4 5 6"),
RawValue("7\n8\n9")]
}
], '[ (a b c) { cde = [ 1,2,3 (4 5 6) (7\n8\n9) ]; } ]')
def get_physical_spec(self):
return Function(
"{ ... }", {
'imports':
[RawValue('<nixpkgs/nixos/modules/profiles/qemu-guest.nix>')],
('boot', 'loader', 'grub', 'device'):
'nodev',
('fileSystems', '/'): {
'device': '/dev/sda1',
'fsType': 'ext4'
},
('users', 'extraUsers', 'root', 'openssh', 'authorizedKeys', 'keys'):
[self.depl.active_resources.get('ssh-key').public_key],
})
def get_physical_spec(self) -> Function:
def prefix_len(netmask):
return bin(int(codecs.encode(socket.inet_aton(netmask), "hex"),
16)).count("1")
networking = {
"defaultGateway":
self.default_gateway,
"nameservers": ["67.207.67.2",
"67.207.67.3"], # default provided by DO
("interfaces", "ens3", "ipv4", "addresses"): [{
"address":
self.public_ipv4,
"prefixLength":
prefix_len(self.netmask)
}],
}
if self.public_ipv6:
networking[("interfaces", "ens3", "ipv6", "addresses")] = [{
"address":
self.public_ipv6["address"],
"prefixLength":
self.public_ipv6["prefixLength"],
}]
if self.default_gateway6:
networking["defaultGateway6"] = self.default_gateway6
return Function(
"{ ... }",
{
"imports":
[RawValue("<nixpkgs/nixos/modules/profiles/qemu-guest.nix>")],
"networking":
networking,
(
"boot",
"loader",
"grub",
"device",
):
"nodev", # keep ubuntu bootloader?
("fileSystems", "/"): {
"device": "/dev/vda1",
"fsType": "ext4"
},
("users", "extraUsers", "root", "openssh", "authorizedKeys", "keys"):
[self.get_ssh_key_resource().public_key],
},
)
def destroy_resources(self, include=[], exclude=[], wipe=False):
"""Destroy all active or obsolete resources."""
with self._get_deployment_lock():
for r in self.resources.itervalues():
r._destroyed_event = threading.Event()
r._errored = False
for rev_dep in r.destroy_before(self.resources.itervalues()):
try:
rev_dep._wait_for.append(r)
except AttributeError:
rev_dep._wait_for = [ r ]
def worker(m):
try:
if not should_do(m, include, exclude): return
try:
for dep in m._wait_for:
dep._destroyed_event.wait()
# !!! Should we print a message here?
if dep._errored:
m._errored = True
return
except AttributeError:
pass
if m.destroy(wipe=wipe): self.delete_resource(m)
except:
m._errored = True
raise
finally:
m._destroyed_event.set()
nixops.parallel.run_tasks(nr_workers=-1, tasks=self.resources.values(), worker_fun=worker)
# Remove the destroyed machines from the rollback profile.
# This way, a subsequent "nix-env --delete-generations old" or
# "nix-collect-garbage -d" will get rid of the machine
# configurations.
if self.rollback_enabled: # and len(self.active) == 0:
profile = self.create_profile()
attrs = {m.name:
Function("builtins.storePath", m.cur_toplevel, call=True)
for m in self.active.itervalues() if m.cur_toplevel}
if subprocess.call(
["nix-env", "-p", profile, "--set", "*", "-I", "nixops=" + self.expr_path,
"-f", "<nixops/update-profile.nix>",
"--arg", "machines", py2nix(attrs, inline=True)]) != 0:
raise Exception("cannot update profile ‘{0}’".format(profile))
def get_physical_spec(self):
prefixLength = bin(int(socket.inet_aton(self.netmask).encode('hex'), 16)).count('1')
return Function("{ ... }", {
'imports': [ RawValue('<nixpkgs/nixos/modules/profiles/qemu-guest.nix>') ],
'networking': {
'defaultGateway': self.default_gateway,
'nameservers': ['8.8.8.8'], # default provided by DO
('interfaces', 'enp0s3'): {
'ip4': [{"address": self.public_ipv4, 'prefixLength': prefixLength}],
},
},
('boot', 'loader', 'grub', 'device'): 'nodev', # keep ubuntu bootloader?
('fileSystems', '/'): { 'device': '/dev/vda1', 'fsType': 'ext4'},
('users', 'extraUsers', 'root', 'openssh', 'authorizedKeys', 'keys'): [self.depl.active_resources.get('ssh-key').public_key],
})
def get_physical_spec(self):
return Function(
"{ ... }", {
'imports':
[RawValue('<nixpkgs/nixos/modules/profiles/qemu-guest.nix>')],
('config', 'boot', 'initrd', 'availableKernelModules'):
["ata_piix", "uhci_hcd", "virtio_pci", "sr_mod", "virtio_blk"],
('config', 'boot', 'loader', 'grub', 'device'):
'/dev/vda',
('config', 'fileSystems', '/'): {
'device': '/dev/vda1',
'fsType': 'btrfs'
},
('config', 'users', 'extraUsers', 'root', 'openssh', 'authorizedKeys', 'keys'):
[self._ssh_public_key]
})
def get_physical_spec(self):
block_device_mapping = {}
for k, v in self.block_device_mapping.items():
if (v.get('encrypt', False)
and v.get('passphrase', "") == ""
and v.get('generatedKey', "") != ""):
block_device_mapping[k] = {
'passphrase': Function("pkgs.lib.mkOverride 10",
v['generatedKey'], call=True),
}
return {
'require': [
RawValue("<nixpkgs/nixos/modules/virtualisation/google-compute-config.nix>")
],
('deployment', 'gce', 'blockDeviceMapping'): block_device_mapping,
}
def get_physical_spec_from_plan(self, public_key):
if self.plan == "c1.small.x86":
return Function("{ ... }", {
('config', 'boot', 'initrd', 'availableKernelModules'): [ "ata_piix", "uhci_hcd", "virtio_pci", "sr_mod", "virtio_blk" ],
('config', 'boot', 'loader', 'grub', 'devices'): [ '/dev/sda', '/dev/sdb' ],
('config', 'fileSystems', '/'): { 'label': 'nixos', 'fsType': 'ext4'},
('config', 'users', 'users', 'root', 'openssh', 'authorizedKeys', 'keys'): [public_key],
('config', 'networking', 'bonds', 'bond0', 'interfaces'): [ "enp1s0f0", "enp1s0f1"],
('config', 'boot', 'kernelParams'): [ "console=ttyS1,115200n8" ],
('config', 'boot', 'loader', 'grub', 'extraConfig'): """
serial --unit=0 --speed=115200 --word=8 --parity=no --stop=1
terminal_output serial console
terminal_input serial console
""",
('config', 'networking', 'bonds', 'bond0', 'driverOptions'): {
"mode": "802.3ad",
"xmit_hash_policy": "layer3+4",
"lacp_rate": "fast",
"downdelay": "200",
"miimon": "100",
"updelay": "200",
},
('config', 'networking', 'nameservers'): [ "8.8.8.8", "8.8.4.4" ], # TODO
('config', 'networking', 'defaultGateway'): {
"address": self.default_gateway,
"interface": "bond0",
},
('config', 'networking', 'defaultGateway6'): {
"address": self.default_gatewayv6,
"interface": "bond0",
},
('config', 'networking', 'dhcpcd', 'enable'): False,
('config', 'networking', 'interfaces', 'bond0'): {
"useDHCP": False,
"ipv4": {
"addresses": [
{ "address": self.public_ipv4, "prefixLength": self.public_cidr },
{ "address": self.private_ipv4, "prefixLength": self.private_cidr },
],
"routes": [
{
"address": "10.0.0.0",
"prefixLength": 8,
"via": self.private_gateway,
},
],
},
"ipv6": {
"addresses": [
{ "address": self.public_ipv6, "prefixLength": self.public_cidrv6 },
],
},
},
})
elif self.plan == "g2.large.x86":
return Function("{ ... }", {
('config', 'boot', 'initrd', 'availableKernelModules'): [ "ata_piix", "uhci_hcd", "virtio_pci", "sr_mod", "virtio_blk" ],
('config', 'boot', 'loader', 'grub', 'devices'): [ '/dev/sda' ],
('config', 'fileSystems', '/'): { 'label': 'nixos', 'fsType': 'ext4'},
('config', 'users', 'users', 'root', 'openssh', 'authorizedKeys', 'keys'): [public_key],
('config', 'networking', 'bonds', 'bond0', 'interfaces'): [ "enp96s0f0", "enp96s0f1"],
('config', 'boot', 'kernelParams'): [ "console=ttyS1,115200n8" ],
('config', 'boot', 'kernelModules'): [ 'kvm-intel' ],
('config', 'boot', 'loader', 'grub', 'extraConfig'): """
serial --unit=0 --speed=115200 --word=8 --parity=no --stop=1
terminal_output serial console
terminal_input serial console
""",
('config', 'networking', 'bonds', 'bond0', 'driverOptions'): {
"mode": "802.3ad",
"xmit_hash_policy": "layer3+4",
"lacp_rate": "fast",
"downdelay": "200",
"miimon": "100",
"updelay": "200",
},
('config', 'networking', 'nameservers'): [ "8.8.8.8", "8.8.4.4" ], # TODO
('config', 'networking', 'defaultGateway'): {
"address": self.default_gateway,
"interface": "bond0",
},
('config', 'networking', 'defaultGateway6'): {
"address": self.default_gatewayv6,
"interface": "bond0",
},
('config', 'networking', 'dhcpcd', 'enable'): False,
('config', 'networking', 'interfaces', 'bond0'): {
"useDHCP": False,
"ipv4": {
"addresses": [
{ "address": self.public_ipv4, "prefixLength": self.public_cidr },
{ "address": self.private_ipv4, "prefixLength": self.private_cidr },
],
"routes": [
{
"address": "10.0.0.0",
"prefixLength": 8,
"via": self.private_gateway,
},
],
},
"ipv6": {
"addresses": [
{ "address": self.public_ipv6, "prefixLength": self.public_cidrv6 },
],
},
},
})
else:
raise Exception("Plan {} not supported by nixops".format(self.plan))
def test_function_call(self):
self.assert_nix(Function("fun_call", {'a': 'b'}, call=True),
'fun_call { a = "b"; }')
self.assert_nix(Function("multiline_call", {'a': 'b'}, call=True),
'multiline_call {\n a = "b";\n}',
maxwidth=0)