def _create_instance(self, defn) -> None:
if not self.public_client_key:
(private, public) = create_key_pair(type="ed25519")
self.public_client_key = public
self.private_client_key = private
if not self.public_host_key:
(private, public) = create_key_pair(type="ed25519")
self.public_host_key = public
self.private_host_key = private
location: BoundLocation = self.get_client().locations.get_by_name(
defn.location)
ssh_keys: List[BoundSSHKey] = [
self._create_ssh_key(self.public_client_key)
]
# Ensure host keys get injected into the base OS
user_data = ("#cloud-config\n"
"ssh_keys:\n"
" ed25519_public: {0}\n"
" ed25519_private: |\n"
" {1}").format(
self.public_host_key,
self.private_host_key.replace("\n", "\n "))
self.logger.log_start(
f"creating {defn.server_type} server at {location.description}...")
response = self.get_client().servers.create(
name=defn.server_name,
labels={
**self.get_common_labels(),
**dict(defn.labels)
},
location=location,
server_type=ServerType(defn.server_type),
ssh_keys=ssh_keys,
user_data=user_data,
image=Image(name="ubuntu-20.04"), # for lustration
start_after_create=True,
)
self.state = self.STARTING
self.wait_on_action(response.action)
with self.depl._db:
self.vm_id = response.server.id
self.public_ipv4 = response.server.public_net.ipv4.ip
self.public_ipv6 = response.server.public_net.ipv6.ip
self.server_name = defn.server_name
self.server_type = defn.server_type
self.legacy_if_scheme = defn.server_type.startswith("cx")
self.location = defn.location
self.labels = dict(defn.labels)
self.private_host_key = None
known_hosts.add(self.public_ipv4, self.public_host_key)
self.logger.log_end(f"{self.public_ipv4}")
def _install_base_system(self):
self.log_start("creating missing directories... ")
cmds = ["mkdir -m 1777 -p /mnt/tmp /mnt/nix/store"]
mntdirs = [
"var", "etc", "bin", "nix/var/nix/gcroots",
"nix/var/nix/temproots", "nix/var/nix/manifests",
"nix/var/nix/userpool", "nix/var/nix/profiles", "nix/var/nix/db",
"nix/var/log/nix/drvs"
]
to_create = ' '.join(map(lambda d: os.path.join("/mnt", d), mntdirs))
cmds.append("mkdir -m 0755 -p {0}".format(to_create))
self.run_command(' && '.join(cmds))
self.log_end("done.")
self.log_start("bind-mounting files in /etc... ")
for etcfile in ("resolv.conf", "passwd", "group"):
self.log_continue("{0}...".format(etcfile))
cmd = ("if ! test -e /mnt/etc/{0}; then"
" touch /mnt/etc/{0} && mount --bind /etc/{0} /mnt/etc/{0};"
" fi").format(etcfile)
self.run_command(cmd)
self.log_end("done.")
self.run_command("touch /mnt/etc/NIXOS")
self.run_command("activate-remote")
self.main_ssh_private_key, self.main_ssh_public_key = create_key_pair(
key_name="NixOps client key of {0}".format(self.name))
self._gen_network_spec()
def _install_base_system(self):
self.log_start("creating missing directories... ")
cmds = ["mkdir -m 1777 -p /mnt/tmp /mnt/nix/store"]
mntdirs = ["var", "etc", "bin", "nix/var/nix/gcroots",
"nix/var/nix/temproots", "nix/var/nix/manifests",
"nix/var/nix/userpool", "nix/var/nix/profiles",
"nix/var/nix/db", "nix/var/log/nix/drvs"]
to_create = ' '.join(map(lambda d: os.path.join("/mnt", d), mntdirs))
cmds.append("mkdir -m 0755 -p {0}".format(to_create))
self.run_command(' && '.join(cmds))
self.log_end("done.")
self.log_start("bind-mounting files in /etc... ")
for etcfile in ("resolv.conf", "passwd", "group"):
self.log_continue("{0}...".format(etcfile))
cmd = ("if ! test -e /mnt/etc/{0}; then"
" touch /mnt/etc/{0} && mount --bind /etc/{0} /mnt/etc/{0};"
" fi").format(etcfile)
self.run_command(cmd)
self.log_end("done.")
self.run_command("touch /mnt/etc/NIXOS")
self.run_command("activate-remote")
self.main_ssh_private_key, self.main_ssh_public_key = create_key_pair(
key_name="NixOps client key of {0}".format(self.name)
)
self._gen_network_spec()
def _install_main_ssh_keys(self):
"""
Create a SSH private/public keypair and put the public key into the
chroot.
"""
private, public = create_key_pair(
key_name="NixOps client key of {0}".format(self.name))
self.main_ssh_private_key, self.main_ssh_public_key = private, public
res = self.run_command(
"umask 077 && mkdir -p /mnt/root/.ssh &&"
" cat > /mnt/root/.ssh/authorized_keys",
stdin_string=public)
def create(self, defn, check, allow_reboot, allow_recreate):
assert isinstance(defn, NoneDefinition)
self.set_common_state(defn)
self.target_host = defn._target_host
if not self.vm_id:
self.log_start("generating new SSH keypair...")
key_name = "NixOps client key for {0}".format(self.name)
self._ssh_private_key, self._ssh_public_key = \
create_key_pair(key_name=key_name)
self.log_end("done.")
self.vm_id = "nixops-{0}-{1}".format(self.depl.uuid, self.name)
def _install_main_ssh_keys(self):
"""
Create a SSH private/public keypair and put the public key into the
chroot.
"""
private, public = create_key_pair(
key_name="NixOps client key of {0}".format(self.name)
)
self.main_ssh_private_key, self.main_ssh_public_key = private, public
res = self.run_command("umask 077 && mkdir -p /mnt/root/.ssh &&"
" cat > /mnt/root/.ssh/authorized_keys",
stdin_string=public)
def create(self, defn, check, allow_reboot, allow_recreate):
assert isinstance(defn, NoneDefinition)
self.set_common_state(defn)
self.target_host = defn._target_host
if not self.vm_id:
self.log_start("generating new SSH keypair...")
key_name = "NixOps client key for {0}".format(self.name)
self._ssh_private_key, self._ssh_public_key = \
create_key_pair(key_name=key_name)
self.log_end("done.")
self.vm_id = "nixops-{0}-{1}".format(self.depl.uuid, self.name)
def create(
self,
defn: NoneDefinition,
check: bool,
allow_reboot: bool,
allow_recreate: bool,
) -> None:
assert isinstance(defn, NoneDefinition)
self.set_common_state(defn)
self.target_host = defn._target_host
self.public_ipv4 = defn._public_ipv4
if not self.vm_id:
if self.provision_ssh_key:
self.logger.log_start("generating new SSH key pair... ")
key_name = "NixOps client key for {0}".format(self.name)
self._ssh_private_key, self._ssh_public_key = create_key_pair(
key_name=key_name
)
self.logger.log_end("done")
self.vm_id = "nixops-{0}-{1}".format(self.depl.uuid, self.name)
def create(self, defn, check, allow_reboot, allow_recreate):
assert isinstance(defn, GCEDefinition)
self.no_project_change(defn)
self.no_region_change(defn)
self.no_change(self.machine_name != defn.machine_name, "instance name")
self.set_common_state(defn)
self.copy_credentials(defn)
self.machine_name = defn.machine_name
self.region = defn.region
if not self.public_client_key:
(private, public) = create_key_pair()
self.public_client_key = public
self.private_client_key = private
self.host_key_type = "ed25519" if self.state_version != "14.12" and nixops.util.parse_nixos_version(
defn.config["nixosRelease"]) >= ["15", "09"] else "ecdsa"
if not self.public_host_key:
(private, public) = create_key_pair(type=self.host_key_type)
self.public_host_key = public
self.private_host_key = private
recreate = False
if check:
try:
node = self.node()
if self.vm_id:
if node.state == NodeState.TERMINATED:
recreate = True
self.warn(
"the instance is terminated and needs a reboot")
self.state = self.STOPPED
self.handle_changed_property('region',
node.extra['zone'].name,
can_fix=False)
# a bit hacky but should work
network_name = node.extra['networkInterfaces'][0][
'network'].split('/')[-1]
if network_name == 'default': network_name = None
self.handle_changed_property('network', network_name)
self.handle_changed_property('instance_type', node.size)
self.handle_changed_property(
'public_ipv4',
node.public_ips[0] if node.public_ips else None,
property_name='public IP address')
if self.public_ipv4:
known_hosts.add(self.public_ipv4, self.public_host_key)
self.handle_changed_property(
'private_ipv4',
node.private_ips[0] if node.private_ips else None,
property_name='private IP address')
if self.ipAddress:
try:
address = self.connect().ex_get_address(
self.ipAddress)
if self.public_ipv4 and self.public_ipv4 != address.address:
self.warn(
"static IP Address {0} assigned to this machine has unexpectely "
"changed from {1} to {2} most likely due to being redeployed"
.format(self.ipAddress, self.public_ipv4,
address.address))
self.ipAddress = None
except libcloud.common.google.ResourceNotFoundError:
self.warn(
"static IP Address resource {0} used by this machine has been destroyed; "
"it is likely that the machine is still holding the address itself ({1}) "
"and this is your last chance to reclaim it before it gets "
"lost in a reboot".format(
self.ipAddress, self.public_ipv4))
self.handle_changed_property('tags',
sorted(node.extra['tags']))
actual_metadata = {
i['key']: i['value']
for i in node.extra['metadata'].get('items', [])
if i['key'] not in [
'ssh_host_{0}_key'.format(self.host_key_type),
'sshKeys', 'ssh_host_{0}_key_pub'.format(
self.host_key_type)
]
}
self.handle_changed_property('metadata', actual_metadata)
self.handle_changed_property(
'automatic_restart',
node.extra['scheduling']["automaticRestart"])
self.handle_changed_property(
'on_host_maintenance',
node.extra['scheduling']["onHostMaintenance"])
attached_disk_names = [
d.get("deviceName", None) for d in node.extra['disks']
]
# check that all disks are attached
for k, v in self.block_device_mapping.iteritems():
disk_name = v['disk_name'] or v['disk']
is_attached = disk_name in attached_disk_names
if not is_attached and not v.get('needsAttach', False):
self.warn(
"disk {0} seems to have been detached behind our back; will reattach..."
.format(disk_name))
v['needsAttach'] = True
self.update_block_device_mapping(k, v)
if is_attached and v.get('needsAttach', False):
self.warn(
"disk {0} seems to have been attached for us; thank you, mr. Elusive Bug!"
.format(disk_name))
del v['needsAttach']
self.update_block_device_mapping(k, v)
# check that no extra disks are attached
defn_disk_names = [
v['disk_name'] or v['disk']
for k, v in defn.block_device_mapping.iteritems()
]
state_disk_names = [
v['disk_name'] or v['disk']
for k, v in self.block_device_mapping.iteritems()
]
unexpected_disks = list(
set(attached_disk_names) - set(defn_disk_names) -
set(state_disk_names))
if unexpected_disks:
self.warn(
"unexpected disk(s) {0} are attached to this instance; "
"not fixing this just in case".format(
unexpected_disks))
else:
self.warn_not_supposed_to_exist(valuable_data=True)
self.confirm_destroy(node, self.full_name)
except libcloud.common.google.ResourceNotFoundError:
if self.vm_id:
self.warn(
"the instance seems to have been destroyed behind our back"
)
if not allow_recreate:
raise Exception("use --allow-recreate to fix")
self._node_deleted()
# check that the disks that should exist do exist
# and that the disks we expected to create don't exist yet
for k, v in defn.block_device_mapping.iteritems():
disk_name = v['disk_name'] or v['disk']
try:
disk = self.connect().ex_get_volume(
disk_name, v.get('region', None))
if k not in self.block_device_mapping and v['disk_name']:
self.warn_not_supposed_to_exist(
resource_name=disk_name, valuable_data=True)
self.confirm_destroy(disk, disk_name)
except libcloud.common.google.ResourceNotFoundError:
if v['disk']:
raise Exception(
"external disk '{0}' is required but doesn't exist"
.format(disk_name))
if k in self.block_device_mapping and v['disk_name']:
self.warn(
"disk '{0}' is supposed to exist, but is missing; will recreate..."
.format(disk_name))
self.update_block_device_mapping(k, None)
# create missing disks
for k, v in defn.block_device_mapping.iteritems():
if k in self.block_device_mapping: continue
if v['disk'] is None:
extra_msg = (" from snapshot '{0}'".format(v['snapshot'])
if v['snapshot'] else " from image '{0}'".format(
v['image']) if v['image'] else "")
self.log("creating GCE disk of {0} GiB{1}...".format(
v['size'] if v['size'] else "auto", extra_msg))
v['region'] = defn.region
try:
self.connect().create_volume(v['size'],
v['disk_name'],
v['region'],
snapshot=v['snapshot'],
image=v['image'],
ex_disk_type="pd-" +
v.get('type', 'standard'),
use_existing=False)
except libcloud.common.google.ResourceExistsError:
raise Exception(
"tried creating a disk that already exists; "
"please run 'deploy --check' to fix this")
v['needsAttach'] = True
self.update_block_device_mapping(k, v)
if self.vm_id:
if self.instance_type != defn.instance_type:
recreate = True
self.warn("change of the instance type requires a reboot")
if self.network != defn.network:
recreate = True
self.warn("change of the network requires a reboot")
if self.email != defn.email or self.scopes != defn.scopes:
recreate = True
self.warn('change of service account requires a reboot')
for k, v in self.block_device_mapping.iteritems():
defn_v = defn.block_device_mapping.get(k, None)
if defn_v and not v.get('needsAttach', False):
if v['bootDisk'] != defn_v['bootDisk']:
recreate = True
self.warn("change of the boot disk requires a reboot")
if v['readOnly'] != defn_v['readOnly']:
recreate = True
self.warn("remounting disk as ro/rw requires a reboot")
if recreate:
if not allow_reboot:
raise Exception(
"reboot is required for the requested changes; please run with --allow-reboot"
)
self.stop()
self.create_node(defn)
if self.node().state == NodeState.STOPPED:
self.start()
def create(self, defn, check, allow_reboot, allow_recreate):
assert isinstance(defn, GCEDefinition)
self.no_project_change(defn)
self.no_region_change(defn)
self.no_change(self.machine_name != defn.machine_name, "instance name")
self.set_common_state(defn)
self.copy_credentials(defn)
self.machine_name = defn.machine_name
self.region = defn.region
if not self.public_client_key:
(private, public) = create_key_pair()
self.public_client_key = public
self.private_client_key = private
self.host_key_type = "ed25519" if self.state_version != "14.12" and nixops.util.parse_nixos_version(defn.config["nixosRelease"]) >= ["15", "09"] else "ecdsa"
if not self.public_host_key:
(private, public) = create_key_pair(type=self.host_key_type)
self.public_host_key = public
self.private_host_key = private
recreate = False
if check:
try:
node = self.node()
if self.vm_id:
if node.state == NodeState.TERMINATED:
recreate = True
self.warn("the instance is terminated and needs a reboot")
self.state = self.STOPPED
self.handle_changed_property('region', node.extra['zone'].name, can_fix = False)
self.handle_changed_property('preemptible', node.extra['scheduling']['preemptible'], can_fix = False)
# a bit hacky but should work
network_name = node.extra['networkInterfaces'][0]['network'].split('/')[-1]
if network_name == 'default': network_name = None
self.handle_changed_property('network', network_name)
self.handle_changed_property('instance_type', node.size)
self.handle_changed_property('public_ipv4',
node.public_ips[0] if node.public_ips else None,
property_name = 'public IP address')
if self.public_ipv4:
known_hosts.add(self.public_ipv4, self.public_host_key)
self.handle_changed_property('private_ipv4',
node.private_ips[0] if node.private_ips else None,
property_name = 'private IP address')
if self.ipAddress:
try:
address = self.connect().ex_get_address(self.ipAddress)
if self.public_ipv4 and self.public_ipv4 != address.address:
self.warn("static IP Address {0} assigned to this machine has unexpectely "
"changed from {1} to {2} most likely due to being redeployed"
.format(self.ipAddress, self.public_ipv4, address.address) )
self.ipAddress = None
except libcloud.common.google.ResourceNotFoundError:
self.warn("static IP Address resource {0} used by this machine has been destroyed; "
"it is likely that the machine is still holding the address itself ({1}) "
"and this is your last chance to reclaim it before it gets "
"lost in a reboot".format(self.ipAddress, self.public_ipv4) )
self.handle_changed_property('tags', sorted(node.extra['tags']))
actual_metadata = { i['key']: i['value']
for i in node.extra['metadata'].get('items', [])
if i['key'] not in [ 'ssh_host_{0}_key'.format(self.host_key_type), 'sshKeys',
'ssh_host_{0}_key_pub'.format(self.host_key_type)] }
self.handle_changed_property('metadata', actual_metadata)
self.handle_changed_property('automatic_restart',
node.extra['scheduling']["automaticRestart"])
self.handle_changed_property('on_host_maintenance',
node.extra['scheduling']["onHostMaintenance"])
attached_disk_names = [d.get("deviceName", None) for d in node.extra['disks'] ]
# check that all disks are attached
for k, v in self.block_device_mapping.iteritems():
disk_name = v['disk_name'] or v['disk']
is_attached = disk_name in attached_disk_names
if not is_attached and not v.get('needsAttach', False):
self.warn("disk {0} seems to have been detached behind our back; will reattach...".format(disk_name))
v['needsAttach'] = True
self.update_block_device_mapping(k, v)
if is_attached and v.get('needsAttach', False):
self.warn("disk {0} seems to have been attached for us; thank you, mr. Elusive Bug!".format(disk_name))
del v['needsAttach']
self.update_block_device_mapping(k, v)
# check that no extra disks are attached
defn_disk_names = [v['disk_name'] or v['disk'] for k,v in defn.block_device_mapping.iteritems()]
state_disk_names = [v['disk_name'] or v['disk'] for k,v in self.block_device_mapping.iteritems()]
unexpected_disks = list( set(attached_disk_names) - set(defn_disk_names) - set(state_disk_names) )
if unexpected_disks:
self.warn("unexpected disk(s) {0} are attached to this instance; "
"not fixing this just in case".format(unexpected_disks))
else:
self.warn_not_supposed_to_exist(valuable_data = True)
self.confirm_destroy(node, self.full_name)
except libcloud.common.google.ResourceNotFoundError:
if self.vm_id:
self.warn("the instance seems to have been destroyed behind our back")
if not allow_recreate: raise Exception("use --allow-recreate to fix")
self._node_deleted()
# check that the disks that should exist do exist
# and that the disks we expected to create don't exist yet
for k,v in defn.block_device_mapping.iteritems():
disk_name = v['disk_name'] or v['disk']
try:
disk = self.connect().ex_get_volume(disk_name, v.get('region', None) )
if k not in self.block_device_mapping and v['disk_name']:
self.warn_not_supposed_to_exist(resource_name = disk_name, valuable_data = True)
self.confirm_destroy(disk, disk_name)
except libcloud.common.google.ResourceNotFoundError:
if v['disk']:
raise Exception("external disk '{0}' is required but doesn't exist".format(disk_name))
if k in self.block_device_mapping and v['disk_name']:
self.warn("disk '{0}' is supposed to exist, but is missing; will recreate...".format(disk_name))
self.update_block_device_mapping(k, None)
# create missing disks
for k, v in defn.block_device_mapping.iteritems():
if k in self.block_device_mapping: continue
if v['disk'] is None:
extra_msg = ( " from snapshot '{0}'".format(v['snapshot']) if v['snapshot']
else " from image '{0}'".format(v['image']) if v['image']
else "" )
self.log("creating GCE disk of {0} GiB{1}..."
.format(v['size'] if v['size'] else "auto", extra_msg))
v['region'] = defn.region
try:
self.connect().create_volume(v['size'], v['disk_name'], v['region'],
snapshot=v['snapshot'], image=v['image'],
ex_disk_type="pd-" + v.get('type', 'standard'),
use_existing=False)
except AttributeError:
# libcloud bug: The region we're trying to create the disk
# in doesn't exist.
raise Exception("tried creating a disk in nonexistent "
"region %r" % v['region'])
except libcloud.common.google.ResourceExistsError:
raise Exception("tried creating a disk that already exists; "
"please run 'deploy --check' to fix this")
v['needsAttach'] = True
self.update_block_device_mapping(k, v)
if self.vm_id:
if self.instance_type != defn.instance_type:
recreate = True
self.warn("change of the instance type requires a reboot")
if self.network != defn.network:
recreate = True
self.warn("change of the network requires a reboot")
if self.email != defn.email or self.scopes != defn.scopes:
recreate = True
self.warn('change of service account requires a reboot')
for k, v in self.block_device_mapping.iteritems():
defn_v = defn.block_device_mapping.get(k, None)
if defn_v and not v.get('needsAttach', False):
if v['bootDisk'] != defn_v['bootDisk']:
recreate = True
self.warn("change of the boot disk requires a reboot")
if v['readOnly'] != defn_v['readOnly']:
recreate = True
self.warn("remounting disk as ro/rw requires a reboot")
if recreate:
if not allow_reboot:
raise Exception("reboot is required for the requested changes; please run with --allow-reboot")
self.stop()
self.create_node(defn)
if self.node().state == NodeState.STOPPED:
self.start()
def create(self, defn: HcloudDefinition, check, allow_reboot,
allow_recreate):
assert isinstance(defn, HcloudDefinition)
hetzner = defn.config.hcloud
self.token = get_access_token(hetzner)
if self.state not in (MachineState.RESCUE, MachineState.UP) or check:
self.check()
self.set_common_state(defn)
self.upgrade_disk = hetzner.upgradeDisk
# TODO maybe bootstrap can be automated with vncdotool
image_id = self._fetch_image_id(hetzner.image, hetzner.image_selector)
if self.image_id is None:
self.image_id = image_id
elif self.image_id != image_id:
self.warn(
f"image_id changed from {self.image_id} to {image_id} but can't update image of a VM."
)
if self.location is None:
self.location = hetzner.location
elif self.location != hetzner.location:
self.warn(
f"location changed from {self.location} to {hetzner.location} but can't update location of a VM."
)
if self.vm_id is not None and hetzner.serverType != self.server_type:
# TODO Check if server can be upgraded before hitting the Hetzner API
# https://docs.hetzner.cloud/#server-actions-change-the-type-of-a-server
do_upgrade = True
# Only confirm if upgrade_disk is True because then the upgrade can't be undone
if self.upgrade_disk:
do_upgrade = self.depl.logger.confirm(
f"are you sure you want to change Hetzner server {self.name} type from "
+ f"{self.server_type} to {hetzner.serverType}?")
if do_upgrade:
self.log_start("Changing Hetzner server type...")
self._server.shutdown().wait_until_finished()
self.wait_for_down(callback=lambda: self.log_continue("."))
self._server.change_type(
ServerType(name=hetzner.serverType),
upgrade_disk=self.upgrade_disk).wait_until_finished()
self._server.power_on()
self.wait_for_up(callback=lambda: self.log_continue("."))
self.log_end("")
self.server_type = hetzner.serverType
ssh_keys = [
k.name if isinstance(k, ResourceEval) else k
for k in hetzner.sshKeys
]
if self.state != MachineState.MISSING and ssh_keys != self.ssh_keys:
self.logger.warn(
f"SSH keys cannot be changed after the server is created.")
volume_ids = []
filesystems = {}
for volumeopts in hetzner.volumes:
volume = volumeopts.volume
if isinstance(volume, str):
volume_model = self._client.volumes.get_by_name(volume)
volume_name = volume
volume_id = volume_model.id
volume_loc = volume_model.location.name
else:
volume_res = self.depl.get_typed_resource(
volume._name, "hcloud-volume", HcloudVolumeState)
volume_name = volume_res.name
volume_id = volume_res.hcloud_id
assert volume_id is not None
volume_loc = volume_res.location
if volume_loc != self.location:
raise Exception(
f"Volume {volume_name!r} is in a different location from server {self.name!r}"
)
volume_ids.append(volume_id)
if volumeopts.mountPoint is not None:
fs = dict(volumeopts.fileSystem)
fs["device"] = f"/dev/disk/by-id/scsi-0HC_Volume_{volume_id}"
filesystems[volumeopts.mountPoint] = fs
has_priv = self._ssh_private_key is not None
has_pub = self._ssh_public_key is not None
assert has_priv == has_pub
if not has_priv:
self.log("Generating SSH keypair...")
(self._ssh_private_key, self._ssh_public_key) = create_key_pair()
if self.vm_id:
if self.volume_ids != volume_ids:
current = set(self.volume_ids)
new = set(volume_ids)
volumes_client = self._client.volumes
self.log_start("Updating volumes...")
for v in current - new:
volumes_client.detach(Volume(id=v))
self.log_continue(".")
for v in new - current:
volumes_client.attach(
Volume(id=v), self._server,
automount=False).wait_until_finished()
self.log_continue(".")
self.log_end("")
self.volume_ids = volume_ids
else:
self.log_start(
"Creating Hetzner Cloud VM (" +
f"image '{image_id}', type '{hetzner.serverType}', location '{hetzner.location}'"
+ ")...")
response = self._client.servers.create(
name=self.name,
ssh_keys=[SSHKey(name=k) for k in ssh_keys],
volumes=[Volume(id=v) for v in volume_ids],
server_type=ServerType(self.server_type),
image=Image(id=self.image_id),
# Set labels so we can find the instance if nixops crashes before writing vm_id
labels=dict(self._server_labels()),
user_data=None if self._ssh_public_key is None else yaml.dump(
{"public-keys": [self._ssh_public_key]}),
)
self.log_end("")
self.public_ipv4 = response.server.public_net.ipv4.ip
self.log_start("waiting for SSH...")
self.wait_for_up(callback=lambda: self.log_continue("."))
self.log_end("")
with self.depl._db:
self.vm_id = response.server.id
# TODO get state from creation response
self.state = MachineState.STARTING
self.ssh_keys = ssh_keys
self.volume_ids = volume_ids
self._detect_hardware()
self._update_host_keys()
self.filesystems = filesystems
def create(self, defn, check, allow_reboot, allow_recreate):
assert isinstance(defn, GCEDefinition)
self.no_project_change(defn)
self.no_region_change(defn)
self.no_change(self.machine_name != defn.machine_name, "instance name")
self.set_common_state(defn)
self.copy_credentials(defn)
self.machine_name = defn.machine_name
self.region = defn.region
if not self.public_client_key:
(private, public) = create_key_pair()
self.public_client_key = public
self.private_client_key = private
if not self.public_host_key:
(private, public) = create_key_pair(type="ecdsa")
self.public_host_key = public
self.private_host_key = private
recreate = False
if check:
try:
node = self.node()
if self.vm_id:
if node.state == NodeState.TERMINATED:
recreate = True
self.warn("the instance is terminated and needs a reboot")
self.state = self.STOPPED
self.handle_changed_property("region", node.extra["zone"].name, can_fix=False)
# a bit hacky but should work
network_name = node.extra["networkInterfaces"][0]["network"].split("/")[-1]
if network_name == "default":
network_name = None
self.handle_changed_property("network", network_name)
self.handle_changed_property("instance_type", node.size)
self.handle_changed_property(
"public_ipv4",
node.public_ips[0] if node.public_ips else None,
property_name="public IP address",
)
if self.public_ipv4:
known_hosts.add(self.public_ipv4, self.public_host_key)
self.handle_changed_property(
"private_ipv4",
node.private_ips[0] if node.private_ips else None,
property_name="private IP address",
)
if self.ipAddress:
try:
address = self.connect().ex_get_address(self.ipAddress)
if self.public_ipv4 and self.public_ipv4 != address.address:
self.warn(
"static IP Address {0} assigned to this machine has unexpectely "
"changed from {1} to {2} most likely due to being redeployed".format(
self.ipAddress, self.public_ipv4, address.address
)
)
self.ipAddress = None
except libcloud.common.google.ResourceNotFoundError:
self.warn(
"static IP Address resource {0} used by this machine has been destroyed; "
"it is likely that the machine is still holding the address itself ({1}) "
"and this is your last chance to reclaim it before it gets "
"lost in a reboot".format(self.ipAddress, self.public_ipv4)
)
self.handle_changed_property("tags", sorted(node.extra["tags"]))
actual_metadata = {
i["key"]: i["value"]
for i in node.extra["metadata"].get("items", [])
if i["key"] not in ["ssh_host_ecdsa_key", "sshKeys", "ssh_host_ecdsa_key_pub"]
}
self.handle_changed_property("metadata", actual_metadata)
self.handle_changed_property("automatic_restart", node.extra["scheduling"]["automaticRestart"])
self.handle_changed_property("on_host_maintenance", node.extra["scheduling"]["onHostMaintenance"])
attached_disk_names = [d.get("deviceName", None) for d in node.extra["disks"]]
# check that all disks are attached
for k, v in self.block_device_mapping.iteritems():
disk_name = v["disk_name"] or v["disk"]
is_attached = disk_name in attached_disk_names
if not is_attached and not v.get("needsAttach", False):
self.warn(
"disk {0} seems to have been detached behind our back; will reattach...".format(
disk_name
)
)
v["needsAttach"] = True
self.update_block_device_mapping(k, v)
if is_attached and v.get("needsAttach", False):
self.warn(
"disk {0} seems to have been attached for us; thank you, mr. Elusive Bug!".format(
disk_name
)
)
del v["needsAttach"]
self.update_block_device_mapping(k, v)
# check that no extra disks are attached
defn_disk_names = [v["disk_name"] or v["disk"] for k, v in defn.block_device_mapping.iteritems()]
state_disk_names = [v["disk_name"] or v["disk"] for k, v in self.block_device_mapping.iteritems()]
unexpected_disks = list(set(attached_disk_names) - set(defn_disk_names) - set(state_disk_names))
if unexpected_disks:
self.warn(
"unexpected disk(s) {0} are attached to this instance; "
"not fixing this just in case".format(unexpected_disks)
)
else:
self.warn_not_supposed_to_exist(valuable_data=True)
self.confirm_destroy(node, self.full_name)
except libcloud.common.google.ResourceNotFoundError:
if self.vm_id:
self.warn("the instance seems to have been destroyed behind our back")
if not allow_recreate:
raise Exception("use --allow-recreate to fix")
self._node_deleted()
# check that the disks that should exist do exist
# and that the disks we expected to create don't exist yet
for k, v in defn.block_device_mapping.iteritems():
disk_name = v["disk_name"] or v["disk"]
try:
disk = self.connect().ex_get_volume(disk_name, v.get("region", None))
if k not in self.block_device_mapping and v["disk_name"]:
self.warn_not_supposed_to_exist(resource_name=disk_name, valuable_data=True)
self.confirm_destroy(disk, disk_name)
except libcloud.common.google.ResourceNotFoundError:
if v["disk"]:
raise Exception("external disk '{0}' is required but doesn't exist".format(disk_name))
if k in self.block_device_mapping and v["disk_name"]:
self.warn("disk '{0}' is supposed to exist, but is missing; will recreate...".format(disk_name))
self.update_block_device_mapping(k, None)
# create missing disks
for k, v in defn.block_device_mapping.iteritems():
if k in self.block_device_mapping:
continue
if v["disk"] is None:
extra_msg = (
" from snapshot '{0}'".format(v["snapshot"])
if v["snapshot"]
else " from image '{0}'".format(v["image"])
if v["image"]
else ""
)
self.log("creating GCE disk of {0} GiB{1}...".format(v["size"] if v["size"] else "auto", extra_msg))
v["region"] = defn.region
try:
self.connect().create_volume(
v["size"],
v["disk_name"],
v["region"],
snapshot=v["snapshot"],
image=v["image"],
ex_disk_type="pd-" + v.get("type", "standard"),
use_existing=False,
)
except libcloud.common.google.ResourceExistsError:
raise Exception(
"tried creating a disk that already exists; " "please run 'deploy --check' to fix this"
)
v["needsAttach"] = True
self.update_block_device_mapping(k, v)
if self.vm_id:
if self.instance_type != defn.instance_type:
recreate = True
self.warn("change of the instance type requires a reboot")
if self.network != defn.network:
recreate = True
self.warn("change of the network requires a reboot")
for k, v in self.block_device_mapping.iteritems():
defn_v = defn.block_device_mapping.get(k, None)
if defn_v and not v.get("needsAttach", False):
if v["bootDisk"] != defn_v["bootDisk"]:
recreate = True
self.warn("change of the boot disk requires a reboot")
if v["readOnly"] != defn_v["readOnly"]:
recreate = True
self.warn("remounting disk as ro/rw requires a reboot")
if recreate:
if not allow_reboot:
raise Exception("reboot is required for the requested changes; please run with --allow-reboot")
self.stop()
self.create_node(defn)
