def _launchNode(self):
if self.label.diskimage:
diskimage = self.provider_config.diskimages[
self.label.diskimage.name]
else:
diskimage = None
if diskimage:
# launch using diskimage
cloud_image = self.handler.zk.getMostRecentImageUpload(
diskimage.name, self.provider_config.name)
if not cloud_image:
raise exceptions.LaunchNodepoolException(
"Unable to find current cloud image %s in %s" %
(diskimage.name, self.provider_config.name)
)
config_drive = diskimage.config_drive
image_external = dict(id=cloud_image.external_id)
image_id = "{path}/{upload_id}".format(
path=self.handler.zk._imageUploadPath(
cloud_image.image_name,
cloud_image.build_id,
cloud_image.provider_name),
upload_id=cloud_image.id)
image_name = diskimage.name
username = cloud_image.username
connection_type = diskimage.connection_type
connection_port = diskimage.connection_port
else:
# launch using unmanaged cloud image
config_drive = self.label.cloud_image.config_drive
image_external = self.label.cloud_image.external
image_id = self.label.cloud_image.name
image_name = self.label.cloud_image.name
username = self.label.cloud_image.username
connection_type = self.label.cloud_image.connection_type
connection_port = self.label.cloud_image.connection_port
hostname = self.provider_config.hostname_format.format(
label=self.label, provider=self.provider_config, node=self.node
)
self.log.info("Creating server with hostname %s in %s from image %s "
"for node id: %s" % (hostname,
self.provider_config.name,
image_name,
self.node.id))
# NOTE: We store the node ID in the server metadata to use for leaked
# instance detection. We cannot use the external server ID for this
# because that isn't available in ZooKeeper until after the server is
# active, which could cause a race in leak detection.
server = self.handler.manager.createServer(
hostname,
image=image_external,
min_ram=self.label.min_ram,
flavor_name=self.label.flavor_name,
key_name=self.label.key_name,
az=self.node.az,
config_drive=config_drive,
nodepool_node_id=self.node.id,
nodepool_node_label=self.node.type[0],
nodepool_image_name=image_name,
networks=self.pool.networks,
security_groups=self.pool.security_groups,
boot_from_volume=self.label.boot_from_volume,
volume_size=self.label.volume_size)
self.node.external_id = server.id
self.node.hostname = hostname
self.node.image_id = image_id
if username:
self.node.username = username
self.node.connection_type = connection_type
self.node.connection_port = connection_port
# Checkpoint save the updated node info
self.zk.storeNode(self.node)
self.log.debug("Waiting for server %s for node id: %s" %
(server.id, self.node.id))
server = self.handler.manager.waitForServer(
server, self.provider_config.launch_timeout,
auto_ip=self.pool.auto_floating_ip)
if server.status != 'ACTIVE':
raise exceptions.LaunchStatusException("Server %s for node id: %s "
"status: %s" %
(server.id, self.node.id,
server.status))
# If we didn't specify an AZ, set it to the one chosen by Nova.
# Do this after we are done waiting since AZ may not be available
# immediately after the create request.
if not self.node.az:
self.node.az = server.location.zone
# Use private_ipv4 on clouds where no public IP is configured.
if self.pool.use_private_ip:
interface_ip = server.private_ipv4
interface_type = 'private'
else:
interface_ip = server.interface_ip
interface_type = 'public'
if not interface_ip:
self.log.debug(
"Server data for failed IP: %s" % pprint.pformat(
server))
raise exceptions.LaunchNetworkException(
"Unable to find %s IP of server" % (interface_type))
self.node.interface_ip = interface_ip
self.node.public_ipv4 = server.public_v4
self.node.public_ipv6 = server.public_v6
self.node.private_ipv4 = server.private_v4
# devstack-gate multi-node depends on private_v4 being populated
# with something. On clouds that don't have a private address, use
# the public.
if not self.node.private_ipv4:
self.node.private_ipv4 = server.public_v4
# Checkpoint save the updated node info
self.zk.storeNode(self.node)
self.log.debug(
"Node %s is running [region: %s, az: %s, ip: %s ipv4: %s, "
"ipv6: %s]" %
(self.node.id, self.node.region, self.node.az,
self.node.interface_ip, self.node.public_ipv4,
self.node.public_ipv6))
# wait and scan the new node and record in ZooKeeper
host_keys = []
if self.pool.host_key_checking:
try:
self.log.debug(
"Gathering host keys for node %s", self.node.id)
# only gather host keys if the connection type is ssh
gather_host_keys = connection_type == 'ssh'
host_keys = utils.nodescan(
interface_ip,
timeout=self.provider_config.boot_timeout,
gather_hostkeys=gather_host_keys,
port=connection_port)
if gather_host_keys and not host_keys:
raise exceptions.LaunchKeyscanException(
"Unable to gather host keys")
except exceptions.ConnectionTimeoutException:
self._logConsole(self.node.external_id, self.node.hostname)
raise
self.node.host_keys = host_keys
self.zk.storeNode(self.node)
def launch(self):
self.log.debug("Starting %s instance" % self.node.type)
attempts = 1
while attempts <= self.retries:
try:
instance = self.handler.manager.createInstance(self.label)
break
except Exception:
if attempts <= self.retries:
self.log.exception(
"Launch attempt %d/%d failed for node %s:", attempts,
self.retries, self.node.id)
if attempts == self.retries:
raise
attempts += 1
time.sleep(1)
instance_id = instance.id
self.node.external_id = instance_id
self.zk.storeNode(self.node)
boot_start = time.monotonic()
while time.monotonic() - boot_start < self.boot_timeout:
state = instance.state.get('Name')
self.log.debug("Instance %s is %s" % (instance_id, state))
if state == 'running':
instance.create_tags(Tags=[{
'Key': 'nodepool_id',
'Value': str(self.node.id)
}])
instance.create_tags(Tags=[{
'Key': 'nodepool_pool',
'Value': str(self.pool.name)
}])
instance.create_tags(Tags=[{
'Key': 'nodepool_provider',
'Value': str(self.provider_name)
}])
break
time.sleep(0.5)
instance.reload()
if state != 'running':
raise exceptions.LaunchStatusException(
"Instance %s failed to start: %s" % (instance_id, state))
server_ip = instance.public_ip_address
if not server_ip:
raise exceptions.LaunchStatusException(
"Instance %s doesn't have a public ip" % instance_id)
self.node.connection_port = self.label.cloud_image.connection_port
self.node.connection_type = self.label.cloud_image.connection_type
if self.pool.host_key_checking:
try:
if (self.node.connection_type == 'ssh'
or self.node.connection_type == 'network_cli'):
gather_hostkeys = True
else:
gather_hostkeys = False
keys = nodescan(server_ip,
port=self.node.connection_port,
timeout=180,
gather_hostkeys=gather_hostkeys)
except Exception:
raise exceptions.LaunchKeyscanException(
"Can't scan instance %s key" % instance_id)
self.log.info("Instance %s ready" % instance_id)
self.node.state = zk.READY
self.node.external_id = instance_id
self.node.hostname = server_ip
self.node.interface_ip = server_ip
self.node.public_ipv4 = server_ip
self.node.host_keys = keys
self.node.username = self.label.cloud_image.username
self.node.python_path = self.label.cloud_image.python_path
self.zk.storeNode(self.node)
self.log.info("Instance %s is ready", instance_id)