async def init(self, image_names=None, networks=None):
"""Initialize provider with data from OpenStack.
Load:
* available flavors
* networks
* network availabilities (number of available IPs for networks)
* images which were defined in `images` option
* account limits (max and current usage of vCPUs, memory, ...)
"""
# session expects that credentials will be set via env variables
logger.info(f"{self.dsp_name}: Initializing provider")
try:
self.session = AuthPassword()
except TypeError as terr:
err = (
"OpenStack credentials not provided. Load OpenStack RC file with valid "
"credentials and try again. E.g.: $ source PROJECT-openrc.sh")
raise NotAuthenticatedError(err) from terr
self.nova = ExtraNovaClient(session=self.session)
self.glance = GlanceClient(session=self.session)
self.neutron = NeutronClient(session=self.session)
login_start = datetime.now()
await asyncio.gather(
self.nova.init_api(self.api_timeout),
self.glance.init_api(self.api_timeout),
self.neutron.init_api(self.api_timeout),
)
login_end = datetime.now()
logger.info(
f"{self.dsp_name}: Login duration {login_end - login_start}")
self.network_pools = networks
object_start = datetime.now()
_, _, limits, _, _ = await asyncio.gather(
self.load_flavors(),
self.load_images(image_names),
self.nova.limits.show(),
self.load_networks(),
self.load_ip_availabilities(),
)
self.limits = limits
object_end = datetime.now()
object_duration = object_end - object_start
logger.info(
f"{self.dsp_name}: Environment objects load duration: {object_duration}"
)
async def init(self, image_names=None):
"""Initialize provider with data from OpenStack.
Load:
* available flavors
* networks
* network availabilities (number of available IPs for networks)
* images which were defined in `images` option
* account limits (max and current usage of vCPUs, memory, ...)
"""
# session expects that credentials will be set via env variables
logger.info(f"{self.dsp_name}: Initializing provider")
self.session = AuthPassword()
self.nova = ExtraNovaClient(session=self.session)
self.glance = GlanceClient(session=self.session)
self.neutron = NeutronClient(session=self.session)
login_start = datetime.now()
await asyncio.gather(
self.nova.init_api(self.api_timeout),
self.glance.init_api(self.api_timeout),
self.neutron.init_api(self.api_timeout),
)
login_end = datetime.now()
login_duration = login_end - login_start
logger.info(f"{self.dsp_name}: Login duration {login_duration}")
object_start = datetime.now()
_, _, limits, _, _ = await asyncio.gather(
self.load_flavors(),
self.load_images(image_names),
self.nova.limits.show(),
self.load_networks(),
self.load_ip_availabilities(),
)
self.limits = limits
object_end = datetime.now()
object_duration = object_end - object_start
logger.info(
f"{self.dsp_name}: Environment objects load duration: {object_duration}"
)
async def wrap(self, *args, **kwargs):
if not hasattr(self, 'auth') or not self.auth.is_token_valid():
self.auth = AuthPassword(auth_url=self.config['auth_url'],
username=self.config['username'],
password=self.config['password'],
project_name=self.config['project_name'],
user_domain_name=self.config['user_domain_name'],
project_domain_name=self.config['project_domain_name'])
self.nova = NovaClient(session=self.auth)
self.glance = GlanceClient(session=self.auth)
await self.nova.init_api(timeout=self.config.get('http_timeout', 10))
await self.glance.init_api(timeout=self.config.get('http_timeout', 10))
if not hasattr(self, 'last_init') or self.last_init < (time.time() - 60):
await self.initialize()
self.last_init = time.time()
return await func(self, *args, **kwargs)
class OpenStackProvider(Provider):
"""
OpenStack Provider.
Provisions servers in OpenStack with added logic to check if requested
resources are available.
"""
def __init__(self):
"""Object initialization."""
self._name = PROVISIONER_KEY
self.dsp_name = "OpenStack"
self.strategy = STRATEGY_RETRY
self.max_attempts = 5 # provisioning retries
self.flavors = {}
self.flavors_by_ref = {}
self.images = {}
self.images_by_ref = {}
self.limits = {}
self.networks = {}
self.networks_by_ref = {}
self.ips = {}
self.ips_by_ref = {}
self.api_timeout = 6 * 60 # timeout for request to OpenStack
self.timeout = 60 # minutes
self.poll_sleep_initial = 15 # seconds
self.poll_sleep = 7 # seconds
self.poll_init_adj = 0 # set based on # of hosts to provisions
self.poll_adj = 0 # set based on # of hosts to provisions
self.status_map = {
"ACTIVE": STATUS_ACTIVE,
"BUILD": STATUS_PROVISIONING,
"DELETED": STATUS_DELETED,
"ERROR": STATUS_ERROR,
# there is much more we can treat it as STATUS_OTHER, see:
# https://docs.openstack.org/api-guide/compute/server_concepts.html
}
async def init(self, image_names=None, networks=None):
"""Initialize provider with data from OpenStack.
Load:
* available flavors
* networks
* network availabilities (number of available IPs for networks)
* images which were defined in `images` option
* account limits (max and current usage of vCPUs, memory, ...)
"""
# session expects that credentials will be set via env variables
logger.info(f"{self.dsp_name}: Initializing provider")
try:
self.session = AuthPassword()
except TypeError as terr:
err = (
"OpenStack credentials not provided. Load OpenStack RC file with valid "
"credentials and try again. E.g.: $ source PROJECT-openrc.sh")
raise NotAuthenticatedError(err) from terr
self.nova = ExtraNovaClient(session=self.session)
self.glance = GlanceClient(session=self.session)
self.neutron = NeutronClient(session=self.session)
login_start = datetime.now()
await asyncio.gather(
self.nova.init_api(self.api_timeout),
self.glance.init_api(self.api_timeout),
self.neutron.init_api(self.api_timeout),
)
login_end = datetime.now()
logger.info(
f"{self.dsp_name}: Login duration {login_end - login_start}")
self.network_pools = networks
object_start = datetime.now()
_, _, limits, _, _ = await asyncio.gather(
self.load_flavors(),
self.load_images(image_names),
self.nova.limits.show(),
self.load_networks(),
self.load_ip_availabilities(),
)
self.limits = limits
object_end = datetime.now()
object_duration = object_end - object_start
logger.info(
f"{self.dsp_name}: Environment objects load duration: {object_duration}"
)
def _set_flavors(self, flavors):
"""Extend provider configuration with list of flavors."""
for flavor in flavors:
self.flavors[flavor["name"]] = flavor
self.flavors_by_ref[flavor["id"]] = flavor
def _set_images(self, images):
"""Extend provider configuration with list of images."""
for image in images:
self.images[image["name"]] = image
self.images_by_ref[image["id"]] = image
def _is_network_type(self, name):
"""Check if name is a configured network type in provisioning config."""
network_type = self.network_pools.get(name)
return bool(network_type)
def _aggregate_networks(self, hosts):
"""
Get how many host require each used network type.
Returns: dict where keys are network types and values are total count.
"""
network_types = {}
for host in hosts:
# skip hosts which have low-level network names defined
# this can be extended to pick network type based on the network name
names = host.get("networks")
if names:
continue
network_type = host.get("network")
if not self._is_network_type(network_type):
continue
count = network_types.get(network_type, 0)
count += 1
network_types[network_type] = count
return network_types
def _pick_network(self, network_type, count):
"""
Pick network based network type and needed amount.
Usable network with most IPs available is picked.
"""
possible_networks = self.network_pools[network_type]
networks = [self.get_network(net) for net in possible_networks]
usable = []
for network in networks:
ips = self.get_ips(ref=network.get("id"))
available = ips["total_ips"] - ips["used_ips"]
logger.debug(f"Network: {network['name']}")
logger.debug(f" total: {ips['total_ips']}")
logger.debug(f" used: {ips['used_ips']}")
logger.debug(f" available: {available}")
if available > count:
usable.append((network["name"], available))
if not usable:
logger.error(f"{self.dsp_name}: Error: no usable network"
f" for {count} hosts with {network_type}")
return None
# sort networks by number of available IPs
usable = sorted(usable, key=lambda u: u[1])
logger.debug(f"{self.dsp_name}: Listing usable networks: {usable}")
# Do not always pick the best, but randomize from good ones to spread
# load for high number of parallel jobs. E.g. if running mrack 100 times
# where each needs 3-4 hosts and best network has 250 IPs then we risk
# to pass the check intially but later fail as the check was not done
# with all the others on mind (race-condition).
# Good == has at least 50% of IPs as the best.
best_pool = usable[-1]
satisfying = [n for n in usable if n[1] / best_pool[1] > 0.5]
logger.debug(
f"{self.dsp_name}: Picking randomly from network pools which satisfy "
f"requirement (size_of_pool/size_of_biggest > 0.5): {satisfying}")
chosen = choice(satisfying)
logger.debug(
f"{self.dsp_name}: Network picked: {chosen[0]} with {chosen[1]} addresses"
)
return chosen[0]
def translate_network_types(self, hosts):
"""Pick the right OpenStack networks for all hosts.
Pick the network based on network type, networks configured for the
type and the available IP addresses. Process all hosts to
be able to pick the network which have enough addresses for all hosts.
All hosts will have either "networks" attribute or "network"
host attribute set with OpenStack network name or ID.
"""
nt_requirements = self._aggregate_networks(hosts)
nt_map = {}
for network_type, count in nt_requirements.items():
network_name = self._pick_network(network_type, count)
nt_map[network_type] = network_name
for host in hosts:
# skip hosts which have low-level network names defined
names = host.get("networks")
if names:
continue
network_type = host.get("network")
# skip if network_type is not network type
if not self.network_pools.get(network_type):
continue
network_name = nt_map[network_type]
host["network"] = network_name
def _set_networks(self, networks):
"""Extend provider configuration with list of networks."""
for network in networks:
self.networks[network["name"]] = network
self.networks_by_ref[network["id"]] = network
def get_flavor(self, name=None, ref=None):
"""Get flavor by name or UUID."""
flavor = self.flavors.get(name)
if not flavor:
flavor = self.flavors_by_ref.get(ref)
return flavor
def get_image(self, name=None, ref=None):
"""Get image by name or UUID."""
image = self.images.get(name)
if not image:
image = self.images_by_ref.get(ref)
return image
def get_network(self, name=None, ref=None):
"""Get network by name or UUID."""
network = self.networks.get(name)
if not network:
network = self.networks_by_ref.get(ref)
return network
def get_ips(self, name=None, ref=None):
"""Get network availability by network name or network UUID."""
aval = self.ips.get(name)
if not aval:
aval = self.ips_by_ref.get(ref)
return aval
async def load_flavors(self):
"""Extend provider configuration by loading all flavors from OpenStack."""
resp = await self.nova.flavors.list()
flavors = resp["flavors"]
self._set_flavors(flavors)
return flavors
async def load_images(self, image_names=None):
"""
Extend provider configuration by loading information about images.
Load everything if image_names list is not specified.
Specifying list of images to load might improve performance if the
OpenStack instance contains a lot of images.
"""
params = {"limit": 1000}
if image_names:
image_filter = ",".join(image_names)
image_filter = "in:" + image_filter
params["name"] = image_filter
images = []
response = await self.glance.images.list(**params)
images.extend(response["images"])
while response.get("next"):
p_result = urlparse(response.get("next"))
query = p_result.query
next_params = parse_qs(query)
for key, val in next_params.items():
if isinstance(val, list) and val:
next_params[key] = val[0]
response = await self.glance.images.list(**next_params)
images.extend(response["images"])
self._set_images(images)
return images
async def load_networks(self):
"""Extend provider configuration by loading all networks from OpenStack."""
resp = await self.neutron.network.list()
networks = resp["networks"]
self._set_networks(networks)
return networks
async def load_ip_availabilities(self):
"""Extend provider configuration by loading networks availabilities."""
resp = await self.neutron.ip.list()
availabilities = resp["network_ip_availabilities"]
for availability in availabilities:
self.ips[availability["network_name"]] = availability
self.ips_by_ref[availability["network_id"]] = availability
return availabilities
def _translate_flavor(self, req):
flavor_spec = req.get("flavor")
flavor_ref = req.get("flavorRef")
flavor = None
if flavor_ref:
flavor = self.get_flavor(ref=flavor_ref)
if flavor_spec:
flavor = self.get_flavor(flavor_spec, flavor_spec)
if not flavor:
specs = f"flavor: {flavor_spec}, ref: {flavor_ref}"
raise ValidationError(f"Flavor not found: {specs}")
return flavor
def _translate_image(self, req):
image_spec = req.get("image")
image_ref = req.get("imageRef")
image = None
if image_ref:
image = self.get_image(ref=image_ref)
if image_spec:
image = self.get_image(image_spec, image_spec)
if not image:
specs = f"image: {image_spec}, ref: {image_ref}"
raise ValidationError(f"Image not found {specs}")
return image
def _translate_networks(self, req, spec=False):
network_req = req.get("network")
network_specs = req.get("networks", [])
network_specs = deepcopy(network_specs)
networks = []
if not isinstance(network_specs, list):
network_specs = []
for network_spec in network_specs:
uuid = network_spec.get("uuid")
network = self.get_network(ref=uuid)
if not network:
raise ValidationError(f"Network not found: {network_spec}")
networks.append(network)
if network_req:
network = self.get_network(name=network_req, ref=network_req)
if not network:
raise ValidationError(f"Network not found: {network_req}")
network_specs.append({"uuid": network["id"]})
networks.append(network)
if spec:
return network_specs
return networks
def validate_host(self, req):
"""Validate that host requirements contains existing required objects."""
self._translate_flavor(req)
self._translate_image(req)
self._translate_networks(req)
return True
def _set_poll_sleep_times(self, reqs):
"""
Compute polling sleep times based on number of hosts.
So that we don't create unnecessary load on server while still checking returns
(initial_sleep, sleep)
"""
count = len(reqs)
# initial poll is the biggest performance saver it should be around
# time when more than half of host is in ACTIVE state
self.poll_init_adj = 0.65 * count
# poll time should ask often enough, to not create unnecessary delays
# while not that many to not load the server much
self.poll_adj = 0.22 * count
async def prepare_provisioning(self, reqs):
"""
Prepare provisioning.
Load missing images if they are not in provisioning-config.yaml
"""
prepare_images = list(
{req["image"]
for req in reqs if req["image"] not in self.images})
if prepare_images:
im_list = ", ".join(prepare_images)
logger.debug(
f"{self.dsp_name}: Loading image info for: '{im_list}'")
await self.load_images(list(prepare_images))
logger.debug(f"{self.dsp_name}: Loading images info done.")
self._set_poll_sleep_times(reqs)
async def validate_hosts(self, reqs):
"""Validate that all hosts requirements contains existing required objects."""
# translate network type to actual network and check network availabilities
self.translate_network_types(reqs)
for req in reqs:
logger.info(
f"{self.dsp_name}: Validating host: {object2json(req)}")
self.validate_host(req)
logger.info(f"{self.dsp_name}: {req['name']} - OK")
def get_host_requirements(self, req):
"""Get vCPU and memory requirements for host requirement."""
flavor_spec = req.get("flavor")
flavor_ref = req.get("flavorRef")
flavor = None
if flavor_ref:
flavor = self.get_flavor(ref=flavor_ref)
if flavor_spec:
flavor = self.get_flavor(flavor_spec, flavor_spec)
try:
res = {"ram": flavor["ram"], "vcpus": flavor["vcpus"]}
except TypeError as flavor_none:
# func does not load flavor so None is used as result
raise ValidationError(
f"Could not load the flavor for requirement: {req}"
) from flavor_none
return res
async def can_provision(self, reqs): # pylint: disable=arguments-differ
"""Check that all host can be provisioned.
Checks:
* available vCPUs and memory based on account limits
* that all host contain available flavors, images, networks
"""
vcpus = 0
ram = 0
for req in reqs:
needs = self.get_host_requirements(req)
vcpus += needs["vcpus"]
ram += needs["ram"]
limits = self.limits["limits"]["absolute"]
used_vcpus = limits["totalCoresUsed"]
used_memory = limits["totalRAMUsed"]
limit_vcpus = limits["maxTotalCores"]
limit_memory = limits["maxTotalRAMSize"]
req_vcpus = used_vcpus + vcpus
req_memory = used_memory + ram
logger.info(f"{self.dsp_name}: Required vcpus: {vcpus}, "
f"used: {used_vcpus}, max: {limit_vcpus}")
logger.info(f"{self.dsp_name}: Required ram: {ram}, "
f"used: {used_memory}, max: {limit_memory}")
return req_vcpus <= limit_vcpus and req_memory <= limit_memory
async def create_server(self, req):
"""Issue creation of a server.
req - dict of server requirements - can contains values defined in
POST /servers official OpenStack API
https://docs.openstack.org/api-ref/compute/?expanded=create-server-detail#create-server
The req object can contain following additional attributes:
* 'flavor': uuid or name of flavor to use
* 'network': uuid or name of network to use. Will be added to networks
list if present
"""
name = req.get("name")
logger.info(f"{self.dsp_name}: Creating server {name}")
specs = deepcopy(req) # work with own copy, do not modify the input
flavor = self._translate_flavor(req)
specs["flavorRef"] = flavor["id"]
if specs.get("flavor"):
del specs["flavor"]
image = self._translate_image(req)
specs["imageRef"] = image["id"]
if specs.get("image"):
del specs["image"]
network_specs = self._translate_networks(req, spec=True)
specs["networks"] = network_specs
if specs.get("network"):
del specs["network"]
error_attempts = 0
while True:
try:
response = await self.nova.servers.create(server=specs)
except ServerError as exc:
logger.debug(exc)
error_attempts += 1
if error_attempts <= SERVER_ERROR_RETRY:
await asyncio.sleep(SERVER_ERROR_SLEEP)
continue # Try again due to ServerError
if error_attempts > SERVER_ERROR_RETRY:
# now we are past to what we would like to wait fail now
raise ProvisioningError(
f"{self.dsp_name}: Failed to create server {req['name']}",
req, # add the requirement dictionary to traceback for later
)
fault = response["server"].get("fault", {})
if fault.get("code") == 500:
# In such scenario OpenStack might run out of hosts to provision
# This is not related to reaching OpenStack quota but to OpenStack
# itself being fully loaded and without free resources to provide
logger.info(
f"{self.dsp_name}: Unable to allocate resources for the required "
f"server (all available resources busy)")
error_attempts += 1
logger.info(
f"{self.dsp_name}: Retrying request in {SERVER_RES_SLEEP} minutes"
)
# We should wait for OpenStack for reasonable time to try to reprovision
# This sleep time should be longer for higher probability for Openstack
# having freed some resources for us even when we are not reaching quota
await asyncio.sleep(SERVER_RES_SLEEP * 60
) # * 60 - sleep for minutes
else:
# provisioning seems to pass correctly break to return result
break
return response.get("server")
async def delete_server(self, uuid):
"""Issue deletion of server.
Doesn't wait for the deletion to happen.
"""
error_attempts = 0
while True:
try:
await self.nova.servers.force_delete(uuid)
except ServerError as exc:
logger.debug(exc)
error_attempts += 1
if error_attempts > SERVER_ERROR_RETRY:
raise ProviderError(uuid) from exc
await asyncio.sleep(SERVER_ERROR_SLEEP)
except NotFoundError:
logger.warning(
f"{self.dsp_name}: Server '{uuid}' not found, probably already "
"deleted")
break
async def wait_till_provisioned(self, resource):
"""
Wait till server is provisioned.
Provisioned means that server is in ACTIVE or ERROR state
State is checked by polling. Polling can be controller via `poll_sleep` and
`poll_sleep_initial` options. This is useful when provisioning a lot of
machines as it is better to increase initial poll to not ask to often as
provisioning resources takes some time.
Waits till timeout happens. Timeout can be either specified or default provider
timeout is used.
Return information about provisioned server.
"""
uuid = resource.get("id")
poll_sleep_initial = self.poll_sleep_initial + self.poll_init_adj
poll_sleep_initial = (poll_sleep_initial / 2 +
poll_sleep_initial * random() * 1.5)
poll_sleep = self.poll_sleep + self.poll_adj
timeout = self.timeout
start = datetime.now()
timeout_time = start + timedelta(minutes=timeout)
# do not check the state immediately, it will take some time
logger.debug(f"{uuid}: sleeping for {poll_sleep_initial} seconds")
await asyncio.sleep(poll_sleep_initial)
resp = {}
logger.debug(f"Waiting for: {uuid}")
error_attempts = 0
while datetime.now() < timeout_time:
try:
resp = await self.nova.servers.get(uuid)
except NotFoundError as nf_err:
raise ServerNotFoundError(uuid) from nf_err
except ServerError as err:
logger.debug(f"{self.dsp_name}: {err}")
error_attempts += 1
if error_attempts > SERVER_ERROR_RETRY:
raise ProvisioningError(uuid) from err
server = resp["server"]
if server["status"] in ["ACTIVE", "ERROR"]:
break
poll_sleep += 0.5 # increase delays to check the longer it takes
logger.debug(f"{uuid}: sleeping for {poll_sleep} seconds")
await asyncio.sleep(poll_sleep)
done_time = datetime.now()
prov_duration = (done_time - start).total_seconds()
if datetime.now() >= timeout_time:
logger.warning(f"{self.dsp_name}: Host {uuid} was not provisioned "
f"within a timeout of {timeout} mins")
else:
logger.info(
f"{self.dsp_name}: Host {uuid} was provisioned in {prov_duration:.1f}s"
)
return server
async def delete_host(self, host_id):
"""Issue deletion of host(server) from OpenStack."""
logger.info(f"{self.dsp_name}: Deleting host {host_id}")
await self.delete_server(host_id)
return True
def prov_result_to_host_data(self, prov_result):
"""Get needed host information from openstack provisioning result."""
result = {}
result["id"] = prov_result.get("id")
result["name"] = prov_result.get("name")
networks = prov_result.get("addresses", {})
result["addresses"] = [
ip.get("addr") for n in networks.values() for ip in n
]
result["fault"] = prov_result.get("fault")
result["status"] = prov_result.get("status")
return result
class OpenStackProvider(Provider):
"""
OpenStack Provider.
Provisions servers in OpenStack with added logic to check if requested
resources are available.
"""
def __init__(self):
"""Object initialization."""
self._name = PROVISIONER_KEY
self.flavors = {}
self.flavors_by_ref = {}
self.images = {}
self.images_by_ref = {}
self.limits = {}
self.networks = {}
self.networks_by_ref = {}
self.ips = {}
self.ips_by_ref = {}
self.timeout = 60 # minutes
self.poll_sleep_initial = 15 # seconds
self.poll_sleep = 7 # seconds
self.STATUS_MAP = {
"ACTIVE": STATUS_ACTIVE,
"BUILD": STATUS_PROVISIONING,
"DELETED": STATUS_DELETED,
"ERROR": STATUS_ERROR,
# there is much more we can treat it as STATUS_OTHER, see:
# https://docs.openstack.org/api-guide/compute/server_concepts.html
}
async def init(self, image_names=None):
"""Initialize provider with data from OpenStack.
Load:
* available flavors
* networks
* network availabilities (number of available IPs for networks)
* images which were defined in `image_names` option
* account limits (max and current usage of vCPUs, memory, ...)
"""
# session expects that credentials will be set via env variables
logger.info("Initializing OpenStack provider")
self.session = AuthPassword()
self.nova = ExtraNovaClient(session=self.session)
self.glance = GlanceClient(session=self.session)
self.neutron = NeutronClient(session=self.session)
login_start = datetime.now()
await asyncio.gather(
self.nova.init_api(), self.glance.init_api(), self.neutron.init_api()
)
login_end = datetime.now()
login_duration = login_end - login_start
logger.info(f"Login duration {login_duration}")
object_start = datetime.now()
flavors, images, limits, networks, ips = await asyncio.gather(
self.load_flavors(),
self.load_images(image_names),
self.nova.limits.show(),
self.load_networks(),
self.load_ip_availabilities(),
)
self.limits = limits
object_end = datetime.now()
object_duration = object_end - object_start
logger.info(
f"Open Stack environment objects " f"load duration: {object_duration}"
)
def set_flavors(self, flavors):
"""Extend provider configuration with list of flavors."""
for flavor in flavors:
self.flavors[flavor["name"]] = flavor
self.flavors_by_ref[flavor["id"]] = flavor
def set_images(self, images):
"""Extend provider configuration with list of images."""
for image in images:
self.images[image["name"]] = image
self.images_by_ref[image["id"]] = image
def set_networks(self, networks):
"""Extend provider configuration with list of networks."""
for network in networks:
self.networks[network["name"]] = network
self.networks_by_ref[network["id"]] = network
def get_flavor(self, name=None, ref=None):
"""Get flavor by name or UUID."""
flavor = self.flavors.get(name)
if not flavor:
flavor = self.flavors_by_ref.get(ref)
return flavor
def get_image(self, name=None, ref=None):
"""Get image by name or UUID."""
image = self.images.get(name)
if not image:
image = self.images_by_ref.get(ref)
return image
def get_network(self, name=None, ref=None):
"""Get network by name or UUID."""
network = self.networks.get(name)
if not network:
network = self.networks_by_ref.get(ref)
return network
def get_ips(self, name=None, ref=None):
"""Get network availability by network name or network UUID."""
aval = self.ips.get(name)
if not aval:
aval = self.ips_by_ref.get(ref)
return aval
async def load_flavors(self):
"""Extend provider configuration by loading all flavors from OpenStack."""
resp = await self.nova.flavors.list()
flavors = resp["flavors"]
self.set_flavors(flavors)
return flavors
async def load_images(self, image_names=None):
"""
Extend provider configuration by loading information about images.
Load everything if image_names list is not specified.
Specifying list of images to load might improve performance if the
OpenStack instance contains a lot of images.
"""
params = {"limit": 1000}
if image_names:
image_filter = ",".join(image_names)
image_filter = "in:" + image_filter
params["name"] = image_filter
images = []
response = await self.glance.images.list(**params)
images.extend(response["images"])
while response.get("next"):
p_result = urlparse(response.get("next"))
query = p_result.query
next_params = parse_qs(query)
for key, val in next_params.items():
if type(val) == list and len(val):
next_params[key] = val[0]
response = await self.glance.images.list(**next_params)
images.extend(response["images"])
self.set_images(images)
return images
async def load_networks(self):
"""Extend provider configuration by loading all networks from OpenStack."""
resp = await self.neutron.network.list()
networks = resp["networks"]
self.set_networks(networks)
return networks
async def load_ip_availabilities(self):
"""Extend provider configuration by loading networks availabilities."""
resp = await self.neutron.ip.list()
availabilities = resp["network_ip_availabilities"]
for availability in availabilities:
self.ips[availability["network_name"]] = availability
self.ips_by_ref[availability["network_id"]] = availability
return availabilities
def _translate_flavor(self, req):
flavor_spec = req.get("flavor")
flavor_ref = req.get("flavorRef")
flavor = None
if flavor_ref:
flavor = self.get_flavor(ref=flavor_ref)
if flavor_spec:
flavor = self.get_flavor(flavor_spec, flavor_spec)
if not flavor:
specs = f"flavor: {flavor_spec}, ref: {flavor_ref}"
raise ValidationError(f"Flavor not found: {specs}")
return flavor
def _translate_image(self, req):
image_spec = req.get("image")
image_ref = req.get("imageRef")
image = None
if image_ref:
image = self.get_image(ref=image_ref)
if image_spec:
image = self.get_image(image_spec, image_spec)
if not image:
specs = f"image: {image_spec}, ref: {image_ref}"
raise ValidationError(f"Image not found {specs}")
return image
def _translate_networks(self, req, spec=False):
network_req = req.get("network")
network_specs = req.get("networks", [])
network_specs = deepcopy(network_specs)
networks = []
if type(network_specs) != list:
network_specs = []
for network_spec in network_specs:
uuid = network_spec.get("uuid")
network = self.get_network(ref=uuid)
if not network:
raise ValidationError(f"Network not found: {network_spec}")
networks.append(network)
if network_req:
network = self.get_network(name=network_req, ref=network_req)
if not network:
raise ValidationError(f"Network not found: {network_req}")
network_specs.append({"uuid": network["id"]})
networks.append(network)
if spec:
return network_specs
return networks
def validate_host(self, req):
"""Validate that host requirements contains existing required objects."""
self._translate_flavor(req)
self._translate_image(req)
self._translate_networks(req)
return True
async def validate_hosts(self, reqs):
"""Validate that all hosts requirements contains existing required objects."""
for req in reqs:
self.validate_host(req)
def get_host_requirements(self, req):
"""Get vCPU and memory requirements for host requirement."""
flavor_spec = req.get("flavor")
flavor_ref = req.get("flavorRef")
if flavor_ref:
flavor = self.get_flavor(ref=flavor_ref)
if flavor_spec:
flavor = self.get_flavor(flavor_spec, flavor_spec)
return {"ram": flavor["ram"], "vcpus": flavor["vcpus"]}
async def can_provision(self, reqs):
"""Check that all host can be provisioned.
Checks:
* available vCPUs and memory based on account limits
* that all host contain available flavors, images, networks
"""
vcpus = 0
ram = 0
for req in reqs:
needs = self.get_host_requirements(req)
vcpus += needs["vcpus"]
ram += needs["ram"]
limits = self.limits["limits"]["absolute"]
used_vcpus = limits["totalCoresUsed"]
used_memory = limits["totalRAMUsed"]
limit_vcpus = limits["maxTotalCores"]
limit_memory = limits["maxTotalRAMSize"]
req_vcpus = used_vcpus + vcpus
req_memory = used_memory + ram
logger.info(
f"Required vcpus: {vcpus}, " f"used: {used_vcpus}, max: {limit_vcpus}"
)
logger.info(f"Required ram: {ram}, used: {used_memory}, max: {limit_memory}")
return req_vcpus <= limit_vcpus and req_memory <= limit_memory
async def create_server(self, req):
"""Issue creation of a server.
req - dict of server requirements - can contains values defined in
POST /servers official OpenStack API
https://docs.openstack.org/api-ref/compute/?expanded=create-server-detail#create-server
The req object can contain following additional attributes:
* 'flavor': uuid or name of flavor to use
* 'network': uuid or name of network to use. Will be added to networks
list if present
"""
logger.info("Creating OpenStack server")
specs = deepcopy(req) # work with own copy, do not modify the input
flavor = self._translate_flavor(req)
specs["flavorRef"] = flavor["id"]
if specs.get("flavor"):
del specs["flavor"]
image = self._translate_image(req)
specs["imageRef"] = image["id"]
if specs.get("image"):
del specs["image"]
network_specs = self._translate_networks(req, spec=True)
specs["networks"] = network_specs
if specs.get("network"):
del specs["network"]
response = await self.nova.servers.create(server=specs)
return response.get("server")
async def delete_server(self, uuid):
"""Issue deletion of server.
Doesn't wait for the deletion to happen.
"""
try:
await self.nova.servers.force_delete(uuid)
except NotFoundError:
logger.warning(f"Server '{uuid}' not found, probably already deleted")
pass
def parse_errors(self, server_results):
"""Parse provisioning errors from provider result."""
errors = []
for res in server_results:
if self.STATUS_MAP.get(res["status"], STATUS_OTHER) == STATUS_ERROR:
errors.append(res)
return errors
async def wait_till_provisioned(
self, instance, timeout=None, poll_sleep=None, poll_sleep_initial=None
):
"""
Wait till server is provisioned.
Provisioned means that server is in ACTIVE or ERROR state
State is checked by polling. Polling can be controller via `poll_sleep` and
`poll_sleep_initial` options. This is useful when provisioning a lot of
machines as it is better to increase initial poll to not ask to often as
provisioning resources takes some time.
Waits till timeout happens. Timeout can be either specified or default provider
timeout is used.
Return information about provisioned server.
"""
uuid = instance.get("id")
if not poll_sleep_initial:
poll_sleep_initial = self.poll_sleep_initial
if not poll_sleep:
poll_sleep = self.poll_sleep
if not timeout:
timeout = self.timeout
start = datetime.now()
timeout_time = start + timedelta(minutes=timeout)
done_states = ["ACTIVE", "ERROR"]
# do not check the state immediately, it will take some time
await asyncio.sleep(poll_sleep_initial)
while datetime.now() < timeout_time:
try:
resp = await self.nova.servers.get(uuid)
except NotFoundError:
raise ServerNotFoundError(uuid)
server = resp["server"]
if server["status"] in done_states:
break
await asyncio.sleep(poll_sleep)
done_time = datetime.now()
prov_duration = (done_time - start).total_seconds()
if datetime.now() >= timeout_time:
logger.warning(
f"{uuid} was not provisioned within a timeout of" f" {timeout} mins"
)
else:
logger.info(f"{uuid} was provisioned in {prov_duration:.1f}s")
return server
async def delete_host(self, host):
"""Issue deletion of host(server) from OpenStack."""
logger.info(f"Deleting OpenStack host {host.id}")
await self.delete_server(host._id)
return True
def prov_result_to_host_data(self, prov_result):
"""Get needed host infromation from openstack provisioning result."""
result = {
"id": None,
"name": None,
"addresses": None,
"status": None,
"fault": None,
}
result["id"] = prov_result.get("id")
result["name"] = prov_result.get("name")
networks = prov_result.get("addresses", {})
result["addresses"] = [ip.get("addr") for n in networks.values() for ip in n]
result["fault"] = prov_result.get("fault")
result["status"] = self.STATUS_MAP.get(prov_result.get("status"), STATUS_OTHER)
return result