async def validate_hosts(self, hosts):
"""Validate that host requirements are well specified."""
for req in hosts:
req_img = req.get("image")
if not req.get("meta_image") and req_img not in self.ami_ids:
raise ValidationError(
f"{self.dsp_name}: Provider does not support "
f"'{req_img}' image in provisioning config")
try:
aws_image = self.ec2.Image(req_img)
if not aws_image: # user is not authorized to use ami - None returned
raise ValidationError(
f"{self.dsp_name}: User does not have enough permissions "
f"to use image: {req_img}")
logger.info(
f"{self.dsp_name}: Requested provisioning of {aws_image.name} image"
)
except ClientError as image_err:
err_msg = (f"{self.dsp_name}: Requested image "
f"'{req_img}' can not be provisioned")
logger.error(err_msg)
err_resp = image_err.response["Error"]["Message"]
raise ValidationError(
f"{err_msg}. Request failed with {err_resp}")
return
async def validate_hosts(self, reqs):
"""Validate that host requirements are well specified."""
for req in reqs:
if "name" not in req:
raise ValidationError("Name not found")
if "ip" not in req:
raise ValidationError("IP address (ip) not found")
return bool(reqs)
async def prepare_provisioning(self, reqs):
"""Prepare provisioning."""
# Check that current user belongs to testcloud group
group = "testcloud"
uname = getpass.getuser()
_name, _pass, gid, _members = grp.getgrnam(group)
groups = os.getgroups()
if gid not in groups:
raise ValidationError(
f"Error: Current user is not a member of a {group} group.\n"
"Make sure that testcloud is installed and add the user to the "
"group, e.g. by:\n"
f" sudo usermod -a -G {group} {uname}")
# Pulling images ahead so that the provider doesn't download the same
# image more than once
pull = set()
for req in reqs:
pull.add(req["image_url"])
awaitables = []
for url in pull:
logger.info(f"{self.dsp_name}: Pulling image '{url}'")
awaitables.append(self.testcloud.pull_image(url))
pull_results = await asyncio.gather(*awaitables)
success = all(pull_results)
if not success:
logger.error(f"{self.dsp_name}: Pulling of images failed")
else:
logger.info(f"{self.dsp_name}: Images prepared")
return success
async def validate_hosts(self, reqs):
"""Validate that host requirements are well specified."""
for req in reqs:
req_dstr = req.get("distro")
if not req.get("meta_distro") and req_dstr not in self.distros:
raise ValidationError(
f"{self.dsp_name} provider does not support "
f"'{req_dstr}' distro in provisioning config")
return
def _translate_image(self, req):
image_spec = req.get("image")
image_ref = req.get("imageRef")
image = None
if image_ref:
image = self.get_image(ref=image_ref)
if image_spec:
image = self.get_image(image_spec, image_spec)
if not image:
specs = f"image: {image_spec}, ref: {image_ref}"
raise ValidationError(f"Image not found {specs}")
return image
def _translate_flavor(self, req):
flavor_spec = req.get("flavor")
flavor_ref = req.get("flavorRef")
flavor = None
if flavor_ref:
flavor = self.get_flavor(ref=flavor_ref)
if flavor_spec:
flavor = self.get_flavor(flavor_spec, flavor_spec)
if not flavor:
specs = f"flavor: {flavor_spec}, ref: {flavor_ref}"
raise ValidationError(f"Flavor not found: {specs}")
return flavor
def _translate_networks(self, req, spec=False):
network_req = req.get("network")
network_specs = req.get("networks", [])
network_specs = deepcopy(network_specs)
networks = []
if type(network_specs) != list:
network_specs = []
for network_spec in network_specs:
uuid = network_spec.get("uuid")
network = self.get_network(ref=uuid)
if not network:
raise ValidationError(f"Network not found: {network_spec}")
networks.append(network)
if network_req:
network = self.get_network(name=network_req, ref=network_req)
if not network:
raise ValidationError(f"Network not found: {network_req}")
network_specs.append({"uuid": network["id"]})
networks.append(network)
if spec:
return network_specs
return networks
async def _provision_base(self, reqs):
"""Provision hosts based on list of host requirements.
Main function which does provisioning and not any validation.
"""
logger.info(f"{self.dsp_name}: Host definitions valid")
logger.info(f"{self.dsp_name}: Checking available resources")
can = await self.can_provision(reqs)
if not can:
raise ValidationError(f"{self.dsp_name}: Not enough resources to provision")
logger.info(f"{self.dsp_name}: Resource availability: OK")
started = datetime.now()
count = len(reqs)
logger.info(f"{self.dsp_name}: Issuing provisioning of {count} hosts")
create_servers = []
for req in reqs:
awaitable = self.create_server(req)
create_servers.append(awaitable)
create_resps = await asyncio.gather(*create_servers)
logger.info(f"{self.dsp_name}: Provisioning issued")
logger.info(f"{self.dsp_name}: Waiting for all hosts to be available")
wait_servers = []
for create_resp in create_resps:
awaitable = self.wait_till_provisioned(create_resp)
wait_servers.append(awaitable)
server_results = await asyncio.gather(*wait_servers)
provisioned = datetime.now()
provi_duration = provisioned - started
logger.info(
f"{self.dsp_name}: "
"All hosts reached provisioning final state (ACTIVE or ERROR)"
)
logger.info(f"{self.dsp_name}: Provisioning duration: {provi_duration}")
hosts = [self.to_host(srv) for srv in server_results]
error_hosts = self.parse_error_hosts(hosts)
success_hosts = [h for h in hosts if h not in error_hosts]
error_host_names = [host.name for host in error_hosts]
missing_reqs = [req for req in reqs if req["name"] in error_host_names]
return (success_hosts, error_hosts, missing_reqs)
def get_host_requirements(self, req):
"""Get vCPU and memory requirements for host requirement."""
flavor_spec = req.get("flavor")
flavor_ref = req.get("flavorRef")
flavor = None
if flavor_ref:
flavor = self.get_flavor(ref=flavor_ref)
if flavor_spec:
flavor = self.get_flavor(flavor_spec, flavor_spec)
try:
res = {"ram": flavor["ram"], "vcpus": flavor["vcpus"]}
except TypeError as flavor_none:
# func does not load flavor so None is used as result
raise ValidationError(
f"Could not load the flavor for requirement: {req}"
) from flavor_none
return res
async def _provision_base(self,
reqs,
res_check_timeout=60,
res_busy_sleep=10): # pylint: disable=too-many-locals, too-many-branches
"""Provision hosts based on list of host requirements.
Main function which does provisioning and validation.
Parameters:
reqs - dictionary with requirements for provider
res_check_timeout (default 60) - timeout (minutes) to wait for resources
res_busy_sleep (default 10) - time to wait before checking again (minutes)
"""
logger.info(f"{self.dsp_name}: Validating hosts definitions")
if not reqs:
raise ProvisioningError(
f"{self.dsp_name}: Can not continue with empty requirement for provider"
)
await self.validate_hosts(reqs)
logger.info(f"{self.dsp_name}: Host definitions valid")
logger.info(f"{self.dsp_name}: Checking available resources")
res_check_start = datetime.now()
while not await self.can_provision(reqs):
await asyncio.sleep(res_busy_sleep * 60)
if datetime.now() - res_check_start >= timedelta(
minutes=res_check_timeout):
raise ValidationError(
f"{self.dsp_name}: Not enough resources to provision")
logger.info(f"{self.dsp_name}: Resource availability: OK")
started = datetime.now()
logger.info(
f"{self.dsp_name}: Issuing provisioning of {len(reqs)} host(s)")
create_servers = []
for req in reqs:
awaitable = self.create_server(req)
create_servers.append(awaitable)
# expect the exception in return data to be parsed later
create_resps = await asyncio.gather(*create_servers,
return_exceptions=True)
logger.info(f"{self.dsp_name}: Provisioning issued")
logger.info(f"{self.dsp_name}: Waiting for all hosts to be active")
error_hosts = []
wait_servers = []
for response in create_resps:
if not isinstance(response, ProvisioningError):
# response might be okay so let us wait for result
awaitable = self.wait_till_provisioned(response)
wait_servers.append(awaitable)
else:
# use ProvisioningError arguments to create missing Host object
# which we append to error hosts list for later usage
error_hosts.append(
Host(
provider=self,
host_id=None,
name=response.args[SPECS]["name"],
ip_addrs=[],
status=STATUS_OTHER,
rawdata=response.args,
error_obj=response.args,
))
server_results = await asyncio.gather(*wait_servers)
provisioned = datetime.now()
logger.info(
f"{self.dsp_name}: "
"All hosts reached provisioning final state (ACTIVE or ERROR)")
logger.info(
f"{self.dsp_name}: Provisioning duration: {provisioned - started}")
hosts = [self.to_host(srv) for srv in server_results if srv]
error_hosts += await self.parse_error_hosts(hosts)
active_hosts = [h for h in hosts if h not in error_hosts]
success_hosts = []
if global_context["config"].get("post_provisioning_ssh_check", True):
# check ssh connectivity to succeeded hosts
wait_ssh = []
for host in active_hosts:
awaitable = self._wait_for_ssh(host)
wait_ssh.append(awaitable)
ssh_results = await asyncio.gather(*wait_ssh)
# We distinguish the success hosts and new error hosts from active by using:
# res[RET_CODE] 0
# - the result of operation returned from self._wait_for_ssh()
# res[HOST_OBJ] 1
# - the host object returned from self._wait_for_ssh()
for res in ssh_results:
if res[RET_CODE]:
success_hosts.append(res[HOST_OBJ])
else:
res[HOST_OBJ].error = (
"Could not establish ssh connection to host "
f"{res[HOST_OBJ].host_id} with IP {res[HOST_OBJ].ip_addr}"
)
error_hosts.append(res[HOST_OBJ])
else: # we do not check the ssh connection to VMs
success_hosts = active_hosts
missing_reqs = [
req for req in reqs
if req["name"] in [host.name for host in error_hosts]
]
return (success_hosts, error_hosts, missing_reqs)
async def provision_hosts(self, hosts):
"""Provision hosts based on list of host requirements.
Main provider method for provisioning.
First it validates that host requirements are valid and that
provider has enough resources(quota).
Then issues provisioning and waits for it succeed. Raises exception if any of
the servers was not successfully provisioned. If that happens it issues deletion
of all already provisioned resources.
Return list of information about provisioned servers.
"""
logger.info("Validating hosts definitions")
await self.validate_hosts(hosts)
logger.info("Host definitions valid")
logger.info("Checking available resources")
can = await self.can_provision(hosts)
if not can:
raise ValidationError("Not enough resources to provision")
logger.info("Resource availability: OK")
started = datetime.now()
count = len(hosts)
logger.info(f"Issuing provisioning of {count} hosts")
create_servers = []
for req in hosts:
awaitable = self.create_server(req)
create_servers.append(awaitable)
create_resps = await asyncio.gather(*create_servers)
logger.info("Provisioning issued")
logger.info("Waiting for all hosts to be available")
wait_servers = []
for create_resp in create_resps:
awaitable = self.wait_till_provisioned(create_resp)
wait_servers.append(awaitable)
server_results = await asyncio.gather(*wait_servers)
provisioned = datetime.now()
provi_duration = provisioned - started
logger.info(
"All hosts reached provisioning final state (ACTIVE or ERROR)")
logger.info(f"Provisioning duration: {provi_duration}")
errors = self.parse_errors(server_results)
if errors:
logger.info("Some host did not start properly")
for err in errors:
self.print_basic_info(err)
logger.info("Given the error, will delete all hosts")
await self.delete_hosts(server_results)
raise ProvisioningError(errors)
hosts = [self.to_host(srv) for srv in server_results]
for host in hosts:
logger.info(host)
return hosts
async def _provision_base(self, reqs): # pylint: disable=too-many-locals
"""Provision hosts based on list of host requirements.
Main function which does provisioning and not any validation.
"""
logger.info(f"{self.dsp_name}: Validating hosts definitions")
if not reqs:
raise ProvisioningError(
f"{self.dsp_name}: Can not continue with empty requirement for provider"
)
await self.validate_hosts(reqs)
logger.info(f"{self.dsp_name}: Host definitions valid")
logger.info(f"{self.dsp_name}: Checking available resources")
if not await self.can_provision(reqs):
raise ValidationError(
f"{self.dsp_name}: Not enough resources to provision")
logger.info(f"{self.dsp_name}: Resource availability: OK")
started = datetime.now()
logger.info(
f"{self.dsp_name}: Issuing provisioning of {len(reqs)} host(s)")
create_servers = []
for req in reqs:
awaitable = self.create_server(req)
create_servers.append(awaitable)
create_resps = await asyncio.gather(*create_servers)
logger.info(f"{self.dsp_name}: Provisioning issued")
logger.info(f"{self.dsp_name}: Waiting for all hosts to be active")
wait_servers = []
for create_resp in create_resps:
awaitable = self.wait_till_provisioned(create_resp)
wait_servers.append(awaitable)
server_results = await asyncio.gather(*wait_servers)
provisioned = datetime.now()
logger.info(
f"{self.dsp_name}: "
"All hosts reached provisioning final state (ACTIVE or ERROR)")
logger.info(
f"{self.dsp_name}: Provisioning duration: {provisioned - started}")
hosts = [self.to_host(srv) for srv in server_results]
error_hosts = await self.parse_error_hosts(hosts)
active_hosts = [h for h in hosts if h not in error_hosts]
success_hosts = []
if global_context["config"].get("post_provisioning_ssh_check", True):
# check ssh connectivity to succeeded hosts
wait_ssh = []
for host in active_hosts:
awaitable = self._wait_for_ssh(host)
wait_ssh.append(awaitable)
ssh_results = await asyncio.gather(*wait_ssh)
# We distinguish the success hosts and new error hosts from active by using:
# res[RET_CODE] 0
# - the result of operation returned from self._wait_for_ssh()
# res[HOST_OBJ] 1
# - the host object returned from self._wait_for_ssh()
for res in ssh_results:
if res[RET_CODE]:
success_hosts.append(res[HOST_OBJ])
else:
res[HOST_OBJ].error = (
"Could not establish ssh connection to host "
f"{res[HOST_OBJ].host_id} with IP {res[HOST_OBJ].ip_addr}"
)
error_hosts.append(res[HOST_OBJ])
else: # we do not check the ssh connection to VMs
success_hosts = active_hosts
missing_reqs = [
req for req in reqs
if req["name"] in [host.name for host in error_hosts]
]
return (success_hosts, error_hosts, missing_reqs)
