def latest_release():
""" Get the latest released Fedora number """
try:
response = self._get_url(KOJI_URL, 'check Fedora composes')
releases = re.findall(r'>(\d\d)/<', response.text)
return releases[-1]
except IndexError:
raise ProvisionError(
f"Latest Fedora release not found at '{KOJI_URL}'.")
def go(self):
""" Provision the guest """
super().go()
api_version = self.get('api-version')
if api_version not in SUPPORTED_API_VERSIONS:
raise ProvisionError(f"API version '{api_version}' not supported.")
try:
user_data = {
key.strip(): value.strip()
for key, value in (pair.split('=', 1)
for pair in self.get('user-data'))
}
except ValueError as exc:
raise ProvisionError('Cannot parse user-data.')
data: GuestDataType = {
'api-url': self.get('api-url'),
'api-version': api_version,
'arch': self.get('arch'),
'image': self.get('image'),
'hardware': self.get('hardware'),
'pool': self.get('pool'),
'priority-group': self.get('priority-group'),
'keyname': self.get('keyname'),
'user-data': user_data,
'guestname': None,
'guest': None,
'user': DEFAULT_USER,
'provision-timeout': self.get('provision-timeout'),
'provision-tick': self.get('provision-tick'),
'api-timeout': self.get('api-timeout'),
'api-retries': self.get('api-retries'),
'api-retry-backoff-factor': self.get('api-retry-backoff-factor')
}
self._guest = GuestArtemis(data, name=self.name, parent=self.step)
self._guest.start()
def _guess_image_url(self, name):
""" Guess image url for given name """
def latest_release():
""" Get the latest released Fedora number """
try:
response = self._get_url(KOJI_URL, 'check Fedora composes')
releases = re.findall(r'>(\d\d)/<', response.text)
return releases[-1]
except IndexError:
raise ProvisionError(
f"Latest Fedora release not found at '{KOJI_URL}'.")
# Try to check if given url is a local file
if os.path.exists(name):
return f'file://{name}'
# Map fedora aliases (e.g. rawhide, fedora, fedora-32, f-32, f32)
name = name.lower().strip()
matched = re.match(r'^f(edora)?-?(\d+)$', name)
if matched:
release = matched.group(2)
elif 'rawhide' in name:
release = 'rawhide'
elif name == 'fedora':
release = latest_release()
else:
raise ProvisionError(f"Could not map '{name}' to compose.")
# Prepare the full qcow name
images = f"{KOJI_URL}/{release}/latest-Fedora-{release.capitalize()}"
images += "/compose/Cloud/x86_64/images"
response = self._get_url(images, 'get the full qcow name')
matched = re.search(">(Fedora-Cloud[^<]*qcow2)<", response.text)
try:
compose_name = matched.group(1)
except AttributeError:
raise ProvisionError(
f"Failed to detect full compose name from '{images}'.")
return f'{images}/{compose_name}'
def _get_url(self, url, message):
""" Get url, retry when fails, return response """
for i in range(1, DEFAULT_CONNECT_TIMEOUT):
try:
response = retry_session().get(url)
if response.ok:
return response
except requests.RequestException:
pass
self.debug(f"Unable to {message} ({url}), retry {i}.")
time.sleep(3)
raise ProvisionError(
f'Failed to {message} ({DEFAULT_CONNECT_TIMEOUT} attempts).')
def import_testcloud():
"""
Import testcloud module only when needed
Until we have a separate package for each plugin.
"""
global testcloud
try:
import testcloud.image
import testcloud.instance
except ImportError:
raise ProvisionError(
"Install 'testcloud' to provision using this method.")
def _guess_image_url(self, name):
""" Guess image url for given name """
# Try to check if given url is a local file
if os.path.isabs(name) and os.path.isfile(name):
return f'file://{name}'
name = name.lower().strip()
url = None
# Map fedora aliases (e.g. rawhide, fedora, fedora-32, f-32, f32)
matched_fedora = re.match(r'^f(edora)?-?(\d+)$', name)
# Map centos aliases (e.g. centos:X, centos, centos-stream:X)
matched_centos = [
re.match(r'^c(entos)?-?(\d+)$', name),
re.match(r'^c(entos-stream)?-?(\d+)$', name)
]
matched_ubuntu = re.match(r'^u(buntu)?-?(\w+)$', name)
matched_debian = re.match(r'^d(ebian)?-?(\w+)$', name)
# Plain name match means we want the latest release
if name == 'fedora':
url = testcloud.util.get_fedora_image_url("latest")
elif name == 'centos':
url = testcloud.util.get_centos_image_url("latest")
elif name == 'centos-stream':
url = testcloud.util.get_centos_image_url("latest", stream=True)
elif name == 'ubuntu':
url = testcloud.util.get_ubuntu_image_url("latest")
elif name == 'debian':
url = testcloud.util.get_debian_image_url("latest")
elif matched_fedora:
url = testcloud.util.get_fedora_image_url(matched_fedora.group(2))
elif matched_centos[0]:
url = testcloud.util.get_centos_image_url(
matched_centos[0].group(2))
elif matched_centos[1]:
url = testcloud.util.get_centos_image_url(
matched_centos[1].group(2), stream=True)
elif matched_ubuntu:
url = testcloud.util.get_ubuntu_image_url(matched_ubuntu.group(2))
elif matched_debian:
url = testcloud.util.get_debian_image_url(matched_debian.group(2))
elif 'rawhide' in name:
url = testcloud.util.get_fedora_image_url("rawhide")
if not url:
raise ProvisionError(f"Could not map '{name}' to compose.")
return url
def _get_url(self, url, message):
""" Get url, retry when fails, return response """
timeout = DEFAULT_CONNECT_TIMEOUT
wait = 1
while True:
try:
response = retry_session().get(url)
if response.ok:
return response
except requests.RequestException:
pass
if timeout < 0:
raise ProvisionError(
f'Failed to {message} in {DEFAULT_CONNECT_TIMEOUT}s.')
self.debug(f'Unable to {message} ({url}), retrying, '
f'{fmf.utils.listed(timeout, "second")} left.')
time.sleep(wait)
wait += 1
timeout -= wait
def _create(self) -> None:
environment: Dict[str, Any] = {
'hw': {
'arch': self.arch
},
'os': {
'compose': self.image
}
}
data: Dict[str, Any] = {
'environment': environment,
'keyname': self.keyname,
'priority_group': self.priority_group,
'user_data': self.user_data
}
if self.pool:
environment['pool'] = self.pool
if self.hardware is not None:
assert isinstance(self.hardware, dict)
environment['hw']['constraints'] = self.hardware
# TODO: snapshots
# TODO: spot instance
# TODO: post-install script
# TODO: log types
response = self.api.create('/guests/', data)
if response.status_code == 201:
self.info('guest', 'has been requested', 'green')
else:
raise ProvisionError(
f"Failed to create, "
f"unhandled API response '{response.status_code}'.")
self.guestname = response.json()['guestname']
self.info('guestname', self.guestname, 'green')
deadline = datetime.datetime.utcnow() + datetime.timedelta(
seconds=self.provision_timeout)
while deadline > datetime.datetime.utcnow():
response = self.api.inspect(f'/guests/{self.guestname}')
if response.status_code != 200:
raise ProvisionError(
f"Failed to create, "
f"unhandled API response '{response.status_code}'.")
current = cast(GuestInspectType, response.json())
state = current['state']
if state == 'error':
raise ProvisionError(f'Failed to create, provisioning failed.')
if state == 'ready':
self.guest = current['address']
self.info('address', self.guest, 'green')
break
time.sleep(self.provision_tick)
else:
raise ProvisionError(
f'Failed to provision in the given amount '
f'of time (--provision-timeout={self.provision_timeout}).')
def start(self):
""" Start provisioned guest """
if self.opt('dry'):
return
# Make sure required directories exist
os.makedirs(TESTCLOUD_DATA, exist_ok=True)
os.makedirs(TESTCLOUD_IMAGES, exist_ok=True)
# Make sure libvirt domain template exists
GuestTestcloud._create_template()
# Prepare config
self.prepare_config()
# If image does not start with http/https/file, consider it a
# mapping value and try to guess the URL
if not re.match(r'^(?:https?|file)://.*', self.image_url):
self.image_url = self._guess_image_url(self.image_url)
# Initialize and prepare testcloud image
self.image = testcloud.image.Image(self.image_url)
self.verbose('qcow', self.image.name, 'green')
if not os.path.exists(self.image.local_path):
self.info('progress', 'downloading...', 'cyan')
try:
self.image.prepare()
except FileNotFoundError as error:
raise ProvisionError(f"Image '{self.image.local_path}' not found.",
original=error)
except (testcloud.exceptions.TestcloudPermissionsError,
PermissionError) as error:
raise ProvisionError(
f"Failed to prepare the image. Check the '{TESTCLOUD_IMAGES}' "
f"directory permissions.",
original=error)
# Create instance
_, run_id = os.path.split(self.parent.plan.my_run.workdir)
self.instance_name = self._random_name(
prefix="tmt-{0}-".format(run_id[-3:]))
self.instance = testcloud.instance.Instance(
name=self.instance_name,
image=self.image,
connection='qemu:///session')
self.verbose('name', self.instance_name, 'green')
# Prepare ssh key
self.prepare_ssh_key()
# Boot the virtual machine
self.info('progress', 'booting...', 'cyan')
self.instance.ram = self.memory
self.instance.disk_size = self.disk
self.instance.prepare()
self.instance.spawn_vm()
try:
self.instance.start(DEFAULT_BOOT_TIMEOUT)
except (testcloud.exceptions.TestcloudInstanceError,
libvirt.libvirtError) as error:
raise ProvisionError(
f'Failed to boot testcloud instance ({error}).')
self.guest = self.instance.get_ip()
self.port = self.instance.get_instance_port()
self.verbose('ip', self.guest, 'green')
self.verbose('port', self.port, 'green')
self.instance.create_ip_file(self.guest)
# Wait a bit until the box is up
timeout = DEFAULT_CONNECT_TIMEOUT
wait = 1
while True:
try:
self.execute('whoami')
break
except tmt.utils.RunError:
if timeout < 0:
raise ProvisionError(
f'Failed to connect in {DEFAULT_CONNECT_TIMEOUT}s.')
self.debug(f'Failed to connect to machine, retrying, '
f'{fmf.utils.listed(timeout, "second")} left.')
time.sleep(wait)
wait += 1
timeout -= wait
def start(self):
""" Start provisioned guest """
if self.opt('dry'):
return
# Make sure required directories exist
os.makedirs(TESTCLOUD_DATA, exist_ok=True)
os.makedirs(TESTCLOUD_IMAGES, exist_ok=True)
# Make sure libvirt domain template exists
GuestTestcloud._create_template()
# Prepare config
self.prepare_config()
# If image does not start with http/https/file, consider it a
# mapping value and try to guess the URL
if not re.match(r'^(?:https?|file)://.*', self.image_url):
self.image_url = self._guess_image_url(self.image_url)
# Initialize and prepare testcloud image
self.image = testcloud.image.Image(self.image_url)
self.verbose('qcow', self.image.name, 'green')
if not os.path.exists(self.image.local_path):
self.info('progress', 'downloading...', 'cyan')
try:
self.image.prepare()
except FileNotFoundError:
raise ProvisionError(f"Image '{self.image.local_path}' not found.")
except testcloud.exceptions.TestcloudPermissionsError:
raise ProvisionError(
f"Failed to prepare the image. "
f"Check the '{TESTCLOUD_IMAGES}' directory permissions.")
# Create instance
self.instance_name = self._random_name()
self.instance = testcloud.instance.Instance(name=self.instance_name,
image=self.image)
self.verbose('name', self.instance_name, 'green')
# Prepare ssh key
self.prepare_ssh_key()
# Boot the virtual machine
self.info('progress', 'booting...', 'cyan')
self.instance.ram = self.memory
self.instance.disk_size = self.disk
self.instance.prepare()
self.instance.spawn_vm()
try:
self.instance.start(DEFAULT_BOOT_TIMEOUT)
except testcloud.exceptions.TestcloudInstanceError as error:
raise ProvisionError(
f'Failed to boot testcloud instance ({error}).')
self.guest = self.instance.get_ip()
self.instance.create_ip_file(self.guest)
# Wait a bit until the box is up
for i in range(1, DEFAULT_CONNECT_TIMEOUT):
try:
self.execute('whoami')
break
except tmt.utils.RunError:
self.debug('Failed to connect to machine, retrying.')
time.sleep(1)
if i == DEFAULT_CONNECT_TIMEOUT:
raise ProvisionError(
'Failed to connect in {DEFAULT_CONNECT_TIMEOUT}s.')
def start(self):
""" Start provisioned guest """
if self.opt('dry'):
return
# Make sure required directories exist
os.makedirs(TESTCLOUD_DATA, exist_ok=True)
os.makedirs(TESTCLOUD_IMAGES, exist_ok=True)
# Make sure libvirt domain template exists
GuestTestcloud._create_template()
# Prepare config
self.prepare_config()
# If image does not start with http/https/file, consider it a
# mapping value and try to guess the URL
if not re.match(r'^(?:https?|file)://.*', self.image_url):
self.image_url = self._guess_image_url(self.image_url)
self.debug(f"Guessed image url: '{self.image_url}'", level=3)
# Initialize and prepare testcloud image
self.image = testcloud.image.Image(self.image_url)
self.verbose('qcow', self.image.name, 'green')
if not os.path.exists(self.image.local_path):
self.info('progress', 'downloading...', 'cyan')
try:
self.image.prepare()
except FileNotFoundError as error:
raise ProvisionError(f"Image '{self.image.local_path}' not found.",
original=error)
except (testcloud.exceptions.TestcloudPermissionsError,
PermissionError) as error:
raise ProvisionError(
f"Failed to prepare the image. Check the '{TESTCLOUD_IMAGES}' "
f"directory permissions.",
original=error)
# Create instance
_, run_id = os.path.split(self.parent.plan.my_run.workdir)
self.instance_name = self._random_name(
prefix="tmt-{0}-".format(run_id[-3:]))
self.instance = testcloud.instance.Instance(
name=self.instance_name,
image=self.image,
connection='qemu:///session')
self.verbose('name', self.instance_name, 'green')
# Decide which networking setup to use
# Autodetect works with libguestfs python bindings
# We fall back to basic heuristics based on file name
# without that installed (eg. from pypi).
# https://bugzilla.redhat.com/show_bug.cgi?id=1075594
try:
import guestfs
except ImportError:
match_legacy = re.search(r'(rhel|centos).*-7',
self.image_url.lower())
if match_legacy:
self.instance.pci_net = "e1000"
else:
self.instance.pci_net = "virtio-net-pci"
# Prepare ssh key
self.prepare_ssh_key()
# Boot the virtual machine
self.info('progress', 'booting...', 'cyan')
self.instance.ram = self.memory
self.instance.disk_size = self.disk
try:
self.instance.prepare()
self.instance.spawn_vm()
self.instance.start(DEFAULT_BOOT_TIMEOUT)
except (testcloud.exceptions.TestcloudInstanceError,
libvirt.libvirtError) as error:
raise ProvisionError(
f'Failed to boot testcloud instance ({error}).')
self.guest = self.instance.get_ip()
self.port = self.instance.get_instance_port()
self.verbose('ip', self.guest, 'green')
self.verbose('port', self.port, 'green')
self.instance.create_ip_file(self.guest)
# Wait a bit until the box is up
timeout = DEFAULT_CONNECT_TIMEOUT
wait = 1
while True:
try:
self.execute('whoami')
break
except tmt.utils.RunError:
if timeout < 0:
raise ProvisionError(
f'Failed to connect in {DEFAULT_CONNECT_TIMEOUT}s.')
self.debug(f'Failed to connect to machine, retrying, '
f'{fmf.utils.listed(timeout, "second")} left.')
time.sleep(wait)
wait += 1
timeout -= wait