def __init__(self, data={}, plan=None, name=None):
""" Initialize and check the step data """
super().__init__(name=name, parent=plan)
# Initialize data
self.plan = plan
self.data = data
self._status = None
self._plugins = []
# Create an empty step by default (can be updated from cli)
if self.data is None:
self.data = [{'name': tmt.utils.DEFAULT_NAME}]
# Convert to list if only a single config provided
elif isinstance(self.data, dict):
# Give it a name unless defined
if not self.data.get('name'):
self.data['name'] = tmt.utils.DEFAULT_NAME
self.data = [self.data]
# Shout about invalid configuration
elif not isinstance(self.data, list):
raise GeneralError(f"Invalid '{self}' config in '{self.plan}'.")
# Final sanity checks
for data in self.data:
# Set 'how' to the default if not specified
if data.get('how') is None:
data['how'] = self.how
# Ensure that each config has a name
if 'name' not in data and len(self.data) > 1:
raise GeneralError(f"Missing '{self}' name in '{self.plan}'.")
def plugin_install(self, name):
""" Install a vagrant plugin if it's not installed yet.
"""
plugin = f'{self.executable}-{name}'
command = ['plugin', 'install']
try:
# is it already present?
run = f"{self.executable} {command[0]} list | grep '^{plugin} '"
return self.run(f"bash -c \"{run}\"")
except GeneralError:
pass
try:
# try to install it
return self.run_vagrant(command[0], command[1], plugin)
except GeneralError as error:
# Let's work-around the error handling limitation for now
# by getting the output manually
command = ' '.join([self.executable] + command + [plugin])
out, err = self.run(f"bash -c \"{command}; :\"")
if re.search(r"Conflicting dependency chains:", err) is None:
raise error
raise GeneralError(
'Dependency conflict detected:\n'
'Please install vagrant plugins from one source only (hint: `dnf remove rubygem-fog-core`).'
)
def execute(self, *args, **kwargs):
if not self.instance:
raise GeneralError('Could not execute without a provisioned VM.')
return self.run(['ssh'] + self.ssh_args + [self.ssh_user_host] +
[f'{self.shell_env} {self.join(args)}'],
shell=False)[0].rstrip()
def install(self, packages):
""" Install specified package(s)
"""
if type(packages) is list:
packages = ' '.join(packages)
## TODO: remove this after run(shell=True) is in provision.prepare()
try:
self.plan.provision.prepare('shell', f"rpm -V {packages}")
except GeneralError:
self.plan.provision.prepare('shell', f"dnf install -y {packages}")
return
## <
failed = False
logf = os.path.join(self.workdir, 'prepare.log')
try:
self.plan.provision.prepare(
'shell',
f"set -o pipefail; ( rpm -V {packages} || sudo dnf install -y {packages} ) 2>&1 | tee -a '{logf}'"
)
except GeneralError:
failed = True
self.plan.provision.sync_workdir_from_guest()
output = open(logf).read()
if failed:
raise GeneralError(f'Install failed:\n{output}')
self.debug(logf, output, 'yellow')
def execute(self, *args, **kwargs):
if not self.container_name:
raise GeneralError(
'Could not execute without provisioned container')
self.info('args', self.join(args), 'red')
self.podman(f'exec {self.container_name} {self.join(args)}')
def get_compose_id(compose_id_url):
response = requests.get(f'{compose_id_url}')
if not response:
raise GeneralError(f'Failed to find compose ID for '
f"'{name}' at '{compose_id_url}'")
return response.text
def go(self):
""" Execute actual provisioning """
self.init()
self.info(f'Provisioning {self.executable}, {self.vf_name}',
self.vf_read())
out, err = self.run_vagrant('up')
status = self.status()
if status != 'running':
raise GeneralError(
f'Failed to provision (status: {status}), log:\n{out}\n{err}')
def execute(self, *args, **kwargs):
""" Execute given commands in podman via shell """
if not self.container_name:
raise GeneralError(
'Could not execute without provisioned container')
# Note that we MUST run commands via bash, so variables
# work as expected
self.podman(['exec'] + self.podman_env +
[self.container_name, 'sh', '-c',
self.join(args)], **kwargs)
def __init__(self, step, data):
""" Store plugin name, data and parent step """
# Ensure that plugin data contains name
if 'name' not in data:
raise GeneralError("Missing 'name' in plugin data.")
# Store name, data and parent step
super().__init__(parent=step, name=data['name'])
self.data = data
self.step = step
def __init__(self, data=None, plan=None, name=None):
""" Initialize and check the step data """
super().__init__(name=name, parent=plan)
# Initialize data
self.plan = plan
self.data = data or {}
self._status = None
self._plugins = []
# Create an empty step by default (can be updated from cli)
if self.data is None:
self.data = [{'name': tmt.utils.DEFAULT_NAME}]
# Convert to list if only a single config provided
elif isinstance(self.data, dict):
# Give it a name unless defined
if not self.data.get('name'):
self.data['name'] = tmt.utils.DEFAULT_NAME
self.data = [self.data]
# Shout about invalid configuration
elif not isinstance(self.data, list):
raise GeneralError(f"Invalid '{self}' config in '{self.plan}'.")
# Add default unique names even to multiple configs so that the users
# don't need to specify it if they don't care about the name
for i, data in enumerate(self.data):
if 'name' not in data:
data['name'] = f'{tmt.utils.DEFAULT_NAME}-{i}'
# Final sanity checks
for data in self.data:
# Set 'how' to the default if not specified
if data.get('how') is None:
data['how'] = self.how
# Ensure that each config has a name
if 'name' not in data and len(self.data) > 1:
raise GeneralError(f"Missing 'name' in the {self} step config "
f"of the '{self.plan}' plan.")
def __init__(self, step, data):
""" Store plugin name, data and parent step """
# Ensure that plugin data contains name
if 'name' not in data:
raise GeneralError("Missing 'name' in plugin data.")
# Store name, data and parent step
super().__init__(parent=step, name=data['name'])
self.data = data
self.step = step
# Initialize plugin order
try:
self.order = int(self.data['order'])
except (ValueError, KeyError):
self.order = tmt.utils.DEFAULT_PLUGIN_ORDER
def status(self, status=None):
"""
Get and set current step status
The meaning of the status is as follows:
todo ... config, data and command line processed (we know what to do)
done ... the final result of the step stored to workdir (we are done)
"""
# Update status
if status is not None:
# Check for valid values
if status not in ['todo', 'done']:
raise GeneralError(f"Invalid status '{status}'.")
# Show status only if changed
elif self._status != status:
self._status = status
self.debug('status', status, color='yellow', level=2)
# Return status
return self._status
def guess_image_url(name):
""" Guess image url for given name """
def get_compose_id(compose_id_url):
response = requests.get(f'{compose_id_url}')
if not response:
raise GeneralError(f'Failed to find compose ID for '
f"'{name}' at '{compose_id_url}'")
return response.text
# map fedora, rawhide or fedora-rawhide to latest rawhide image
if re.match(r'^(fedora|fedora-rawhide|rawhide)$', name, re.IGNORECASE):
compose_id = get_compose_id(RAWHIDE_ID)
compose_name = compose_id.replace('Fedora-Rawhide',
'Fedora-Cloud-Base-Rawhide')
return f'{RAWHIDE_IMAGE_URL}/{compose_name}.x86_64.qcow2'
raise GeneralError("Could not map '{name}' to compose")
def prepare(self, how, what):
""" add single 'preparator' and run it """
name = 'prepare'
cmd = 'provision'
self.vf_backup("Prepare")
# decide what to do
if how == 'ansible':
name = how
# Prepare verbose level based on the --debug option count
verbose = self.opt('debug') * 'v' if self.opt('debug') else 'false'
self.add_config_block(cmd, name, f'become = true',
self.kve('become_user', self.data['user']),
self.kve('playbook', what),
self.kve('verbose', verbose))
# I'm not sure whether this is needed:
# run: 'never'
else:
if self.is_uri(what):
method = 'path'
else:
method = 'inline'
self.add_config('vm', cmd, quote(name), self.kv('type', how),
self.kv('privileged', 'true'),
self.kv('run', 'never'), self.kv(method, what))
try:
self.validate()
except GeneralError as error:
self.vf_restore()
raise GeneralError(
f'Invalid input for vagrant prepare ({how}):\n{what}')
return self.run_vagrant(cmd, f'--{cmd}-with', name)
def __init__(self, data, step):
super(ProvisionTestcloud, self).__init__(data, step)
self._prepare_map = {
'ansible': self._prepare_ansible,
'shell': self._prepare_shell,
}
# Get image from provision options
if not self.option('image'):
raise GeneralError('No image specified')
# Initialize testcloud image
self.image = None
# Testcloud instance and ip
self.instance = None
self.ip = None
# Default user
self.user = self.option('user') or DEFAULT_USER
# Create ssh key
self.ssh_key = os.path.join(self.provision_dir, 'id_rsa')
self.ssh_pubkey = os.path.join(self.provision_dir, 'id_rsa.pub')
# Common ssh args
self.ssh_args = (f'-i {self.ssh_key} -o StrictHostKeyChecking=no '
f'-o UserKnownHostsFile=/dev/null')
# Make sure required directories exist
os.makedirs(TESTCLOUD_DATA, exist_ok=True)
os.makedirs(TESTCLOUD_IMAGES, exist_ok=True)
# Make sure libvirt domain template exists
ProvisionTestcloud._create_template()
def execute(self, *args, **kwargs):
if not self.instance:
raise GeneralError('Could not execute without provisioned VM')
return self._ssh_run(f'{self.join(args)}')
def go(self):
super(ProvisionTestcloud, self).go()
# If image does not start with http/https/file, consider it a mapping
# value and try to guess the URL
image_url = self.option('image') or DEFAULT_IMAGE
if not re.match(r'^(?:https?|file)://.*', image_url):
image_url = guess_image_url(image_url)
# Import testcloud module only when needed (until we have a
# separate package for each plugin)
try:
import testcloud.image
import testcloud.instance
except ImportError:
raise GeneralError(
"Install 'testcloud' to provision using this method.")
# Get configuration
config = testcloud.config.get_config()
# Make sure download progress is disabled, so it
# does not spoil our logging
config.DOWNLOAD_PROGRESS = False
# Configure to tmt's storage directories
config.DATA_DIR = TESTCLOUD_DATA
config.STORE_DIR = TESTCLOUD_IMAGES
# Initialize testcloud image
self.image = testcloud.image.Image(image_url)
# Show which image we are using
self.info('image', f'{self.image.name}', 'green')
status = f'{self.image.name}'
if not os.path.exists(self.image.local_path):
self.info('status', 'downloading', 'green')
# prepare testcloud image
try:
self.image.prepare()
except FileNotFoundError:
raise GeneralError(
f"Could not find image '{self.image.local_path}'")
self.instance = testcloud.instance.Instance(self.instance_name,
image=self.image)
# generate ssh key
self.run(f'ssh-keygen -f {self.ssh_key} -N ""')
with open(self.ssh_pubkey, 'r') as pubkey:
config.USER_DATA = USER_DATA.format(user_name=self.user,
public_key=pubkey.read())
self.info('status', 'booting', 'green')
self.instance.ram = self.option('memory') or DEFAULT_MEMORY
self.instance.disk_size = DEFAULT_DISK_SIZE
self.instance.prepare()
self.instance.spawn_vm()
try:
self.instance.start(DEFAULT_BOOT_TIMEOUT)
except testcloud.exceptions.TestcloudInstanceError:
# TODO: find out how to get detailed information about boot problem
raise GeneralError('Failed to boot instance')
self.ip = self.instance.get_ip()
self.instance.create_ip_file(self.ip)
self.ssh_user_host = f'{self.user}@{self.instance.get_ip()}'
for i in range(1, DEFAULT_SSH_CONNECT_TIMEOUT):
try:
self.execute('whoami')
break
except GeneralError:
self.debug('failed to connect to machine, retrying')
time.sleep(1)
if i == DEFAULT_BOOT_TIMEOUT:
raise GeneralError(
'Failed to login to the machine in {DEFAULT_BOOT_TIMEOUT}s')
self.info('instance', self.ssh_user_host, 'green')