def bootstrap(enable_kvm=False):
"""
Prepare the environment for execution
:params enable_kvm: Flag to enable kvm environment bootstrap
"""
env_clean()
logger.info("Bootstrapping Avocado")
get_repo(AVOCADO_REPO, BASE_PATH, True)
if enable_kvm:
kvm_bootstrap()
helper.runcmd('mkdir -p %s' % TEST_DIR,
debug_str="Creating test repo dir %s" % TEST_DIR,
err_str="Failed to create test repo dir. Error: ")
for repo in TEST_REPOS:
get_repo(repo, TEST_DIR)
if os.path.isdir(prescript):
if len(os.listdir(prescript)):
if not os.path.exists(prescript_dir):
os.makedirs(prescript_dir)
helper.copy_dir_file(prescript, prescript_dir)
if os.path.isdir(postscript):
if len(os.listdir(postscript)):
if not os.path.exists(postscript_dir):
os.makedirs(postscript_dir)
helper.copy_dir_file(postscript, postscript_dir)
def get_repo(repo, basepath, install=False):
"""
To get given repo cloned/updated and install
:param repo: repo link
:param basepath: base path where the repository has to be downloaded
:param install: To enable the flag to install the repo
"""
repo_name = repo.split('/')[-1].split('.')[0]
repo_path = os.path.join(basepath, repo_name)
if os.path.isdir(repo_path) and ('-b ' or '--branch ' in repo):
shutil.rmtree(repo_path)
if os.path.isdir(repo_path):
cmd = "cd %s;git pull --no-edit" % repo_path
helper.runcmd(cmd,
info_str="Updating the repo: %s in %s" %
(repo_name, repo_path),
err_str="Failed to update %s repository:" % repo_name)
else:
cmd = "cd %s;git clone %s %s" % (basepath, repo, repo_name)
helper.runcmd(cmd,
info_str="Cloning the repo: %s in %s" %
(repo_name, repo_path),
err_str="Failed to clone %s repository:" % repo_name)
if install:
install_repo(repo_path, repo_name)
def guest_download(guestos):
"""
Guest image downloading
"""
avocado_bin = helper.get_avocado_bin()
cmd = '%s vt-bootstrap --vt-guest-os %s --yes-to-all' % (avocado_bin,
guestos)
helper.runcmd(cmd, err_str="Failed to Download Guest OS. Error:",
info_str="Downloading the guest os image")
def env_clean():
"""
Clean/uninstall avocado and autotest
"""
for package in ['avocado', 'avocado_plugins_vt', 'autotest']:
cmd = "pip uninstall %s -y --disable-pip-version-check" % package
helper.runcmd(cmd, ignore_status=True,
err_str="Error in removing package: %s" % package,
debug_str="Uninstalling %s" % package)
def install_repo(path, name):
"""
Install the given repo
:param repo: repository path
:param name: name of the repository
"""
cmd = "cd %s;make requirements;make requirements-selftests;python setup.py install" % path
helper.runcmd(cmd, info_str="Installing %s from %s" % (name, path),
err_str="Failed to install %s repository:" % name)
def env_check(enable_kvm):
"""
Check if the environment is proper
"""
logger.info("Check for environment")
# create a folder to store all edited multiplexer files
if not os.path.isdir("/tmp/mux/"):
logger.info("Creating temporary mux dir")
os.makedirs("/tmp/mux/")
not_found = []
(env_ver, env_type, cmd_pat) = helper.get_env_type(enable_kvm)
# try to check base packages using major version numbers
env_ver = env_ver.split('.')[0]
env_deps = []
if not CONFIGFILE.has_section('deps_%s' % env_ver):
# Fallback to base name if specific version is not found
dist = helper.get_dist()
env_ver = dist[0]
if CONFIGFILE.has_section('deps_%s' % env_ver):
packages = CONFIGFILE.get('deps_%s' % env_ver, 'packages')
if packages != '':
env_deps = packages.split(',')
for dep in env_deps:
if helper.runcmd(cmd_pat % dep, ignore_status=True)[0] != 0:
not_found.append(dep)
env_deps = []
# try to check env specific packages
if CONFIGFILE.has_section('deps_%s_%s' % (env_ver, env_type)):
packages = CONFIGFILE.get('deps_%s_%s' % (env_ver, env_type),
'packages')
if packages != '':
env_deps = packages.split(',')
for dep in env_deps:
if helper.runcmd(cmd_pat % dep, ignore_status=True)[0] != 0:
not_found.append(dep)
if not_found:
if args.no_deps_check:
logger.warning(
"No dependancy check flag is set, proceeding with bootstrap")
logger.info("Please install following "
"dependancy packages %s", " ".join(not_found))
elif args.install_deps:
logger.warning("Installing missing packages %s",
" ".join(not_found))
if helper.install_packages(not_found):
logger.error("Some packages not installed")
sys.exit(1)
else:
logger.error("Please install following "
"dependancy packages %s", " ".join(not_found))
sys.exit(1)
def pip_install():
"""
install package using pip
"""
logger.info("install packages via pip interface")
pip_cmd = 'pip%s' % sys.version_info[0]
if CONFIGFILE.has_section('pip-package'):
package = CONFIGFILE.get('pip-package', 'package').split(',')
for dep in package:
cmd = '%s install %s' % (pip_cmd, dep)
helper.runcmd(cmd, err_str='Package installation via pip failed: package %s' % dep,
debug_str='Installing python package %s using pip' % dep)
def kvm_bootstrap():
"""
Prepare KVM Test environment
"""
get_repo(AVOCADO_VT_REPO, BASE_PATH, True)
avocado_bin = helper.get_avocado_bin()
libvirt_cmd = '%s vt-bootstrap --vt-type libvirt \
--vt-update-providers --vt-skip-verify-download-assets \
--yes-to-all' % avocado_bin
helper.runcmd(libvirt_cmd, err_str="Failed to bootstrap vt libvirt. Error:",
info_str="Bootstrapping vt libvirt")
qemu_cmd = '%s vt-bootstrap --vt-type qemu --vt-update-providers \
--vt-skip-verify-download-assets --yes-to-all' % avocado_bin
helper.runcmd(qemu_cmd, err_str="Failed to bootstrap vt qemu. Error:",
info_str="Bootstrapping vt qemu")
def install_optional_plugin(plugin):
"""
To install optional avocado plugin
:param plugin: optional plugin name
"""
if not is_avocado_plugin_avl(plugin):
plugin_path = "%s/avocado/optional_plugins/%s" % (BASE_PATH, plugin)
if os.path.isdir(plugin_path):
cmd = "cd %s;python setup.py install" % plugin_path
helper.runcmd(cmd, ignore_status=True,
err_str="Error installing optional plugin: %s" % plugin,
info_str="Installing optional plugin: %s" % plugin)
else:
# plugin already installed
pass
def jobdir(self):
cmd = 'grep %s %s/*/id|grep job-' % (self.id, self.resultdir)
status, self.job_dir = helper.runcmd(cmd, ignore_status=True)
if status != 0:
return ''
self.job_dir = self.job_dir.split(':')[0]
return os.path.dirname(self.job_dir)
def bootstrap(enable_kvm=False):
"""
Prepare the environment for execution
:params enable_kvm: Flag to enable kvm environment bootstrap
"""
env_clean()
logger.info("Bootstrapping Avocado")
get_repo(AVOCADO_REPO, BASE_PATH, True)
if enable_kvm:
kvm_bootstrap()
helper.runcmd('mkdir -p %s' % TEST_DIR,
debug_str="Creating test repo dir %s" % TEST_DIR,
err_str="Failed to create test repo dir. Error: ")
for repo in TEST_REPOS:
get_repo(repo, TEST_DIR)
def is_avocado_plugin_avl(plugin):
"""
Check if the given avocado plugin installed
"""
cmd = 'avocado plugins|grep %s' % plugin
if helper.runcmd(cmd, ignore_status=True)[0] != 0:
logger.warning("Avocado %s plugin not installed", plugin)
return False
return True
def env_check(enable_kvm):
"""
Check if the environment is proper
"""
logger.info("Check for environment")
not_found = []
(env_ver, env_type, cmd_pat) = helper.get_env_type(enable_kvm)
# try to check base packages using major version numbers
env_ver = env_ver.split('.')[0]
env_deps = []
if not CONFIGFILE.has_section('deps_%s' % env_ver):
# Fallback to base name if specific version is not found
dist = helper.get_dist()
env_ver = dist[0]
if CONFIGFILE.has_section('deps_%s' % env_ver):
packages = CONFIGFILE.get('deps_%s' % env_ver, 'packages')
if packages != '':
env_deps = packages.split(',')
for dep in env_deps:
if helper.runcmd(cmd_pat % dep, ignore_status=True)[0] != 0:
not_found.append(dep)
env_deps = []
# try to check env specific packages
if CONFIGFILE.has_section('deps_%s_%s' % (env_ver, env_type)):
packages = CONFIGFILE.get('deps_%s_%s' % (env_ver, env_type),
'packages')
if packages != '':
env_deps = packages.split(',')
for dep in env_deps:
if helper.runcmd(cmd_pat % dep, ignore_status=True)[0] != 0:
not_found.append(dep)
if not_found:
if args.install_deps:
logger.warning("Installing missing packages %s",
" ".join(not_found))
if helper.install_packages(not_found):
logger.error("Some packages not installed")
sys.exit(1)
else:
logger.error("Please install following "
"dependancy packages %s", " ".join(not_found))
sys.exit(1)
def bootstrap(enable_kvm=False, guest_os=None):
"""
Prepare the environment for execution
:params enable_kvm: Flag to enable kvm environment bootstrap
"""
env_clean()
logger.info("Bootstrapping Framework")
create_config(outputdir)
if enable_kvm:
install_str = "\t2. Installing Avocado and Avocado-VT(KVM) Framework"
else:
install_str = "\t2. Installing Avocado Framework"
logger.info(install_str)
pipManager.install()
if enable_kvm:
logger.info("\t2a.Bootstrapping Avocado-VT(KVM) Framework")
# Copy if any isos present in the local folder
dst_iso_path = "%s/avocado-vt/isos/linux/" % DATA_DIR
os.makedirs(dst_iso_path, exist_ok=True)
for fle in os.listdir("%s/isos" % BASE_PATH):
if fle.endswith(".iso"):
file_path = os.path.join(BASE_PATH, 'isos', fle)
dst_file = os.path.join(dst_iso_path, fle)
shutil.copyfile(file_path, dst_file)
kvm_bootstrap(guest_os)
helper.runcmd('mkdir -p %s' % TEST_DIR,
debug_str="Creating test repo dir %s" % TEST_DIR,
err_str="Failed to create test repo dir. Error: ")
for repo in TEST_REPOS:
get_repo(repo, TEST_DIR)
if os.path.isdir(prescript):
if len(os.listdir(prescript)):
if not os.path.exists(prescript_dir):
os.makedirs(prescript_dir)
helper.copy_dir_file(prescript, prescript_dir)
if os.path.isdir(postscript):
if len(os.listdir(postscript)):
if not os.path.exists(postscript_dir):
os.makedirs(postscript_dir)
helper.copy_dir_file(postscript, postscript_dir)
def get_repo(repo, basepath):
"""
To get given repo cloned/updated and install
:param repo: tuple of repo link and branch(optional)
:param basepath: base path where the repository has to be downloaded
"""
if not repo[1]:
branch = "master"
else:
branch = repo[1]
cmd_update = "b=%s;git reset --hard && git checkout master && git remote update && (git branch|grep $b||(git checkout $b && git switch -c $b))" % branch
repo_name = repo[0].split('/')[-1].split('.git')[0]
repo_path = os.path.join(basepath, repo_name)
cmd_clone = "git clone %s %s" % (repo[0], repo_path)
if os.path.isdir(repo_path):
cmd = "cd %s && %s" % (repo_path, cmd_update)
else:
cmd = "%s && cd %s && %s" % (cmd_clone, repo_path, cmd_update)
helper.runcmd(cmd,
info_str="\t3. Cloning/Updating the repo: %s with branch %s under %s" % (repo_name, branch, repo_path),
err_str="Failed to clone/update %s repository:" % repo_name)
def kvm_bootstrap(guest_os):
"""
Prepare KVM Test environment
"""
avocado_bin = helper.get_avocado_bin()
libvirt_cmd = '%s vt-bootstrap --vt-type libvirt \
--vt-update-providers --vt-skip-verify-download-assets \
--yes-to-all' % avocado_bin
helper.runcmd(libvirt_cmd, err_str="Failed to bootstrap vt libvirt. Error:",
info_str="\t\ti) Bootstrapping VT libvirt")
qemu_cmd = '%s vt-bootstrap --vt-type qemu --vt-update-providers \
--vt-skip-verify-download-assets --yes-to-all' % avocado_bin
helper.runcmd(qemu_cmd, err_str="Failed to bootstrap vt qemu. Error:",
info_str="\t\tii) Bootstrapping VT qemu")
if guest_os:
guestos_cmd = '%s vt-bootstrap --vt-guest-os %s --yes-to-all' % (avocado_bin,
guest_os)
helper.runcmd(guestos_cmd, err_str="Failed to Download Guest OS. Error:",
info_str="\t\tiii) Downloading guest OS(%s) image" % guest_os)
def parse_test_config(test_config_file, avocado_bin, enable_kvm):
"""
Parses Test Config file and returns list of indivual tests dictionaries,
with test path and yaml file path.
"""
test_config_type = test_config_file[:test_config_file.find("_")]
test_config_name = test_config_file[test_config_file.find("_") + 1:]
test_config_file = "%s/%s/%s.cfg" % (TEST_CONF_PATH, test_config_type,
test_config_name)
if not os.path.isfile(test_config_file):
logger.error("Test Config %s not present", test_config_file)
else:
(env_ver, env_type, cmdpat) = helper.get_env_type(enable_kvm)
norun_tests = []
# Get common set of not needed tests
dist = 'norun_%s' % helper.get_dist()[0]
major = 'norun_%s' % env_ver.split('.')[0]
minor = 'norun_%s' % env_ver
minor_env = 'norun_%s_%s' % (env_ver, env_type)
for section in [dist, major, minor, minor_env]:
if NORUNTESTFILE.has_section(section):
norun_tests.extend(
NORUNTESTFILE.get(section, 'tests').split(','))
with open(test_config_file, 'r') as fp:
test_config_contents = fp.read()
test_list = []
mux_flag = 0
for line in test_config_contents.splitlines():
test_dic = {}
if line in norun_tests:
continue
if line.startswith("#") or not line:
continue
line = line.split()
test_dic['test'] = line[0].strip('$')
test_dic['name'] = test_dic['test'].split("/")[-1]
if ":" in test_dic['test'].split("/")[-1]:
test_dic['name'] = "%s_%s" % (test_dic['name'].split(
".")[0], test_dic['name'].split(":")[-1].replace(".", "_"))
test_dic['test'] = "%s$" % test_dic['test']
else:
test_dic['name'] = test_dic['name'].split(".")[0]
cmd = "%s list %s 2> /dev/null" % (avocado_bin, test_dic['test'])
if helper.runcmd(cmd, ignore_status=True)[0] != 0:
logger.debug("%s does not exist", test_dic['test'])
continue
if len(line) > 1:
test_dic['mux'] = line[1]
mux_flag = 1
test_dic['name'] = "%s_%s" % (
test_dic['name'],
test_dic['mux'].split("/")[-1].split(".")[0])
if args.inputfile:
mux_file = os.path.join(TEST_DIR, test_dic['mux'])
if not os.path.isfile(mux_file):
logger.debug("%s does not exist", mux_file)
continue
tmp_mux_path = os.path.join(
'/tmp/mux/',
"%s_%s.yaml" % (test_config_name, test_dic['name']))
edit_mux_file(test_config_name, mux_file, tmp_mux_path)
test_dic['mux'] = tmp_mux_path
count = 0
for list_dic in test_list:
if test_dic['name'] == list_dic['name'].split('.')[0]:
count += 1
if count:
test_dic['name'] += ".%d" % (count + 1)
if len(line) > 2:
test_dic['args'] = " --execution-order %s " % line[2]
test_list.append(test_dic)
if mux_flag == 0:
single_test_dic = {}
single_test_dic['name'] = test_config_name
single_test_dic['test'] = ''
for test in test_list:
single_test_dic['test'] += " %s" % test['test']
return [single_test_dic]
return test_list
def parse_test_config(test_config_file, avocado_bin, enable_kvm):
"""
Parses Test Config file and returns list of indivual tests dictionaries,
with test path and yaml file path.
"""
test_config_type = test_config_file[:test_config_file.find("_")]
test_config_name = test_config_file[test_config_file.find("_") + 1:]
test_config_file = "%s/%s/%s.cfg" % (TEST_CONF_PATH, test_config_type,
test_config_name)
if not os.path.isfile(test_config_file):
logger.error("Test Config %s not present", test_config_file)
else:
(env_ver, env_type, cmdpat) = helper.get_env_type(enable_kvm)
norun_tests = []
# Get common set of not needed tests
env = 'norun_%s' % env_type
dist = 'norun_%s' % helper.get_dist()[0]
major = 'norun_%s' % env_ver.split('.')[0]
minor = 'norun_%s' % env_ver
minor_env = 'norun_%s_%s' % (env_ver, env_type)
for section in [env, dist, major, minor, minor_env]:
if NORUNTESTFILE.has_section(section):
norun_tests.extend(NORUNTESTFILE.get(section, 'tests').split(','))
norun_tests = list(filter(None, norun_tests))
with open(test_config_file, 'r') as fp:
test_config_contents = fp.read()
test_list = []
mux_flag = 0
arg_flag = 0
for line in test_config_contents.splitlines():
norun_flag = False
test_dic = {}
# Comment line or Empty line filtering
if line.startswith("#") or not line:
norun_flag = True
# Filtering <test yaml> combination
elif line in norun_tests:
norun_flag = True
# Filtering <string*> pattern
else:
for norun_test in norun_tests:
if norun_test.endswith('*') and line.startswith(norun_test[:-1]):
norun_flag = True
break
if norun_flag:
continue
# split line ignoring quotes used for additional args
line = shlex.split(line)
test_dic['test'] = line[0].strip('$')
test_dic['name'] = test_dic['test'].split("/")[-1]
if ":" in test_dic['test'].split("/")[-1]:
test_dic['name'] = "%s_%s" % (test_dic['name'].split(".")[0],
test_dic['name'].split(":")[-1].replace(".", "_"))
test_dic['test'] = "%s$" % test_dic['test']
else:
test_dic['name'] = test_dic['name'].split(".")[0]
cmd = "%s list %s 2> /dev/null" % (avocado_bin, test_dic['test'])
if helper.runcmd(cmd, ignore_status=True)[0] != 0:
logger.debug("%s does not exist", test_dic['test'])
continue
# Handling parameters after test from cfg
if len(line) > 1:
# Handling yaml file from second param
if '.yaml' in line[1]:
test_dic['mux'] = line[1]
mux_flag = 1
test_dic['name'] = "%s_%s" % (test_dic['name'], test_dic['mux'].split("/")[-1].split(".")[0])
if args.inputfile:
mux_file = os.path.join(TEST_DIR, test_dic['mux'])
if not os.path.isfile(mux_file):
logger.debug("%s does not exist", mux_file)
continue
tmp_mux_path = os.path.join('/tmp/mux/', "%s_%s.yaml" % (test_config_name, test_dic['name']))
edit_mux_file(test_config_name, mux_file, tmp_mux_path)
test_dic['mux'] = tmp_mux_path
# Handling additional args from second param
else:
arg_flag = 1
test_dic['args'] = " %s" % line[1]
count = 0
for list_dic in test_list:
if test_dic['name'] == list_dic['name'].split('.')[0]:
count += 1
if count:
test_dic['name'] += ".%d" % (count + 1)
# Handle additional args after yaml(second arg) from third param
if len(line) > 2:
arg_flag = 1
test_dic['args'] = " %s" % line[2]
test_list.append(test_dic)
if mux_flag == 0 and arg_flag == 0:
single_test_dic = {}
single_test_dic['name'] = test_config_name
single_test_dic['test'] = ''
for test in test_list:
single_test_dic['test'] += " %s" % test['test']
return [single_test_dic]
return test_list