def copy_config(self):
cmd = 'docker exec -t %s sudo mkdir -p /etc/rally' % self.container_id
utils.run_cmd(cmd)
cmd = ("docker exec -t {} sudo "
"cp {}/rally.conf /etc/rally/rally.conf".format(
self.container_id, self.home))
utils.run_cmd(cmd)
def start_container(self):
LOG.debug("Starting Rally container")
add_host = ""
if self.access_data["auth_fqdn"] != '':
add_host = "--add-host={fqdn}:{endpoint}".format(
fqdn=self.access_data["auth_fqdn"],
endpoint=self.access_data["ips"]["endpoint"])
res = subprocess.Popen(
["docker", "run", "-d", "-P=true"] +
[add_host] * (add_host != "") +
["-p", "6000:6000",
"-e", "OS_AUTH_URL=" + self.access_data['auth_url'],
"-e", "OS_TENANT_NAME=" + self.access_data["tenant_name"],
"-e", "OS_USERNAME="******"username"],
"-e", "OS_PASSWORD="******"password"],
"-e", "KEYSTONE_ENDPOINT_TYPE=publicUrl",
"-e", "OS_REGION_NAME=" + self.access_data["region_name"],
"-v", ':'.join([self.homedir, self.home]), "-w", self.home,
"-t", "mcv-rally"],
stdout=subprocess.PIPE,
preexec_fn=utils.ignore_sigint).stdout.read()
LOG.debug('Finish starting Rally container. Result: {result}'.format(
result=str(res)))
self._verify_rally_container_is_up()
cmd = "chmod a+r %s/images/cirros-0.3.1-x86_64-disk.img" % self.homedir
utils.run_cmd(cmd)
self._patch_rally()
self.copy_config()
def clear_shaker(self):
cleanup = utils.GET(self.config, 'cleanup', 'shaker') or 'True'
if cleanup == 'True':
LOG.info("Removing shaker's image and flavor")
cmd = "docker exec -t %s shaker-cleanup --image-name %s " \
"--flavor-name %s" % (self.container_id, self.image_name,
self.flavor_name)
utils.run_cmd(cmd)
def clear_shaker(self):
if CONF.shaker.cleanup:
LOG.info("Removing shaker's image and flavor")
cmd = "docker exec -t %s shaker-cleanup --image-name %s " "--flavor-name %s" % (
self.container_id,
self.image_name,
self.flavor_name,
)
utils.run_cmd(cmd)
def fix_shaker(file_location):
LOG.debug('Fixing Shaker report')
if not os.path.isfile(file_location):
return LOG.debug('File not found %s' % file_location)
cmd = ("sed -i '/<div\ class=\"container\"\ id=\"container\">/"
" a\ <li class=\"active\" style=\"list-style-type: none;\"><a "
"href=\"../index.html\">Back to Index</a></li>' "
"%s" % file_location)
utils.run_cmd(cmd)
def _run_ostf(self, task):
# The task can be either a test or suite
if ':' in task:
_cmd = 'run_test'
_arg = '--test'
else:
_cmd = 'run_suite'
_arg = '--suite'
cmd = ('{home}/execute.sh '
'fuel-ostf.{mos_version} '
'"cloudvalidation-cli '
'--output-file={home}/ostf_report.json '
'--config-file={home}/conf/{conf_fname} '
'cloud-health-check {cmd} '
'--validation-plugin-name fuel_health {arg} {task}"'
).format(home=self.homedir,
mos_version=self.mos_version,
conf_fname=self.config_filename,
cmd=_cmd,
arg=_arg,
task=task)
utils.run_cmd(cmd)
try:
results = []
try:
fpath = os.path.join(self.homedir, 'ostf_report.json')
with open(fpath) as fp:
results = json.load(fp)
os.remove(fpath)
except OSError as e:
LOG.error('Error while removing report file: %s', str(e))
except ValueError as e:
LOG.error('Error while parsing report file: %s', str(e))
def fix_suite(result):
result['suite'] = result['suite'].split(':')[1]
return result
map(fix_suite, results)
# store raw results
self.dump_raw_results(task, results)
self.save_report(results)
except subprocess.CalledProcessError as e:
LOG.error("Task %s has failed with: %s", task, e)
self.failures.append(task)
self.time_of_tests[task] = {'duration': '0s'}
return
def make_detailed_report(self, task):
LOG.debug('Generating detailed report')
details_dir = os.path.join(self.home, 'reports/details/')
details_file = os.path.join(details_dir, task + '.txt')
cmd = "docker exec -t %(cid)s " \
"rally deployment list | grep existing | awk \'{print $2}\'" \
% dict(cid=self.container_id)
deployment_id = utils.run_cmd(cmd, quiet=True).strip()
cmd = 'docker exec -t {cid} mkdir -p {out_dir}'
utils.run_cmd(cmd.format(cid=self.container_id, out_dir=details_dir),
quiet=True)
# store tempest.conf
self.store_config(os.path.join(self.homedir,
"for-deployment-{ID}/tempest.conf"
.format(ID=deployment_id)))
self.store_config(os.path.join(self.homedir, "conf/existing.json"))
# Note(ogrytsenko): tool subunit2pyunit returns exit code '1' if
# at leas one test failed in a test suite. It also returns exit
# code '1' if some error occurred during processing a file, like:
# "Permission denied".
# We force 'exit 0' here and will check the real status lately
# by calling 'test -e <details_file>'
cmd = 'docker exec -t {cid} /bin/sh -c \" ' \
'subunit2pyunit /mcv/for-deployment-{ID}/subunit.stream ' \
'2> {out_file}\"; ' \
'exit 0'.format(cid=self.container_id,
ID=deployment_id,
out_file=details_file)
out = utils.run_cmd(cmd, quiet=True)
cmd = 'docker exec -t {cid} test -e {out_file} ' \
'&& echo -n yes || echo -n no'.format(cid=self.container_id,
out_file=details_file)
exists = utils.run_cmd(cmd)
if exists == 'no':
LOG.debug('ERROR: Failed to create detailed report for '
'{task} set. Output: {out}'.format(task=task, out=out))
return
cmd = 'mkdir -p {path}/details'.format(path=self.path)
utils.run_cmd(cmd, quiet=True)
reports_dir = os.path.join(self.homedir, 'reports')
cmd = 'cp {reports}/details/{task}.txt {path}/details'
utils.run_cmd(
cmd.format(reports=reports_dir, task=task, path=self.path),
quiet=True
)
LOG.debug(
"Finished creating detailed report for '{task}'. "
"File: {details_file}".format(task=task, details_file=details_file)
)
def _patch_rally(self):
dist = '/tempest/requirements.txt'
LOG.debug('Patching tempest requirements')
tempest_patch = '/mcv/custom_patches/requirements.patch'
self._os_patch(dist, tempest_patch, self.container_id)
git_commit_cmd = (
'cd /tempest && git config --global user.name \"mcv-team\" && '
'git config --global user.email '
'\"[email protected]\" && '
'sudo git add . && sudo git commit -m \"added markupsafe to '
'requirements, which is needed for pbr\"')
utils.run_cmd('docker exec -t {cid} sh -c "{cmd}"'.format(
cid=self.container_id,
cmd=git_commit_cmd))
def _patch_rally(self):
from os.path import join
dist = '/usr/local/lib/python2.7/dist-packages/'
LOG.debug('Patching rally.siege regex')
siege = join(dist, 'rally/plugins/workload/siege.py')
siege_patch = '/mcv/custom_patches/rally_siege_regex.patch'
self._os_patch(siege, siege_patch, self.container_id)
LOG.debug('Patching sahara_job_binaries.py for rally')
sahara_job = join(
dist,
'rally/plugins/openstack/context/sahara/sahara_job_binaries.py'
)
sahara_patch = '/mcv/custom_patches/rally_sahara_job_binaries.patch'
self._os_patch(sahara_job, sahara_patch, self.container_id)
LOG.debug('Start patching hosts')
template_path = join(self.home, 'tests/templates/wp_instances.yaml')
cmd = ("""docker exec -t %s """
"""sed -i "61s/.*/"""
""" sudo sh -c 'echo %s %s >> """
"""\/etc\/hosts'/" %s""") % (
self.container_id,
self.access_data['ips']['endpoint'],
self.access_data['auth_fqdn'],
template_path)
res = utils.run_cmd(cmd)
LOG.debug('Finish patching hosts. Result: {res}'.format(res=res))
def verify_container_is_up(self, plugin=None, attempts=3, interval=10,
quiet=False):
plugin = plugin or self.identity
LOG.debug("Checking %s container...", plugin)
docker_image = 'mcv-{}'.format(plugin)
cmd = 'docker ps ' \
'--filter "ancestor=%(image)s" ' \
'--format "{{.ID}}\t{{.Status}}" ' \
'| tail -1' % dict(image=docker_image)
for _ in range(attempts):
out = utils.run_cmd(cmd)
if not out:
LOG.debug('Container is not running')
interval and time.sleep(interval)
continue
cid, status = out.strip().split('\t')
break
else:
if quiet:
return
err = 'Failed to start docker container: {}'.format(plugin)
raise ProgramError(err)
self.container_id = cid
LOG.debug("Container %s is fine. Status: %s", plugin, status)
return cid
def get_ostf(cid, mos_version):
"""Discover OSTF tests.
:param cid: docker container id
:param mos_version: 'mos_version' from config file
Ostf is a special case because we may run either 'tests' or a 'suites'
and they can be mixed together.
Thus resulted list of all discovered tests will contain tests and suites
as well.
"""
py_exe = '/home/mcv/venv/fuel-ostf.{v}/bin/python'.format(v=mos_version)
cmd = '2>/dev/null {python} -c "{code}" '
if cid is not None:
cmd = 'docker exec {cid} ' + cmd
cmd = cmd.format(cid=cid, python=py_exe, code=ostf_py)
out = utils.run_cmd(cmd, quiet=True)
all_tests = json.loads(out)
tests = set()
suites = set()
for test_path in all_tests:
test_case = test_path.split('.')[-1]
tests.add(test_case)
if ':' in test_case:
suites.add(test_case.split(':')[0])
res = set(tests) | set(suites)
return list(res)
def _run_rally(self, task):
LOG.debug("Running task %s" % task)
# warning: at this point task must be transformed to a full path
path_to_task = self._get_task_path(task)
p = utils.run_cmd("rally task start " + path_to_task)
out = p.split('\n')[-4].lstrip('\t')
return out
def _do_config_extraction(self):
path = os.path.join(self.homedir, 'conf', self.config_filename)
if not CONF.ostf.reload_config:
LOG.debug("Checking for existing OSTF configuration file...")
if os.path.isfile(path):
LOG.debug("File '%s' exists. Skip extraction.",
self.config_filename)
return
else:
LOG.debug("File '%s' does not exist", self.config_filename)
LOG.debug("Trying to obtain OSTF configuration file")
cmd = ' '.join([os.path.join(self.homedir, 'execute.sh'),
'fuel-ostf.{}'.format(self.mos_version),
'"ostf-config-extractor -o {}"'.format(path)])
utils.run_cmd(cmd)
def _run_tempest_on_docker(self, task, *args, **kwargs):
LOG.debug("Starting verification")
run_by_name = kwargs.get('run_by_name')
if run_by_name:
cmd = ("docker exec -t {cid} rally "
"--log-file {home}/log/tempest.log --rally-debug"
" verify start --system-wide "
"--regex {_set}").format(cid=self.container_id,
home=self.home,
_set=task)
else:
cmd = ("docker exec -t {cid} rally "
"--log-file {home}/log/tempest.log --rally-debug"
" verify start --system-wide "
"--set {_set}").format(cid=self.container_id,
home=self.home,
_set=task)
utils.run_cmd(cmd, quiet=True)
cmd = "docker exec -t {cid} rally verify list".format(
cid=self.container_id)
# TODO(ogrytsenko): double-check this approach
try:
p = utils.run_cmd(cmd)
except subprocess.CalledProcessError as e:
LOG.error("Task %s failed with: %s" % (task, e))
return ''
run = p.split('\n')[-3].split('|')[8]
if run == 'failed':
LOG.error('Verification failed, unable to generate report')
return ''
LOG.debug('Generating html report')
cmd = ("docker exec -t {cid} rally verify results --html "
"--out={home}/reports/{task}.html").format(
cid=self.container_id, home=self.home, task=task)
utils.run_cmd(cmd, quiet=True)
reports_dir = os.path.join(self.homedir, 'reports')
cmd = "cp {reports}/{task}.html {path} ".format(
reports=reports_dir, task=task, path=self.path)
utils.run_cmd(cmd, quiet=True)
try:
self.make_detailed_report(task)
except Exception:
LOG.debug('ERROR: \n' + traceback.format_exc())
cmd = "docker exec -t {cid} /bin/sh -c " \
"\"rally verify results --json 2>/dev/null\" "\
.format(cid=self.container_id)
return utils.run_cmd(cmd, quiet=True)
def make_results_archive(self):
LOG.debug('Creating a .tar.gz archive with test reports')
try:
archive_file = '%s.tar.gz' % self.results_dir
cmd = "tar -zcf {arch_file} -C {results_dir} .".format(
arch_file=archive_file, results_dir=self.results_dir)
utils.run_cmd(cmd)
cmd = "rm -rf %s" % self.results_dir
utils.run_cmd(cmd)
except subprocess.CalledProcessError:
LOG.warning('Creation of .tar.gz archive has failed. See log '
'for details. You can still get your files from: '
'%s', self.results_dir, exc_info=1)
return
LOG.debug("Finished creating a report.")
LOG.info("One page report could be found in %s\n", archive_file)
def get_tempest(cid):
if cid is None:
# This is a special case which is currently used only for 'full' group
return TEMPEST_DUMMY
cmd = 'docker exec {cid} 2>/dev/null python -c "{code}" '\
.format(cid=cid, code=tempest_py)
out = utils.run_cmd(cmd, quiet=True)
res = json.loads(out)
return res
def _get_task_result_from_docker(self):
cmd = 'docker exec -t {cid} /bin/sh -c ' \
'"rally task results 2>/dev/null"'.format(cid=self.container_id)
p = utils.run_cmd(cmd, quiet=True)
try:
return json.loads(p)[0] # actual test result as a dictionary
except ValueError:
LOG.error("Gotten not-JSON object. Please see mcv-log")
LOG.debug("Not-JSON object: %s, After command: %s", p, cmd)
def _run_shaker(self, task):
LOG.debug("Running task %s" % task)
# warning: at this point task must be transformed to a full path
path_to_task = self._get_task_path(task)
p = utils.run_cmd("rally task start " + path_to_task)
# here out is in fact a command which can be run to obtain task results
# thus it is returned directly.
out = p.split('\n')[-4].lstrip('\t')
return out
def _do_config_extraction(self):
LOG.debug("Checking for existing OSTF configuration file...")
# NOTE(albartash): in case of multiple clouds, we probably would have
# 5% of possibility that config for one of clouds won't be created, so
# Consoler will try to run OSTF for one cloud using config from
# another one. Just notice it.
path = os.path.join(self.home, 'conf', self.config_filename)
if os.path.isfile(path):
LOG.debug("File '%s' exists. Skip extraction." %
self.config_filename)
return
LOG.debug("File '%s' does not exist." % self.config_filename)
LOG.debug("Trying to obtain OSTF configuration file")
cmd = ('docker exec -t {cid} /mcv/execute.sh fuel-ostf.{version} '
'"ostf-config-extractor -o {path}"').format(
cid=self.container_id,
version=self.mos_version,
path=path)
utils.run_cmd(cmd)
def copy_tempest_image(self):
LOG.info('Copying image files required by tempest')
# here we fix glance image issues
subprocess.Popen(["sudo", "chmod", "a+r",
os.path.join(self.home,
"images",
"cirros-0.3.4-x86_64-disk.img")],
stdout=subprocess.PIPE,
preexec_fn=utils.ignore_sigint).stdout.read()
cmd = "mkdir " + os.path.join(self.homedir, "data")
utils.run_cmd(cmd)
cmd = ("sudo ln -s hotexamples_com/images/cirros-0.3.4-x86_64-disk.img "
"hotexamples_com/data/").format(homedir=self.homedir)
cmd = shlex.split(cmd)
subprocess.Popen(cmd, stdout=subprocess.PIPE,
preexec_fn=utils.ignore_sigint).stdout.read()
def _setup_shaker_on_docker(self):
self.verify_container_is_up("shaker")
self._check_shaker_setup()
p = utils.run_cmd("docker ps")
p = p.split('\n')
for line in p:
elements = line.split()
if elements[1].find("shaker") != -1:
self.container = elements[0]
status = elements[4]
LOG.debug('Container status: %s' % str(status))
break
def _do_finalization(self, run_results):
if run_results is None:
LOG.warning("For some reason test tools have returned nothing")
return
self.describe_results(run_results)
self.update_config(run_results)
try:
reporter.brew_a_report(run_results,
self.results_vault + "/index.html")
except Exception as e:
LOG.warning("Brewing a report has failed with error: %s" % str(e))
LOG.debug(traceback.format_exc())
return
result_dict = {
"timestamp": self.timestamp_str,
"location": self.results_vault
}
LOG.debug('Creating a .tar.gz archive with test reports')
try:
cmd = ("tar -zcf /tmp/mcv_run_%(timestamp)s.tar.gz"
" -C %(location)s .") % result_dict
utils.run_cmd(cmd)
cmd = "rm -rf %(location)s" % {"location": self.results_vault}
utils.run_cmd(cmd)
except subprocess.CalledProcessError:
LOG.warning('Creation of .tar.gz archive has failed. See log '
'for details. You can still get your files from: '
'%s' % self.results_vault)
LOG.debug(traceback.format_exc())
return
LOG.debug("Finished creating a report.")
LOG.info("One page report could be found in "
"/tmp/mcv_run_%(timestamp)s.tar.gz\n" % result_dict)
return result_dict
def _setup_ostf_on_docker(self):
# Find docker container:
self._verify_ostf_container_is_up()
self._do_config_extraction()
p = utils.run_cmd("docker ps")
p = p.split('\n')
for line in p:
elements = line.split()
if elements[1].find("ostf") != -1:
self.container = elements[0]
status = elements[4]
LOG.debug("OSTF container status: " + status)
break
def start_container(self):
LOG.debug("Bringing up Tempest container with credentials")
add_host = ""
# TODO(albartash): Refactor this place!
if self.access_data["auth_fqdn"] != '':
add_host = "--add-host={fqdn}:{endpoint}".format(
fqdn=self.access_data["auth_fqdn"],
endpoint=self.access_data["public_endpoint_ip"])
res = subprocess.Popen(
["docker", "run", "-d", "-P=true"] +
[add_host] * (add_host != "") +
["-p", "6001:6001",
"-e", "OS_AUTH_URL=" + self.access_data["auth_url"],
"-e", "OS_TENANT_NAME=" + self.access_data["tenant_name"],
"-e", "OS_REGION_NAME" + self.access_data["region_name"],
"-e", "OS_USERNAME="******"username"],
"-e", "OS_PASSWORD="******"password"],
"-e", "KEYSTONE_ENDPOINT_TYPE=publicUrl",
"-v", '%s:/home/rally/.rally/tempest' % self.homedir,
"-v", "%s:%s" % (self.homedir, self.home), "-w", self.home,
"-t", "mcv-tempest"],
stdout=subprocess.PIPE,
preexec_fn=utils.ignore_sigint).stdout.read()
LOG.debug('Finish bringing up Tempest container.'
'ID = %s' % str(res))
self.verify_container_is_up()
self._patch_rally()
# Hotfix. set rally's permission for .rally/ folder
# Please remove this. Use: `sudo -u rally docker run` when
# rally user gets its permissions to start docker containers
cmd = 'docker exec -t {cid} sudo chown rally:rally /home/rally/.rally'
utils.run_cmd(cmd.format(cid=self.container_id))
self.copy_config()
self.install_tempest()
def _rally_deployment_check(self):
LOG.debug("Checking if Rally deployment is present.")
res = subprocess.Popen(["docker", "exec", "-t",
self.container_id,
"rally", "deployment", "check"],
stdout=subprocess.PIPE,
preexec_fn=utils.ignore_sigint).stdout.read()
if "There is no" in res:
LOG.debug("It is not. Trying to set up rally deployment.")
self.create_rally_json()
cmd = "docker exec -t %s rally deployment create " \
"--file=%s/conf/existing.json --name=existing" % \
(self.container_id, self.home)
utils.run_cmd(cmd)
else:
LOG.debug("Seems like it is present.")
LOG.debug('Trying to use Rally deployment')
cmd = ("docker exec -t {cid} "
"rally deployment use existing").format(
cid=self.container_id)
utils.run_cmd(cmd, quiet=True)
def install_tempest(self):
LOG.debug("Searching for installed tempest")
super(TempestOnDockerRunner, self)._rally_deployment_check()
LOG.debug("Generating additional config")
path_to_conf = os.path.join(self.homedir, 'additional.conf')
with open(path_to_conf, 'wb') as conf_file:
config = ConfigParser.ConfigParser()
config._sections = tempest_additional_conf
config.write(conf_file)
LOG.debug("Installing tempest...")
cmd = ("docker exec -t {cid} "
"rally verify install --system-wide "
"--deployment existing --source /tempest").format(
cid=self.container_id)
utils.run_cmd(cmd, quiet=True)
cmd = "docker exec -t %(container)s rally verify genconfig " \
"--add-options %(conf_path)s" % \
{"container": self.container_id,
"conf_path": os.path.join(self.home, 'additional.conf')}
utils.run_cmd(cmd, quiet=True)
def _check_shaker_setup(self):
LOG.info("Start shaker-image-builder. Creating infrastructure. "
"Please wait...")
cmd = "docker exec -t %s shaker-image-builder --image-name %s " \
"--flavor-name %s" % (self.container_id, self.image_name,
self.flavor_name)
p = utils.run_cmd(cmd)
if 'ERROR' in p:
LOG.debug("shaker-image-builder failed")
for stack in self.heat.stacks.list():
if 'shaker' in stack.stack_name:
stack.delete()
raise RuntimeError
LOG.debug('Finish running shaker-image-builder.')
def _get_task_result(self, task_id):
# TODO(albartash): Fix the issue mentioned below:
# this function is not using task id contrary to what it says, but in
# current state of affair direct command produced by rally. task_id
# is left as is for now, but will be moved in future.
# if asked kindly rally just spits resulting json directly to stdout
p = utils.run_cmd(task_id)
try:
res = json.loads(p)[0]
return res
except ValueError:
LOG.error("Gotten not-JSON object. Please see mcv-log")
LOG.debug("Not-JSON object: %s", p)
return "Not-JSON object"
def _get_task_result_from_docker(self, task_id):
LOG.info("Retrieving task results for %s" % task_id)
cmd = "docker exec -t %s %s" % (self.container, task_id)
p = utils.run_cmd(cmd)
if task_id.find("detailed") == -1:
try:
res = json.loads(p)[0]
return res
except ValueError:
LOG.error("Gotten not-JSON object. Please see mcv-log")
LOG.debug("Not-JSON object: %s, After command: %s", p, cmd)
return "Not-JSON object"
else:
return p.split('\n')[-4:-1]
def _os_patch(target, patch, container_id=None):
"""Silently patch a file. Errors are ignored
params:
'target' - absolute system path to a file that needs to be changed
'patch' - absolute system path to a .patch file
'container_id' (optional). If provided - perform an operation
inside a docker container
"""
tmp = 'sudo patch --dry-run --forward {target} -i {patch} && ' \
'sudo patch {target} -i {patch}'
if container_id:
tmp = 'docker exec -t {cid} /bin/sh -c \"' + tmp + '\"'
cmd = tmp.format(cid=container_id, target=target, patch=patch)
try:
return utils.run_cmd(cmd)
except subprocess.CalledProcessError:
pass
