def copy_config(self): cmd = 'docker exec -t %s sudo mkdir -p /etc/rally' % self.container_id utils.run_cmd(cmd) cmd = ("docker exec -t {} sudo " "cp {}/rally.conf /etc/rally/rally.conf".format( self.container_id, self.home)) utils.run_cmd(cmd)
def start_container(self): LOG.debug("Starting Rally container") add_host = "" if self.access_data["auth_fqdn"] != '': add_host = "--add-host={fqdn}:{endpoint}".format( fqdn=self.access_data["auth_fqdn"], endpoint=self.access_data["ips"]["endpoint"]) res = subprocess.Popen( ["docker", "run", "-d", "-P=true"] + [add_host] * (add_host != "") + ["-p", "6000:6000", "-e", "OS_AUTH_URL=" + self.access_data['auth_url'], "-e", "OS_TENANT_NAME=" + self.access_data["tenant_name"], "-e", "OS_USERNAME="******"username"], "-e", "OS_PASSWORD="******"password"], "-e", "KEYSTONE_ENDPOINT_TYPE=publicUrl", "-e", "OS_REGION_NAME=" + self.access_data["region_name"], "-v", ':'.join([self.homedir, self.home]), "-w", self.home, "-t", "mcv-rally"], stdout=subprocess.PIPE, preexec_fn=utils.ignore_sigint).stdout.read() LOG.debug('Finish starting Rally container. Result: {result}'.format( result=str(res))) self._verify_rally_container_is_up() cmd = "chmod a+r %s/images/cirros-0.3.1-x86_64-disk.img" % self.homedir utils.run_cmd(cmd) self._patch_rally() self.copy_config()
def clear_shaker(self): cleanup = utils.GET(self.config, 'cleanup', 'shaker') or 'True' if cleanup == 'True': LOG.info("Removing shaker's image and flavor") cmd = "docker exec -t %s shaker-cleanup --image-name %s " \ "--flavor-name %s" % (self.container_id, self.image_name, self.flavor_name) utils.run_cmd(cmd)
def clear_shaker(self): if CONF.shaker.cleanup: LOG.info("Removing shaker's image and flavor") cmd = "docker exec -t %s shaker-cleanup --image-name %s " "--flavor-name %s" % ( self.container_id, self.image_name, self.flavor_name, ) utils.run_cmd(cmd)
def fix_shaker(file_location): LOG.debug('Fixing Shaker report') if not os.path.isfile(file_location): return LOG.debug('File not found %s' % file_location) cmd = ("sed -i '/<div\ class=\"container\"\ id=\"container\">/" " a\ <li class=\"active\" style=\"list-style-type: none;\"><a " "href=\"../index.html\">Back to Index</a></li>' " "%s" % file_location) utils.run_cmd(cmd)
def _run_ostf(self, task): # The task can be either a test or suite if ':' in task: _cmd = 'run_test' _arg = '--test' else: _cmd = 'run_suite' _arg = '--suite' cmd = ('{home}/execute.sh ' 'fuel-ostf.{mos_version} ' '"cloudvalidation-cli ' '--output-file={home}/ostf_report.json ' '--config-file={home}/conf/{conf_fname} ' 'cloud-health-check {cmd} ' '--validation-plugin-name fuel_health {arg} {task}"' ).format(home=self.homedir, mos_version=self.mos_version, conf_fname=self.config_filename, cmd=_cmd, arg=_arg, task=task) utils.run_cmd(cmd) try: results = [] try: fpath = os.path.join(self.homedir, 'ostf_report.json') with open(fpath) as fp: results = json.load(fp) os.remove(fpath) except OSError as e: LOG.error('Error while removing report file: %s', str(e)) except ValueError as e: LOG.error('Error while parsing report file: %s', str(e)) def fix_suite(result): result['suite'] = result['suite'].split(':')[1] return result map(fix_suite, results) # store raw results self.dump_raw_results(task, results) self.save_report(results) except subprocess.CalledProcessError as e: LOG.error("Task %s has failed with: %s", task, e) self.failures.append(task) self.time_of_tests[task] = {'duration': '0s'} return
def make_detailed_report(self, task): LOG.debug('Generating detailed report') details_dir = os.path.join(self.home, 'reports/details/') details_file = os.path.join(details_dir, task + '.txt') cmd = "docker exec -t %(cid)s " \ "rally deployment list | grep existing | awk \'{print $2}\'" \ % dict(cid=self.container_id) deployment_id = utils.run_cmd(cmd, quiet=True).strip() cmd = 'docker exec -t {cid} mkdir -p {out_dir}' utils.run_cmd(cmd.format(cid=self.container_id, out_dir=details_dir), quiet=True) # store tempest.conf self.store_config(os.path.join(self.homedir, "for-deployment-{ID}/tempest.conf" .format(ID=deployment_id))) self.store_config(os.path.join(self.homedir, "conf/existing.json")) # Note(ogrytsenko): tool subunit2pyunit returns exit code '1' if # at leas one test failed in a test suite. It also returns exit # code '1' if some error occurred during processing a file, like: # "Permission denied". # We force 'exit 0' here and will check the real status lately # by calling 'test -e <details_file>' cmd = 'docker exec -t {cid} /bin/sh -c \" ' \ 'subunit2pyunit /mcv/for-deployment-{ID}/subunit.stream ' \ '2> {out_file}\"; ' \ 'exit 0'.format(cid=self.container_id, ID=deployment_id, out_file=details_file) out = utils.run_cmd(cmd, quiet=True) cmd = 'docker exec -t {cid} test -e {out_file} ' \ '&& echo -n yes || echo -n no'.format(cid=self.container_id, out_file=details_file) exists = utils.run_cmd(cmd) if exists == 'no': LOG.debug('ERROR: Failed to create detailed report for ' '{task} set. Output: {out}'.format(task=task, out=out)) return cmd = 'mkdir -p {path}/details'.format(path=self.path) utils.run_cmd(cmd, quiet=True) reports_dir = os.path.join(self.homedir, 'reports') cmd = 'cp {reports}/details/{task}.txt {path}/details' utils.run_cmd( cmd.format(reports=reports_dir, task=task, path=self.path), quiet=True ) LOG.debug( "Finished creating detailed report for '{task}'. " "File: {details_file}".format(task=task, details_file=details_file) )
def _patch_rally(self): dist = '/tempest/requirements.txt' LOG.debug('Patching tempest requirements') tempest_patch = '/mcv/custom_patches/requirements.patch' self._os_patch(dist, tempest_patch, self.container_id) git_commit_cmd = ( 'cd /tempest && git config --global user.name \"mcv-team\" && ' 'git config --global user.email ' '\"[email protected]\" && ' 'sudo git add . && sudo git commit -m \"added markupsafe to ' 'requirements, which is needed for pbr\"') utils.run_cmd('docker exec -t {cid} sh -c "{cmd}"'.format( cid=self.container_id, cmd=git_commit_cmd))
def _patch_rally(self): from os.path import join dist = '/usr/local/lib/python2.7/dist-packages/' LOG.debug('Patching rally.siege regex') siege = join(dist, 'rally/plugins/workload/siege.py') siege_patch = '/mcv/custom_patches/rally_siege_regex.patch' self._os_patch(siege, siege_patch, self.container_id) LOG.debug('Patching sahara_job_binaries.py for rally') sahara_job = join( dist, 'rally/plugins/openstack/context/sahara/sahara_job_binaries.py' ) sahara_patch = '/mcv/custom_patches/rally_sahara_job_binaries.patch' self._os_patch(sahara_job, sahara_patch, self.container_id) LOG.debug('Start patching hosts') template_path = join(self.home, 'tests/templates/wp_instances.yaml') cmd = ("""docker exec -t %s """ """sed -i "61s/.*/""" """ sudo sh -c 'echo %s %s >> """ """\/etc\/hosts'/" %s""") % ( self.container_id, self.access_data['ips']['endpoint'], self.access_data['auth_fqdn'], template_path) res = utils.run_cmd(cmd) LOG.debug('Finish patching hosts. Result: {res}'.format(res=res))
def verify_container_is_up(self, plugin=None, attempts=3, interval=10, quiet=False): plugin = plugin or self.identity LOG.debug("Checking %s container...", plugin) docker_image = 'mcv-{}'.format(plugin) cmd = 'docker ps ' \ '--filter "ancestor=%(image)s" ' \ '--format "{{.ID}}\t{{.Status}}" ' \ '| tail -1' % dict(image=docker_image) for _ in range(attempts): out = utils.run_cmd(cmd) if not out: LOG.debug('Container is not running') interval and time.sleep(interval) continue cid, status = out.strip().split('\t') break else: if quiet: return err = 'Failed to start docker container: {}'.format(plugin) raise ProgramError(err) self.container_id = cid LOG.debug("Container %s is fine. Status: %s", plugin, status) return cid
def get_ostf(cid, mos_version): """Discover OSTF tests. :param cid: docker container id :param mos_version: 'mos_version' from config file Ostf is a special case because we may run either 'tests' or a 'suites' and they can be mixed together. Thus resulted list of all discovered tests will contain tests and suites as well. """ py_exe = '/home/mcv/venv/fuel-ostf.{v}/bin/python'.format(v=mos_version) cmd = '2>/dev/null {python} -c "{code}" ' if cid is not None: cmd = 'docker exec {cid} ' + cmd cmd = cmd.format(cid=cid, python=py_exe, code=ostf_py) out = utils.run_cmd(cmd, quiet=True) all_tests = json.loads(out) tests = set() suites = set() for test_path in all_tests: test_case = test_path.split('.')[-1] tests.add(test_case) if ':' in test_case: suites.add(test_case.split(':')[0]) res = set(tests) | set(suites) return list(res)
def _run_rally(self, task): LOG.debug("Running task %s" % task) # warning: at this point task must be transformed to a full path path_to_task = self._get_task_path(task) p = utils.run_cmd("rally task start " + path_to_task) out = p.split('\n')[-4].lstrip('\t') return out
def _do_config_extraction(self): path = os.path.join(self.homedir, 'conf', self.config_filename) if not CONF.ostf.reload_config: LOG.debug("Checking for existing OSTF configuration file...") if os.path.isfile(path): LOG.debug("File '%s' exists. Skip extraction.", self.config_filename) return else: LOG.debug("File '%s' does not exist", self.config_filename) LOG.debug("Trying to obtain OSTF configuration file") cmd = ' '.join([os.path.join(self.homedir, 'execute.sh'), 'fuel-ostf.{}'.format(self.mos_version), '"ostf-config-extractor -o {}"'.format(path)]) utils.run_cmd(cmd)
def _run_tempest_on_docker(self, task, *args, **kwargs): LOG.debug("Starting verification") run_by_name = kwargs.get('run_by_name') if run_by_name: cmd = ("docker exec -t {cid} rally " "--log-file {home}/log/tempest.log --rally-debug" " verify start --system-wide " "--regex {_set}").format(cid=self.container_id, home=self.home, _set=task) else: cmd = ("docker exec -t {cid} rally " "--log-file {home}/log/tempest.log --rally-debug" " verify start --system-wide " "--set {_set}").format(cid=self.container_id, home=self.home, _set=task) utils.run_cmd(cmd, quiet=True) cmd = "docker exec -t {cid} rally verify list".format( cid=self.container_id) # TODO(ogrytsenko): double-check this approach try: p = utils.run_cmd(cmd) except subprocess.CalledProcessError as e: LOG.error("Task %s failed with: %s" % (task, e)) return '' run = p.split('\n')[-3].split('|')[8] if run == 'failed': LOG.error('Verification failed, unable to generate report') return '' LOG.debug('Generating html report') cmd = ("docker exec -t {cid} rally verify results --html " "--out={home}/reports/{task}.html").format( cid=self.container_id, home=self.home, task=task) utils.run_cmd(cmd, quiet=True) reports_dir = os.path.join(self.homedir, 'reports') cmd = "cp {reports}/{task}.html {path} ".format( reports=reports_dir, task=task, path=self.path) utils.run_cmd(cmd, quiet=True) try: self.make_detailed_report(task) except Exception: LOG.debug('ERROR: \n' + traceback.format_exc()) cmd = "docker exec -t {cid} /bin/sh -c " \ "\"rally verify results --json 2>/dev/null\" "\ .format(cid=self.container_id) return utils.run_cmd(cmd, quiet=True)
def make_results_archive(self): LOG.debug('Creating a .tar.gz archive with test reports') try: archive_file = '%s.tar.gz' % self.results_dir cmd = "tar -zcf {arch_file} -C {results_dir} .".format( arch_file=archive_file, results_dir=self.results_dir) utils.run_cmd(cmd) cmd = "rm -rf %s" % self.results_dir utils.run_cmd(cmd) except subprocess.CalledProcessError: LOG.warning('Creation of .tar.gz archive has failed. See log ' 'for details. You can still get your files from: ' '%s', self.results_dir, exc_info=1) return LOG.debug("Finished creating a report.") LOG.info("One page report could be found in %s\n", archive_file)
def get_tempest(cid): if cid is None: # This is a special case which is currently used only for 'full' group return TEMPEST_DUMMY cmd = 'docker exec {cid} 2>/dev/null python -c "{code}" '\ .format(cid=cid, code=tempest_py) out = utils.run_cmd(cmd, quiet=True) res = json.loads(out) return res
def _get_task_result_from_docker(self): cmd = 'docker exec -t {cid} /bin/sh -c ' \ '"rally task results 2>/dev/null"'.format(cid=self.container_id) p = utils.run_cmd(cmd, quiet=True) try: return json.loads(p)[0] # actual test result as a dictionary except ValueError: LOG.error("Gotten not-JSON object. Please see mcv-log") LOG.debug("Not-JSON object: %s, After command: %s", p, cmd)
def _run_shaker(self, task): LOG.debug("Running task %s" % task) # warning: at this point task must be transformed to a full path path_to_task = self._get_task_path(task) p = utils.run_cmd("rally task start " + path_to_task) # here out is in fact a command which can be run to obtain task results # thus it is returned directly. out = p.split('\n')[-4].lstrip('\t') return out
def _do_config_extraction(self): LOG.debug("Checking for existing OSTF configuration file...") # NOTE(albartash): in case of multiple clouds, we probably would have # 5% of possibility that config for one of clouds won't be created, so # Consoler will try to run OSTF for one cloud using config from # another one. Just notice it. path = os.path.join(self.home, 'conf', self.config_filename) if os.path.isfile(path): LOG.debug("File '%s' exists. Skip extraction." % self.config_filename) return LOG.debug("File '%s' does not exist." % self.config_filename) LOG.debug("Trying to obtain OSTF configuration file") cmd = ('docker exec -t {cid} /mcv/execute.sh fuel-ostf.{version} ' '"ostf-config-extractor -o {path}"').format( cid=self.container_id, version=self.mos_version, path=path) utils.run_cmd(cmd)
def copy_tempest_image(self): LOG.info('Copying image files required by tempest') # here we fix glance image issues subprocess.Popen(["sudo", "chmod", "a+r", os.path.join(self.home, "images", "cirros-0.3.4-x86_64-disk.img")], stdout=subprocess.PIPE, preexec_fn=utils.ignore_sigint).stdout.read() cmd = "mkdir " + os.path.join(self.homedir, "data") utils.run_cmd(cmd) cmd = ("sudo ln -s hotexamples_com/images/cirros-0.3.4-x86_64-disk.img " "hotexamples_com/data/").format(homedir=self.homedir) cmd = shlex.split(cmd) subprocess.Popen(cmd, stdout=subprocess.PIPE, preexec_fn=utils.ignore_sigint).stdout.read()
def _setup_shaker_on_docker(self): self.verify_container_is_up("shaker") self._check_shaker_setup() p = utils.run_cmd("docker ps") p = p.split('\n') for line in p: elements = line.split() if elements[1].find("shaker") != -1: self.container = elements[0] status = elements[4] LOG.debug('Container status: %s' % str(status)) break
def _do_finalization(self, run_results): if run_results is None: LOG.warning("For some reason test tools have returned nothing") return self.describe_results(run_results) self.update_config(run_results) try: reporter.brew_a_report(run_results, self.results_vault + "/index.html") except Exception as e: LOG.warning("Brewing a report has failed with error: %s" % str(e)) LOG.debug(traceback.format_exc()) return result_dict = { "timestamp": self.timestamp_str, "location": self.results_vault } LOG.debug('Creating a .tar.gz archive with test reports') try: cmd = ("tar -zcf /tmp/mcv_run_%(timestamp)s.tar.gz" " -C %(location)s .") % result_dict utils.run_cmd(cmd) cmd = "rm -rf %(location)s" % {"location": self.results_vault} utils.run_cmd(cmd) except subprocess.CalledProcessError: LOG.warning('Creation of .tar.gz archive has failed. See log ' 'for details. You can still get your files from: ' '%s' % self.results_vault) LOG.debug(traceback.format_exc()) return LOG.debug("Finished creating a report.") LOG.info("One page report could be found in " "/tmp/mcv_run_%(timestamp)s.tar.gz\n" % result_dict) return result_dict
def _setup_ostf_on_docker(self): # Find docker container: self._verify_ostf_container_is_up() self._do_config_extraction() p = utils.run_cmd("docker ps") p = p.split('\n') for line in p: elements = line.split() if elements[1].find("ostf") != -1: self.container = elements[0] status = elements[4] LOG.debug("OSTF container status: " + status) break
def start_container(self): LOG.debug("Bringing up Tempest container with credentials") add_host = "" # TODO(albartash): Refactor this place! if self.access_data["auth_fqdn"] != '': add_host = "--add-host={fqdn}:{endpoint}".format( fqdn=self.access_data["auth_fqdn"], endpoint=self.access_data["public_endpoint_ip"]) res = subprocess.Popen( ["docker", "run", "-d", "-P=true"] + [add_host] * (add_host != "") + ["-p", "6001:6001", "-e", "OS_AUTH_URL=" + self.access_data["auth_url"], "-e", "OS_TENANT_NAME=" + self.access_data["tenant_name"], "-e", "OS_REGION_NAME" + self.access_data["region_name"], "-e", "OS_USERNAME="******"username"], "-e", "OS_PASSWORD="******"password"], "-e", "KEYSTONE_ENDPOINT_TYPE=publicUrl", "-v", '%s:/home/rally/.rally/tempest' % self.homedir, "-v", "%s:%s" % (self.homedir, self.home), "-w", self.home, "-t", "mcv-tempest"], stdout=subprocess.PIPE, preexec_fn=utils.ignore_sigint).stdout.read() LOG.debug('Finish bringing up Tempest container.' 'ID = %s' % str(res)) self.verify_container_is_up() self._patch_rally() # Hotfix. set rally's permission for .rally/ folder # Please remove this. Use: `sudo -u rally docker run` when # rally user gets its permissions to start docker containers cmd = 'docker exec -t {cid} sudo chown rally:rally /home/rally/.rally' utils.run_cmd(cmd.format(cid=self.container_id)) self.copy_config() self.install_tempest()
def _rally_deployment_check(self): LOG.debug("Checking if Rally deployment is present.") res = subprocess.Popen(["docker", "exec", "-t", self.container_id, "rally", "deployment", "check"], stdout=subprocess.PIPE, preexec_fn=utils.ignore_sigint).stdout.read() if "There is no" in res: LOG.debug("It is not. Trying to set up rally deployment.") self.create_rally_json() cmd = "docker exec -t %s rally deployment create " \ "--file=%s/conf/existing.json --name=existing" % \ (self.container_id, self.home) utils.run_cmd(cmd) else: LOG.debug("Seems like it is present.") LOG.debug('Trying to use Rally deployment') cmd = ("docker exec -t {cid} " "rally deployment use existing").format( cid=self.container_id) utils.run_cmd(cmd, quiet=True)
def install_tempest(self): LOG.debug("Searching for installed tempest") super(TempestOnDockerRunner, self)._rally_deployment_check() LOG.debug("Generating additional config") path_to_conf = os.path.join(self.homedir, 'additional.conf') with open(path_to_conf, 'wb') as conf_file: config = ConfigParser.ConfigParser() config._sections = tempest_additional_conf config.write(conf_file) LOG.debug("Installing tempest...") cmd = ("docker exec -t {cid} " "rally verify install --system-wide " "--deployment existing --source /tempest").format( cid=self.container_id) utils.run_cmd(cmd, quiet=True) cmd = "docker exec -t %(container)s rally verify genconfig " \ "--add-options %(conf_path)s" % \ {"container": self.container_id, "conf_path": os.path.join(self.home, 'additional.conf')} utils.run_cmd(cmd, quiet=True)
def _check_shaker_setup(self): LOG.info("Start shaker-image-builder. Creating infrastructure. " "Please wait...") cmd = "docker exec -t %s shaker-image-builder --image-name %s " \ "--flavor-name %s" % (self.container_id, self.image_name, self.flavor_name) p = utils.run_cmd(cmd) if 'ERROR' in p: LOG.debug("shaker-image-builder failed") for stack in self.heat.stacks.list(): if 'shaker' in stack.stack_name: stack.delete() raise RuntimeError LOG.debug('Finish running shaker-image-builder.')
def _get_task_result(self, task_id): # TODO(albartash): Fix the issue mentioned below: # this function is not using task id contrary to what it says, but in # current state of affair direct command produced by rally. task_id # is left as is for now, but will be moved in future. # if asked kindly rally just spits resulting json directly to stdout p = utils.run_cmd(task_id) try: res = json.loads(p)[0] return res except ValueError: LOG.error("Gotten not-JSON object. Please see mcv-log") LOG.debug("Not-JSON object: %s", p) return "Not-JSON object"
def _get_task_result_from_docker(self, task_id): LOG.info("Retrieving task results for %s" % task_id) cmd = "docker exec -t %s %s" % (self.container, task_id) p = utils.run_cmd(cmd) if task_id.find("detailed") == -1: try: res = json.loads(p)[0] return res except ValueError: LOG.error("Gotten not-JSON object. Please see mcv-log") LOG.debug("Not-JSON object: %s, After command: %s", p, cmd) return "Not-JSON object" else: return p.split('\n')[-4:-1]
def _os_patch(target, patch, container_id=None): """Silently patch a file. Errors are ignored params: 'target' - absolute system path to a file that needs to be changed 'patch' - absolute system path to a .patch file 'container_id' (optional). If provided - perform an operation inside a docker container """ tmp = 'sudo patch --dry-run --forward {target} -i {patch} && ' \ 'sudo patch {target} -i {patch}' if container_id: tmp = 'docker exec -t {cid} /bin/sh -c \"' + tmp + '\"' cmd = tmp.format(cid=container_id, target=target, patch=patch) try: return utils.run_cmd(cmd) except subprocess.CalledProcessError: pass