def gen_scenario_json(self, task_id): if self.openstack: cmd = "source {}; ".format(get_workload_venv('rally', True)) else: cmd = "source {}; ".format(get_workload_venv('rally-ovs', True)) cmd += "rally task results {}".format(task_id) return self.tools.run_cmd(cmd)['stdout']
def gen_scenario_html(self, task_ids, test_name): all_task_ids = ' '.join(task_ids) if self.openstack: cmd = "source {}; ".format(get_workload_venv('rally', True)) else: cmd = "source {}; ".format(get_workload_venv('rally-ovs', True)) cmd += "rally task report {} --out {}.html".format( all_task_ids, test_name) return self.tools.run_cmd(cmd)['stdout']
def run_scenario(self, task_file, scenario_args, result_dir, test_name, benchmark): self.logger.debug("--------------------------------") self.logger.debug("task_file: {}".format(task_file)) self.logger.info("Running with scenario_args: {}".format( json.dumps(scenario_args))) self.logger.debug("result_dir: {}".format(result_dir)) self.logger.debug("test_name: {}".format(test_name)) self.logger.debug("--------------------------------") from_ts = int(time.time() * 1000) if 'sleep_before' in self.config['rally']: time.sleep(self.config['rally']['sleep_before']) task_args = json.dumps(scenario_args) plugins = [] if "plugins" in self.config['rally']: if len(self.config['rally']['plugins']) > 0: for plugin in self.config['rally']['plugins']: for name in plugin: plugins.append(plugin[name]) plugin_string = "" if len(plugins) > 0: plugin_string = "--plugin-paths {}".format(",".join(plugins)) cmd = "source {}; ".format(get_workload_venv('rally', True)) cmd += "rally {} task start {} --task-args \'{}\' 2>&1 | tee {}.log".format( plugin_string, task_file, task_args, test_name) from_time = time.time() self.tools.run_cmd(cmd)['stdout'] to_time = time.time() if 'sleep_after' in self.config['rally']: time.sleep(self.config['rally']['sleep_after']) to_ts = int(time.time() * 1000) self.grafana.create_grafana_urls({'from_ts': from_ts, 'to_ts': to_ts}) self.grafana.print_dashboard_url(test_name) return (from_time, to_time)
def run_scenario(self, task_file, scenario_args, result_dir, test_name, benchmark): self.logger.debug("--------------------------------") self.logger.debug("task_file: {}".format(task_file)) self.logger.info("Running with scenario_args: {}".format(json.dumps(scenario_args))) self.logger.debug("result_dir: {}".format(result_dir)) self.logger.debug("test_name: {}".format(test_name)) self.logger.debug("--------------------------------") from_ts = int(time.time() * 1000) if 'sleep_before' in self.config['rally']: time.sleep(self.config['rally']['sleep_before']) task_args = json.dumps(scenario_args) plugins = [] if "plugins" in self.config['rally']: if len(self.config['rally']['plugins']) > 0: for plugin in self.config['rally']['plugins']: for name in plugin: plugins.append(plugin[name]) plugin_string = "" if len(plugins) > 0: plugin_string = "--plugin-paths {}".format(",".join(plugins)) cmd = "source {}; ".format(get_workload_venv('rally', True)) cmd += "rally {} task start {} --task-args \'{}\' 2>&1 | tee {}.log".format( plugin_string, task_file, task_args, test_name) from_time = time.time() self.tools.run_cmd(cmd)['stdout'] to_time = time.time() if 'sleep_after' in self.config['rally']: time.sleep(self.config['rally']['sleep_after']) to_ts = int(time.time() * 1000) self.grafana.create_grafana_urls({'from_ts': from_ts, 'to_ts': to_ts}) self.grafana.print_dashboard_url(test_name) return (from_time, to_time)
def shaker_checks(self): cmd = "source {}; source {}; glance image-list | grep -w shaker-image".format( get_workload_venv('shaker', True), self.overcloudrc) if self.tools.run_cmd(cmd)['stdout'] == "": self.logger.error("Shaker Image is not built, try again") exit(1) else: self.logger.info("Shaker image is built, continuing")
def run_scenario(self, scenario, result_dir, test_name, filename, shaker_uuid, es_ts, es_list, run): server_endpoint = self.config['shaker']['server'] port_no = self.config['shaker']['port'] flavor = self.config['shaker']['flavor'] venv = get_workload_venv('shaker', True) shaker_region = self.config['shaker']['shaker_region'] timeout = self.config['shaker']['join_timeout'] self.logger.info( "The uuid for this shaker scenario is {}".format(shaker_uuid)) cmd_env = ("source {0}; source {1}").format(venv, self.overcloudrc) if 'external' in filename and 'external_host' in self.config['shaker']: external_host = self.config['shaker']['external_host'] cmd_shaker = ( 'shaker --server-endpoint {0}:{1} --flavor-name {2} --scenario {3}' ' --os-region-name {7} --agent-join-timeout {6}' ' --report {4}/{5}.html --output {4}/{5}.json' ' --book {4}/{5} --matrix "{{host: {8}}}" --debug' ' > {4}/{5}.log 2>&1').format(server_endpoint, port_no, flavor, filename, result_dir, test_name, timeout, shaker_region, external_host) else: cmd_shaker = ( 'shaker --server-endpoint {0}:{1} --flavor-name {2} --scenario {3}' ' --os-region-name {7} --agent-join-timeout {6}' ' --report {4}/{5}.html --output {4}/{5}.json' ' --book {4}/{5} --debug' ' > {4}/{5}.log 2>&1').format(server_endpoint, port_no, flavor, filename, result_dir, test_name, timeout, shaker_region) cmd = ("{}; {}").format(cmd_env, cmd_shaker) from_ts = int(time.time() * 1000) if 'sleep_before' in self.config['shaker']: time.sleep(self.config['shaker']['sleep_before']) from_time = time.time() self.tools.run_cmd(cmd) to_time = time.time() self.update_total_tests() outputfile = os.path.join(result_dir, test_name + "." + "json") if 'sleep_after' in self.config['shaker']: time.sleep(self.config['shaker']['sleep_after']) to_ts = int(time.time() * 1000) # Snapshotting self.grafana.create_grafana_urls({'from_ts': from_ts, 'to_ts': to_ts}) self.grafana.print_dashboard_url(test_name) # Send Data to elastic if self.config['elasticsearch']['enabled']: index_status = self.send_to_elastic(outputfile, scenario['name'], shaker_uuid, es_ts, es_list, run, test_name, result_dir) self.result_check(result_dir, test_name, scenario, to_time, from_time, index_status) else: self.result_check(result_dir, test_name, scenario, to_time, from_time)
def run_scenario(self, scenario, result_dir, test_name, filename, shaker_uuid, es_ts, es_list, run): server_endpoint = self.config['shaker']['server'] port_no = self.config['shaker']['port'] flavor = self.config['shaker']['flavor'] venv = get_workload_venv('shaker', True) shaker_region = self.config['shaker']['shaker_region'] timeout = self.config['shaker']['join_timeout'] self.logger.info( "The uuid for this shaker scenario is {}".format(shaker_uuid)) cmd_env = ( "source {0}; source {1}").format(venv, self.overcloudrc) if 'external' in filename and 'external_host' in self.config['shaker']: external_host = self.config['shaker']['external_host'] cmd_shaker = ( 'shaker --server-endpoint {0}:{1} --flavor-name {2} --scenario {3}' ' --os-region-name {7} --agent-join-timeout {6}' ' --report {4}/{5}.html --output {4}/{5}.json' ' --book {4}/{5} --matrix "{{host: {8}}}" --debug' ' > {4}/{5}.log 2>&1').format(server_endpoint, port_no, flavor, filename, result_dir, test_name, timeout, shaker_region, external_host) else: cmd_shaker = ( 'shaker --server-endpoint {0}:{1} --flavor-name {2} --scenario {3}' ' --os-region-name {7} --agent-join-timeout {6}' ' --report {4}/{5}.html --output {4}/{5}.json' ' --book {4}/{5} --debug' ' > {4}/{5}.log 2>&1').format(server_endpoint, port_no, flavor, filename, result_dir, test_name, timeout, shaker_region) cmd = ("{}; {}").format(cmd_env, cmd_shaker) from_ts = int(time.time() * 1000) if 'sleep_before' in self.config['shaker']: time.sleep(self.config['shaker']['sleep_before']) from_time = time.time() self.tools.run_cmd(cmd) to_time = time.time() self.update_total_tests() outputfile = os.path.join(result_dir, test_name + "." + "json") if 'sleep_after' in self.config['shaker']: time.sleep(self.config['shaker']['sleep_after']) to_ts = int(time.time() * 1000) # Snapshotting self.grafana.create_grafana_urls({'from_ts': from_ts, 'to_ts': to_ts}) self.grafana.print_dashboard_url(test_name) # Send Data to elastic if self.config['elasticsearch']['enabled']: index_status = self.send_to_elastic(outputfile, scenario['name'], shaker_uuid, es_ts, es_list, run, test_name, result_dir) self.result_check(result_dir, test_name, scenario, to_time, from_time, index_status) else: self.result_check(result_dir, test_name, scenario, to_time, from_time)
def run_workload(self, workload, run_iteration): self.logger.info("Starting Shaker workloads") time_stamp = datetime.datetime.utcnow().strftime("%Y%m%d-%H%M%S") self.logger.debug("Time Stamp (Prefix): {}".format(time_stamp)) venv = get_workload_venv('shaker', False) self.shaker_checks() self.update_total_scenarios() shaker_uuid = uuid.uuid4() es_ts = datetime.datetime.utcnow() es_list = [] # Default test time to 60 test_time = workload.get("time", 60) for interval in range(0, test_time + 9): es_list.append( datetime.datetime.utcnow() + datetime.timedelta(0, interval)) rerun_range = range(self.config["browbeat"]["rerun"]) if self.config["browbeat"]["rerun_type"] == "complete": # Compelete rerun type means force rerun_range = range(run_iteration, run_iteration + 1) for run in rerun_range: self.logger.info("Scenario: {}".format(workload['name'])) self.logger.info("Run: {}".format(run)) fname = os.path.join(venv, workload['file']) self.set_scenario(workload, fname, 60) self.logger.debug("Set Scenario File: {}".format(fname)) result_dir = self.tools.create_results_dir( results_path, self.result_dir_ts, "shaker", workload['name'] + "-" + str(run)) self.workload_logger(self.__class__.__name__) time_stamp1 = datetime.datetime.now().strftime( "%Y%m%d-%H%M%S") test_name = "{}-browbeat-{}-{}-{}".format( time_stamp1, "shaker", workload['name'], run) self.run_scenario( workload, result_dir, test_name, fname, shaker_uuid, es_ts, es_list, run)
def gen_scenario_json_file(self, task_id, test_name): cmd = "source {}; ".format(get_workload_venv('rally', True)) cmd += "rally task results {} > {}.json".format(task_id, test_name) return self.tools.run_cmd(cmd)['stdout']
def run_benchmark(self, benchmark_config, result_dir, test_name, cloud_type="OpenStack"): self.logger.debug("--------------------------------") self.logger.debug("Benchmark_config: {}".format(benchmark_config)) self.logger.debug("result_dir: {}".format(result_dir)) self.logger.debug("test_name: {}".format(test_name)) self.logger.debug("--------------------------------") # Build command to run if 'enabled' in benchmark_config: del benchmark_config['enabled'] if 'type' in benchmark_config: del benchmark_config['type'] cmd = ("source {0}; source {1}; " "{2}/PerfKitBenchmarker/pkb.py " "--cloud={3} --run_uri=browbeat".format( get_workload_venv('perfkit', True), self.overcloudrc, get_workload_venv('perfkit', False), cloud_type)) for parameter, value in benchmark_config.iteritems(): if not parameter == 'name': self.logger.debug( "Parameter: {}, Value: {}".format(parameter, value)) cmd += " --{}={}".format(parameter, value) # Remove any old results if os.path.exists("/tmp/perfkitbenchmarker/runs/browbeat"): shutil.rmtree("/tmp/perfkitbenchmarker/runs/browbeat") self.logger.info("Running Perfkit Command: {}".format(cmd)) stdout_file = open("{}/pkb.stdout.log".format(result_dir), 'w') stderr_file = open("{}/pkb.stderr.log".format(result_dir), 'w') from_ts = time.time() if 'sleep_before' in self.config['perfkit']: time.sleep(self.config['perfkit']['sleep_before']) process = subprocess.Popen( cmd, shell=True, stdout=stdout_file, stderr=stderr_file) process.communicate() if 'sleep_after' in self.config['perfkit']: time.sleep(self.config['perfkit']['sleep_after']) to_ts = time.time() # Determine success success = False try: with open("{}/pkb.stderr.log".format(result_dir), 'r') as stderr: if any('SUCCEEDED' in line for line in stderr): self.logger.info("Benchmark completed.") success = True else: self.logger.error("Benchmark failed.") except IOError: self.logger.error( "File missing: {}/pkb.stderr.log".format(result_dir)) # Copy all results for perfkit_file in glob.glob("/tmp/perfkitbenchmarker/runs/browbeat/*"): shutil.move(perfkit_file, result_dir) if os.path.exists("/tmp/perfkitbenchmarker/runs/browbeat"): shutil.rmtree("/tmp/perfkitbenchmarker/runs/browbeat") # Grafana integration self.grafana.create_grafana_urls( {'from_ts': int(from_ts * 1000), 'to_ts': int(to_ts * 1000)}) self.grafana.print_dashboard_url(test_name) return success, to_ts, from_ts
def gen_scenario_json(self, task_id): cmd = "source {}; ".format(get_workload_venv('rally', True)) cmd += "rally task results --uuid {}".format(task_id) return self.tools.run_cmd(cmd)['stdout']
def run_workload(self, workload, run_iteration): """Runs a Browbeat Rally workload""" results = [] self.logger.info("Running Rally workload: {}".format(workload["name"])) es_ts = datetime.datetime.utcnow() def_concurrencies = workload["concurrency"] def_times = workload["times"] self.logger.debug( "Default Concurrencies: {}".format(def_concurrencies)) self.logger.debug("Default Times: {}".format(def_times)) for scenario in workload["scenarios"]: if not scenario["enabled"]: self.logger.info("{} scenario is disabled".format( scenario['name'])) continue self.logger.info("Running Scenario: {}".format(scenario["name"])) self.logger.debug("Scenario File: {}".format(scenario["file"])) self.update_total_scenarios() scenario_name = scenario["name"] scenario_file = scenario["file"] del scenario["enabled"] del scenario["file"] del scenario["name"] if len(scenario) > 0: self.logger.debug( "Overriding Scenario Args: {}".format(scenario)) result_dir = self.tools.create_results_dir( results_path, self.result_dir_ts, self.__class__.__name__.lower(), workload["name"], scenario_name) self.logger.debug( "Created result directory: {}".format(result_dir)) self.workload_logger(self.__class__.__name__) # Override concurrency/times if "concurrency" in scenario: concurrencies = scenario["concurrency"] del scenario["concurrency"] else: concurrencies = def_concurrencies if "times" not in scenario: scenario["times"] = def_times if "rally_deployment" in scenario: _rally_deployment = scenario["rally_deployment"] elif "rally_deployment" in workload: scenario["rally_deployment"] = workload["rally_deployment"] _rally_deployment = scenario["rally_deployment"] else: _rally_deployment = 'overcloud' self.logger.info("Default rally deployment {} in use.".format( _rally_deployment)) rally_deployments = ['undercloud', 'overcloud'] if _rally_deployment in rally_deployments: cmd = "source {}; ".format(get_workload_venv('rally', True)) cmd += "rally deployment use {}".format(_rally_deployment) cmd_stdout = self.tools.run_cmd(cmd)['stdout'] if cmd_stdout == "Deployment {} is not found.".format( _rally_deployment): self.logger.error( "Rally deployment {} is not found.".format( _rally_deployment)) exit(1) else: self.logger.error("Wrong rally benchmark name specified.") continue concurrency_count_dict = collections.Counter() for concurrency in concurrencies: scenario["concurrency"] = concurrency # Correct iteration/rerun rerun_range = range(self.config["browbeat"]["rerun"]) if self.config["browbeat"]["rerun_type"] == "complete": rerun_range = range(run_iteration, run_iteration + 1) for run in rerun_range: self.update_total_tests() concurrency_count_dict[concurrency] += 1 test_name = "{}-browbeat-{}-{}-{}-iteration-{}".format( es_ts.strftime("%Y%m%d-%H%M%S"), scenario_name, concurrency, concurrency_count_dict[concurrency], run) if not result_dir: self.logger.error("Failed to create result directory") exit(1) from_time, to_time = self.run_scenario( scenario_file, scenario, result_dir, test_name, workload["name"]) new_test_name = test_name.split("-") new_test_name = new_test_name[3:] new_test_name = "-".join(new_test_name) # Find task id (if task succeeded in running) task_id = self.get_task_id(test_name) if task_id: self.logger.info( "Generating Rally HTML for task_id : {}".format( task_id)) self.gen_scenario_html([task_id], test_name) self.gen_scenario_json_file(task_id, test_name) results.append(task_id) self.update_total_pass_tests() if self.config["elasticsearch"]["enabled"]: # Start indexing index_status = self.json_result( task_id, scenario_name, run, test_name, result_dir) if not index_status: self.update_index_failures() self.get_time_dict(to_time, from_time, workload["name"], new_test_name, self.__class__.__name__, "pass", index_status) else: self.get_time_dict( to_time, from_time, workload["name"], new_test_name, self.__class__.__name__, "pass", ) else: self.logger.error("Cannot find task_id") self.update_total_fail_tests() self.get_time_dict(to_time, from_time, workload["name"], new_test_name, self.__class__.__name__, "fail") for data in glob.glob("./{}*".format(test_name)): shutil.move(data, result_dir) self.logger.debug("Creating Combined Rally Reports") combined_html_name = "all-rally-run-{}".format(run_iteration) self.gen_scenario_html(results, combined_html_name) if os.path.isfile("{}.html".format(combined_html_name)): shutil.move( "{}.html".format(combined_html_name), "{}/{}/{}/{}".format(results_path, self.result_dir_ts, self.__class__.__name__.lower(), workload["name"]))
def gen_scenario_json(self, task_id): cmd = "source {}; ".format(get_workload_venv('rally', True)) cmd += "rally task results {}".format(task_id) return self.tools.run_cmd(cmd)['stdout']
def gen_scenario_html(self, task_ids, test_name): all_task_ids = ' '.join(task_ids) cmd = "source {}; ".format(get_workload_venv('rally', True)) cmd += "rally task report --task {} --out {}.html".format( all_task_ids, test_name) return self.tools.run_cmd(cmd)['stdout']