def main(argv): argv_obj = parse_args(argv) if argv_obj.jobfile == '-': job_cfg = sys.stdin.read() else: job_cfg = open(argv_obj.jobfile).read() params = {} for param_val in argv_obj.params: assert '=' in param_val name, val = param_val.split("=", 1) params[name] = parse_value(val) sec_it = fio_cfg_compile(job_cfg, argv_obj.jobfile, params) if argv_obj.action == 'estimate': print sec_to_str(sum(map(execution_time, sec_it))) elif argv_obj.action == 'num_tests': print sum(map(len, map(list, sec_it))) elif argv_obj.action == 'compile': splitter = "\n#" + "-" * 70 + "\n\n" print splitter.join(map(str, sec_it)) return 0
def main(argv): argv_obj = parse_args(argv) if argv_obj.jobfile == '-': job_cfg = sys.stdin.read() else: job_cfg = open(argv_obj.jobfile).read() params = {} for param_val in argv_obj.params: assert '=' in param_val name, val = param_val.split("=", 1) params[name] = parse_value(val) sec_it = fio_cfg_compile(job_cfg, argv_obj.jobfile, params) if argv_obj.action == 'estimate': print sec_to_str(sum(map(execution_time, sec_it))) elif argv_obj.action == 'num_tests': print sum(map(len, map(list, sec_it))) elif argv_obj.action == 'compile': splitter = "\n#" + "-" * 70 + "\n\n" print splitter.join(map(str, sec_it)) return 0
def show_test_execution_time(self): if len(self.fio_configs) > 1: # +10% - is a rough estimation for additional operations # like sftp, etc exec_time = int(sum(map(execution_time, self.fio_configs)) * 1.1) exec_time_s = sec_to_str(exec_time) now_dt = datetime.datetime.now() end_dt = now_dt + datetime.timedelta(0, exec_time) msg = "Entire test should takes aroud: {0} and finished at {1}" logger.info(msg.format(exec_time_s, end_dt.strftime("%H:%M:%S")))
def show_test_execution_time(self): if len(self.fio_configs) > 1: # +10% - is a rough estimation for additional operations # like sftp, etc exec_time = int(sum(map(execution_time, self.fio_configs)) * 1.1) exec_time_s = sec_to_str(exec_time) now_dt = datetime.datetime.now() end_dt = now_dt + datetime.timedelta(0, exec_time) msg = "Entire test should takes aroud: {0} and finished at {1}" logger.info(msg.format(exec_time_s, end_dt.strftime("%H:%M:%S")))
def run(self): logger.debug("Run preparation") self.pre_run() self.show_test_execution_time() tname = os.path.basename(self.config_fname) if tname.endswith('.cfg'): tname = tname[:-4] barrier = Barrier(len(self.config.nodes)) results = [] # set of Operation_Mode_BlockSize str's # which should not be tested anymore, as # they already too slow with previous thread count lat_bw_limit_reached = set() with ThreadPoolExecutor(len(self.config.nodes)) as pool: for pos, fio_cfg in enumerate(self.fio_configs): test_descr = get_test_summary(fio_cfg.vals).split("th")[0] if test_descr in lat_bw_limit_reached: continue else: logger.info("Will run {0} test".format(fio_cfg.name)) templ = "Test should takes about {0}." + \ " Should finish at {1}," + \ " will wait at most till {2}" exec_time = execution_time(fio_cfg) exec_time_str = sec_to_str(exec_time) timeout = int(exec_time + max(300, exec_time)) now_dt = datetime.datetime.now() end_dt = now_dt + datetime.timedelta(0, exec_time) wait_till = now_dt + datetime.timedelta(0, timeout) logger.info( templ.format(exec_time_str, end_dt.strftime("%H:%M:%S"), wait_till.strftime("%H:%M:%S"))) func = functools.partial(self.do_run, barrier=barrier, fio_cfg=fio_cfg, pos=pos) max_retr = 3 for idx in range(max_retr): try: intervals = list(pool.map(func, self.config.nodes)) if None not in intervals: break except (EnvironmentError, SSHException) as exc: logger.exception("During fio run") if idx == max_retr - 1: raise StopTestError("Fio failed", exc) logger.info("Reconnectiong, sleeping %ss and retrying", self.retry_time) wait([ pool.submit(node.connection.close) for node in self.config.nodes ]) time.sleep(self.retry_time) wait([ pool.submit(reconnect, node.connection, node.conn_url) for node in self.config.nodes ]) fname = "{0}_task.fio".format(pos) with open(os.path.join(self.config.log_directory, fname), "w") as fd: fd.write(str(fio_cfg)) params = {'vm_count': len(self.config.nodes)} params['name'] = fio_cfg.name params['vals'] = dict(fio_cfg.vals.items()) params['intervals'] = intervals params['nodes'] = [ node.get_conn_id() for node in self.config.nodes ] fname = "{0}_params.yaml".format(pos) with open(os.path.join(self.config.log_directory, fname), "w") as fd: fd.write(dumps(params)) res = load_test_results(self.config.log_directory, pos) results.append(res) if self.max_latency is not None: lat_50, _ = res.get_lat_perc_50_95_multy() # conver us to ms if self.max_latency < lat_50: logger.info( ("Will skip all subsequent tests of {0} " + "due to lat/bw limits").format(fio_cfg.name)) lat_bw_limit_reached.add(test_descr) test_res = res.get_params_from_fio_report() if self.min_bw_per_thread is not None: if self.min_bw_per_thread > average(test_res['bw']): lat_bw_limit_reached.add(test_descr) return IOTestResults(self.config.params['cfg'], results, self.config.log_directory)
def run(self): logger.debug("Run preparation") self.pre_run() self.show_test_execution_time() tname = os.path.basename(self.config_fname) if tname.endswith('.cfg'): tname = tname[:-4] barrier = Barrier(len(self.config.nodes)) results = [] # set of Operation_Mode_BlockSize str's # which should not be tested anymore, as # they already too slow with previous thread count lat_bw_limit_reached = set() with ThreadPoolExecutor(len(self.config.nodes)) as pool: for pos, fio_cfg in enumerate(self.fio_configs): test_descr = get_test_summary(fio_cfg.vals).split("th")[0] if test_descr in lat_bw_limit_reached: continue else: logger.info("Will run {0} test".format(fio_cfg.name)) templ = "Test should takes about {0}." + \ " Should finish at {1}," + \ " will wait at most till {2}" exec_time = execution_time(fio_cfg) exec_time_str = sec_to_str(exec_time) timeout = int(exec_time + max(300, exec_time)) now_dt = datetime.datetime.now() end_dt = now_dt + datetime.timedelta(0, exec_time) wait_till = now_dt + datetime.timedelta(0, timeout) logger.info(templ.format(exec_time_str, end_dt.strftime("%H:%M:%S"), wait_till.strftime("%H:%M:%S"))) func = functools.partial(self.do_run, barrier=barrier, fio_cfg=fio_cfg, pos=pos) max_retr = 3 for idx in range(max_retr): try: intervals = list(pool.map(func, self.config.nodes)) break except (EnvironmentError, SSHException) as exc: logger.exception("During fio run") if idx == max_retr - 1: raise StopTestError("Fio failed", exc) logger.info("Sleeping 30s and retrying") time.sleep(30) fname = "{0}_task.fio".format(pos) with open(os.path.join(self.config.log_directory, fname), "w") as fd: fd.write(str(fio_cfg)) params = {'vm_count': len(self.config.nodes)} params['name'] = fio_cfg.name params['vals'] = dict(fio_cfg.vals.items()) params['intervals'] = intervals params['nodes'] = [node.get_conn_id() for node in self.config.nodes] fname = "{0}_params.yaml".format(pos) with open(os.path.join(self.config.log_directory, fname), "w") as fd: fd.write(dumps(params)) res = load_test_results(self.config.log_directory, pos) results.append(res) if self.max_latency is not None: lat_50, _ = res.get_lat_perc_50_95_multy() # conver us to ms if self.max_latency < lat_50: logger.info(("Will skip all subsequent tests of {0} " + "due to lat/bw limits").format(fio_cfg.name)) lat_bw_limit_reached.add(test_descr) test_res = res.get_params_from_fio_report() if self.min_bw_per_thread is not None: if self.min_bw_per_thread > average(test_res['bw']): lat_bw_limit_reached.add(test_descr) return IOTestResults(self.config.params['cfg'], results, self.config.log_directory)