def print_task_ratio(locusts, total=False, level=0, parent_ratio=1.0): """ Output table with task execution ratio info to console_logger """ ratio = {} for locust in locusts: ratio.setdefault(locust, 0) ratio[locust] += 1 # get percentage ratio_percent = dict( map(lambda x: (x[0], float(x[1]) / len(locusts) * parent_ratio), ratio.iteritems())) for locust, ratio in ratio_percent.iteritems(): console_logger.info(" %-10s %-50s" % (" " * level + "%-6.1f" % (ratio * 100), " " * level + locust.__name__)) if inspect.isclass(locust): if issubclass(locust, Locust): if total: print_task_ratio(locust.task_set.tasks, total, level + 1, ratio) else: print_task_ratio(locust.task_set.tasks, total, level + 1) elif issubclass(locust, TaskSet): if total: print_task_ratio(locust.tasks, total, level + 1, ratio) else: print_task_ratio(locust.tasks, total, level + 1)
def _print_task_ratio(x, level=0): for k, v in x.iteritems(): padding = 2*" "*level ratio = v.get('ratio', 1) console_logger.info(" %-10s %-50s" % (padding + "%-6.1f" % (ratio*100), padding + k)) if 'tasks' in v: _print_task_ratio(v['tasks'], level + 1)
def print_task_ratio(locusts, total=False, level=0, parent_ratio=1.0): """ Output table with task execution ratio info to console_logger """ ratio = {} for locust in locusts: ratio.setdefault(locust, 0) ratio[locust] += 1 # get percentage ratio_percent = dict(map(lambda x: (x[0], float(x[1])/len(locusts) * parent_ratio), ratio.iteritems())) for locust, ratio in ratio_percent.iteritems(): console_logger.info(" %-10s %-50s" % (" "*level + "%-6.1f" % (ratio*100), " "*level + locust.__name__)) if inspect.isclass(locust): if issubclass(locust, Locust): if total: print_task_ratio(locust.task_set.tasks, total, level+1, ratio) else: print_task_ratio(locust.task_set.tasks, total, level+1) elif issubclass(locust, TaskSet): if total: print_task_ratio(locust.tasks, total, level+1, ratio) else: print_task_ratio(locust.tasks, total, level+1)
def get_final_stats_after_shutdown(): from runners import locust_runner import os pkg_version = os.getenv('PKG_VERSION', 'PKG_VERSION_ENV_NOT_FOUND') ret = {pkg_version: get_all_stats(locust_runner.request_stats)} console_logger.info("== percentile_stats = {0}".format(ret)) return ret
def local_file_append_df( df: pd.DataFrame, local_path: str, **kw, ): ''' 追加写入本地文件 ''' if not os.path.exists(local_path): mkdir(os.path.dirname(local_path)) if not os.path.exists(local_path): df_partial = partial( df.to_csv, local_path, encoding='utf-8-sig', ) else: # * 文件存在时追加不写header df_partial = partial( df.to_csv, local_path, mode='a', header=False, encoding='utf-8-sig', ) df_shape = df.shape if df_shape[0]: logger.info(f'{df_shape} append to -> {local_path}') df_partial(**kw) return
def _print_task_ratio(x, level=0): for k, v in x.iteritems(): padding = 2 * " " * level ratio = v.get('ratio', 1) console_logger.info(" %-10s %-50s" % (padding + "%-6.1f" % (ratio * 100), padding + k)) if 'tasks' in v: _print_task_ratio(v['tasks'], level + 1)
def init_clean_file( local_path: str, clean_flag=0, ): ''' 清理路径下的文件 ''' try: if not clean_flag: logger.info(f'Init Clean File -> {local_path}') os.remove(local_path) except Exception: pass
def write_df_to_csv( total_df: pd.DataFrame, filter_df: pd.DataFrame, folder_name: str, ): match_result_path = os.path.join(folder_name, 'match_result.csv') if args.csv and os.path.exists(match_result_path): logger.info(f'match result saved as -> {match_result_path}') summary_path = os.path.join(folder_name, 'summary_result.csv') if len(total_df): local_file_append_df(total_df, summary_path, index=True) logger.info(f'summary result saved as -> {summary_path}') filtered_path = os.path.join(folder_name, 'filter_result.csv') if len(filter_df): local_file_append_df(filter_df, filtered_path, index=True) logger.info(f'filtered result saved as -> {filtered_path}')
def download_single_file( cos_client: CosS3Client, bucket: str, cos_path: str, local_path: str, **kw, ): """ 下载单个文件 """ if os.path.exists(local_path): logger.info(f'{cos_path} already exists') return local_path logger.info( f'start download file form bucket -> {bucket}, Path -> {cos_path}') resp = cos_client.download_file( Bucket=bucket, Key=cos_path, DestFilePath=local_path, ) result = local_path if resp is None else resp logger.info(f'success, saved as -> {result}') return result
def print_error_report(): console_logger.info("Error report") console_logger.info(" %-18s %-100s" % ("# occurences", "Error")) console_logger.info("-" * (80 + STATS_NAME_WIDTH)) for error, count in RequestStats.errors.iteritems(): console_logger.info(" %-18i %-100s" % (count, error)) console_logger.info("-" * (80 + STATS_NAME_WIDTH)) console_logger.info("")
def print_stats(stats): console_logger.info((" %-" + str(STATS_NAME_WIDTH) + "s %7s %12s %7s %7s %7s | %7s %7s") % ('Name', '# reqs', '# fails', 'Avg', 'Min', 'Max', 'Median', 'req/s')) console_logger.info("-" * (80 + STATS_NAME_WIDTH)) total_rps = 0 total_reqs = 0 total_failures = 0 avg_response_time = [] median_response_time = [] min_response_time = None max_response_time = None avg_content_length = [] for key in sorted(stats.iterkeys()): r = stats[key] total_rps += r.current_rps total_reqs += r.num_requests total_failures += r.num_failures median_response_time.append(r.median_response_time) avg_response_time.append(r.avg_response_time) if min_response_time is None: min_response_time = r.min_response_time else: if r.min_response_time < min_response_time: min_response_time = r.min_response_time if max_response_time is None: max_response_time = r.max_response_time else: if r.max_response_time > max_response_time: max_response_time = r.max_response_time avg_content_length.append(r.avg_content_length) console_logger.info(r) console_logger.info("-" * (80 + STATS_NAME_WIDTH)) try: fail_percent = (total_failures/float(total_reqs))*100 except ZeroDivisionError: fail_percent = 0 console_logger.info((" %-" + str(STATS_NAME_WIDTH) + "s %7d %12s %42.2f") % ('Total', total_reqs, "%d(%.2f%%)" % (total_failures, fail_percent), total_rps)) console_logger.info("total_reqs,total_failures,median_response_time,avg_response_time,min_response_time,max_response_time,avg_content_length,total_rps") console_logger.info("{},{},{},{},{},{},{},{}".format( total_reqs, total_failures, sum(median_response_time)/len(median_response_time), sum(avg_response_time)/len(avg_response_time), min_response_time, max_response_time, sum(avg_content_length)/len(avg_content_length), total_rps )) console_logger.info("")
def main(): config = load_config(args) bucket = config.get('bucket') cos_client = get_cos_client(**config) global FILTER_RESULT, DIFF_RESULT, TOTAL_PREFIX_SET, MANIFEST_DATA log_tmp_folder = 'log_tmp' result_folder = 'result' manifest_folder = 'manifest' result_folder_name = os.path.join(result_folder, now_time_fmt) mkdir(log_tmp_folder, result_folder, manifest_folder) MANIFEST_DATA = get_manifest_data( cos_client, bucket, manifest_folder, args.manifest, ) MANIFEST_DATA = get_mainfest_header(MANIFEST_DATA) cos_file_list, file_schema = MANIFEST_DATA['files'], MANIFEST_DATA[ f'_{FILE_SCHEMA}'] logger.info(f'{len(cos_file_list)} csv files in total') local_file_path = download_csv_files( cos_client, bucket, cos_file_list, log_tmp_folder, ) if not len(local_file_path): logger.info('Not Found Manifest File') sys.exit() logger_debug('The manifest file has been saved.') prefix_level = -1 if args.prefix_level is not None: prefix_level = int(args.prefix_level) index = 0 file_bar = tqdm( local_file_path, desc='Manifest File Processing', unit='file', leave=False, ) for _file in file_bar: logger_debug(f'loading file to df -> {_file}') dfs = pd.read_csv( _file, compression='gzip', names=file_schema, chunksize=CHUNK_SIZE, ) for df in dfs: # * 对数据进行预处理 filter_df, diff_df = pre_proc_df(df, prefix_level) del df filter_r, diff_r = proc_single_csv( filter_df, diff_df, file_schema, ) # * 写入预处理后的df FILTER_RESULT = merge_df(FILTER_RESULT, filter_r) DIFF_RESULT = merge_df(DIFF_RESULT, diff_r) if args.csv: match_result_file_name = os.path.join( result_folder_name, 'match_result.csv') # * 生成的csv文件名 filter_df = filter_df[file_schema] local_file_append_df(filter_df, match_result_file_name, index) del filter_df, diff_r index += 1 total_df, filter_df = get_manifest_result( FILTER_RESULT, DIFF_RESULT, TOTAL_PREFIX_SET, ) write_df_to_csv( total_df, filter_df, result_folder_name, )
def print_stats(stats): console_logger.info( (" %-" + str(STATS_NAME_WIDTH) + "s %7s %12s %7s %7s %7s | %7s %7s") % ("Name", "# reqs", "# fails", "Avg", "Min", "Max", "Median", "req/s") ) console_logger.info("-" * (80 + STATS_NAME_WIDTH)) total_rps = 0 total_reqs = 0 total_failures = 0 for key in sorted(stats.iterkeys()): r = stats[key] total_rps += r.current_rps total_reqs += r.num_requests total_failures += r.num_failures console_logger.info(r) console_logger.info("-" * (80 + STATS_NAME_WIDTH)) try: fail_percent = (total_failures / float(total_reqs)) * 100 except ZeroDivisionError: fail_percent = 0 console_logger.info( (" %-" + str(STATS_NAME_WIDTH) + "s %7d %12s %42.2f") % ("Total", total_reqs, "%d(%.2f%%)" % (total_failures, fail_percent), total_rps) ) console_logger.info("")
def main(args=None): parser, options, arguments = parse_options(args) # setup logging setup_logging(options.loglevel, options.logfile) logger = logging.getLogger(__name__) if options.show_version: print "Locust %s" % (version,) sys.exit(0) locustfile = find_locustfile(options.locustfile) if not locustfile: logger.error("Could not find any locustfile! Ensure file ends in '.py' and see --help for available options.") sys.exit(1) docstring, locusts = load_locustfile(locustfile) if options.list_commands: console_logger.info("Available Locusts:") for name in locusts: console_logger.info(" " + name) sys.exit(0) if not locusts: logger.error("No Locust class found!") sys.exit(1) # make sure specified Locust exists if arguments: missing = set(arguments) - set(locusts.keys()) if missing: logger.error("Unknown Locust(s): %s\n" % (", ".join(missing))) sys.exit(1) else: names = set(arguments) & set(locusts.keys()) locust_classes = [locusts[n] for n in names] else: locust_classes = locusts.values() if options.show_task_ratio: console_logger.info("\n Task ratio per locust class") console_logger.info( "-" * 80) print_task_ratio(locust_classes) console_logger.info("\n Total task ratio") console_logger.info("-" * 80) print_task_ratio(locust_classes, total=True) sys.exit(0) if options.show_task_ratio_json: from json import dumps task_data = { "per_class": get_task_ratio_dict(locust_classes), "total": get_task_ratio_dict(locust_classes, total=True) } console_logger.info(dumps(task_data)) sys.exit(0) # if --master is set, make sure --no-web isn't set if options.master and options.no_web: logger.error("Locust can not run distributed with the web interface disabled (do not use --no-web and --master together)") sys.exit(0) if not options.no_web and not options.slave: # spawn web greenlet logger.info("Starting web monitor at %s:%s" % (options.web_host or "*", options.port)) main_greenlet = gevent.spawn(web.start, locust_classes, options) if not options.master and not options.slave: runners.locust_runner = LocalLocustRunner(locust_classes, options) # spawn client spawning/hatching greenlet if options.no_web: runners.locust_runner.start_hatching(wait=True) main_greenlet = runners.locust_runner.greenlet elif options.master: runners.locust_runner = MasterLocustRunner(locust_classes, options) elif options.slave: try: runners.locust_runner = SlaveLocustRunner(locust_classes, options) main_greenlet = runners.locust_runner.greenlet except socket.error, e: logger.error("Failed to connect to the Locust master: %s", e) sys.exit(-1)
def print_percentile_stats(stats): console_logger.info( "Percentage of the requests completed within given times") console_logger.info((" %-" + str(STATS_NAME_WIDTH) + "s %8s %6s %6s %6s %6s %6s %6s %6s %6s %6s") % ('Name', '# reqs', '50%', '66%', '75%', '80%', '90%', '95%', '98%', '99%', '100%')) console_logger.info("-" * (80 + STATS_NAME_WIDTH)) for key in sorted(stats.iterkeys()): r = stats[key] if r.response_times: console_logger.info(r.percentile()) console_logger.info("-" * (80 + STATS_NAME_WIDTH)) total_stats = global_stats.aggregated_stats() if total_stats.response_times: console_logger.info(total_stats.percentile()) console_logger.info("")
def main(): parser, options, arguments = parse_options() # setup logging setup_logging(options.loglevel, options.logfile) logger = logging.getLogger(__name__) if options.show_version: print "Locust %s" % (version,) sys.exit(0) locustfile = find_locustfile(options.locustfile) if not locustfile: logger.error("Could not find any locustfile! See --help for available options.") sys.exit(1) docstring, locusts = load_locustfile(locustfile) if options.list_commands: print "Available Locusts:" for name in locusts: print " " + name sys.exit(0) if not locusts: logger.error("No Locust class found!") sys.exit(1) # make sure specified Locust exists if arguments: missing = set(arguments) - set(locusts.keys()) if missing: logger.error("Unknown Locust(s): %s\n" % (", ".join(missing))) sys.exit(1) else: names = set(arguments) & set(locusts.keys()) locust_classes = [locusts[n] for n in names] else: locust_classes = locusts.values() if options.show_task_ratio: console_logger.info("\n Task ratio per locust class") console_logger.info("-" * 80) print_task_ratio(locust_classes) console_logger.info("\n Total task ratio") console_logger.info("-" * 80) print_task_ratio(locust_classes, total=True) sys.exit(0) if options.show_task_ratio_json: from json import dumps task_data = { "per_class": get_task_ratio_dict(locust_classes), "total": get_task_ratio_dict(locust_classes, total=True), } console_logger.info(dumps(task_data)) sys.exit(0) # if --master is set, make sure --no-web isn't set if options.master and options.no_web: logger.error( "Locust can not run distributed with the web interface disabled (do not use --no-web and --master together)" ) sys.exit(0) if not options.no_web and not options.slave: # spawn web greenlet logger.info("Starting web monitor on port %s" % options.port) main_greenlet = gevent.spawn( web.start, locust_classes, options.hatch_rate, options.num_clients, options.num_requests, options.ramp, options.port, ) if not options.master and not options.slave: runners.locust_runner = LocalLocustRunner( locust_classes, options.hatch_rate, options.num_clients, options.num_requests, options.host ) # spawn client spawning/hatching greenlet if options.no_web: runners.locust_runner.start_hatching(wait=True) main_greenlet = runners.locust_runner.greenlet elif options.master: runners.locust_runner = MasterLocustRunner( locust_classes, options.hatch_rate, options.num_clients, num_requests=options.num_requests, host=options.host, master_host=options.master_host, ) elif options.slave: runners.locust_runner = SlaveLocustRunner( locust_classes, options.hatch_rate, options.num_clients, num_requests=options.num_requests, host=options.host, master_host=options.master_host, ) main_greenlet = runners.locust_runner.greenlet if options.print_stats or (options.no_web and not options.slave): # spawn stats printing greenlet gevent.spawn(stats_printer) def shutdown(code=0): """ Shut down locust by firing quitting event, printing stats and exiting """ logger.info("Shutting down, bye..") events.quitting.fire() print_stats(runners.locust_runner.request_stats) print_percentile_stats(runners.locust_runner.request_stats) print_error_report() sys.exit(code) # install SIGTERM handler def sig_term_handler(): logger.info("Got SIGTERM signal") shutdown(0) gevent.signal(signal.SIGTERM, sig_term_handler) try: logger.info("Starting Locust %s" % version) main_greenlet.join() shutdown(0) except KeyboardInterrupt, e: shutdown(0)
def print_total_stats(stats): console_logger.info( (" %-" + str(STATS_NAME_WIDTH) + "s %7s %12s %7s %7s %7s | %7s %7s") % ('Name', '# reqs', '# fails', 'Avg', 'Min', 'Max', 'Median', 'req/s')) console_logger.info("-" * (80 + STATS_NAME_WIDTH)) current_rps = 0 total_rps = 0 total_reqs = 0 total_failures = 0 for key in sorted(stats.iterkeys()): r = stats[key] current_rps += r.current_rps total_rps += r.total_rps total_reqs += r.num_requests total_failures += r.num_failures console_logger.info(r) console_logger.info("-" * (80 + STATS_NAME_WIDTH)) try: fail_percent = (total_failures/float(total_reqs))*100 except ZeroDivisionError: fail_percent = 0 console_logger.info( (" %-" + str(STATS_NAME_WIDTH) + "s %7d %12s %42.2f") % ('Current', total_reqs, "%d(%.2f%%)" % (total_failures, fail_percent), current_rps)) console_logger.info( (" %-" + str(STATS_NAME_WIDTH) + "s %7d %12s %42.2f") % ('Total', total_reqs, "%d(%.2f%%)" % (total_failures, fail_percent), total_rps)) console_logger.info("")
def print_percentile_stats(stats): console_logger.info("Percentage of the requests completed within given times") console_logger.info((" %-" + str(STATS_NAME_WIDTH) + "s %8s %6s %6s %6s %6s %6s %6s %6s %6s %6s") % ('Name', '# reqs', '50%', '66%', '75%', '80%', '90%', '95%', '98%', '99%', '100%')) console_logger.info("-" * (80 + STATS_NAME_WIDTH)) for key in sorted(stats.iterkeys()): r = stats[key] if r.response_times: console_logger.info(r.percentile()) console_logger.info("-" * (80 + STATS_NAME_WIDTH)) total_stats = global_stats.aggregated_stats() if total_stats.response_times: console_logger.info(total_stats.percentile()) console_logger.info("")
def print_percentile_stats(stats): console_logger.info("Percentage of the requests completed within given times") console_logger.info( (" %-" + str(STATS_NAME_WIDTH) + "s %8s %6s %6s %6s %6s %6s %6s %6s %6s %6s") % ("Name", "# reqs", "50%", "66%", "75%", "80%", "90%", "95%", "98%", "99%", "100%") ) console_logger.info("-" * (80 + STATS_NAME_WIDTH)) for key in sorted(stats.iterkeys()): r = stats[key] if r.response_times: console_logger.info(r.percentile()) console_logger.info("-" * (80 + STATS_NAME_WIDTH)) total_stats = global_stats.aggregated_stats() if total_stats.response_times: console_logger.info(total_stats.percentile()) console_logger.info("")
def main(): parser, options, arguments = parse_options() #print "Options:", options, dir(options) #print "Arguments:", arguments #print "largs:", parser.largs #print "rargs:", parser.rargs # setup logging setup_logging(options.loglevel, options.logfile) logger = logging.getLogger(__name__) if options.show_version: print "Locust %s" % (version) sys.exit(0) locustfile = find_locustfile(options.locustfile) if not locustfile: logger.error( "Could not find any locustfile! See --help for available options.") sys.exit(1) docstring, locusts = load_locustfile(locustfile) if options.list_commands: print "Available Locusts:" for name in locusts: print " " + name sys.exit(0) if not locusts: logger.error("No Locust class found!") sys.exit(1) # make sure specified Locust exists if arguments: missing = set(arguments) - set(locusts.keys()) if missing: logger.error("Unknown Locust(s): %s\n" % (", ".join(missing))) sys.exit(1) else: names = set(arguments) & set(locusts.keys()) locust_classes = [locusts[n] for n in names] else: locust_classes = locusts.values() if options.show_task_ratio: console_logger.info("\n Task ratio per locust class") console_logger.info("-" * 80) print_task_ratio(locust_classes) console_logger.info("\n Total task ratio") console_logger.info("-" * 80) print_task_ratio(locust_classes, total=True) sys.exit(0) if options.show_task_ratio_json: from json import dumps task_data = { "per_class": get_task_ratio_dict(locust_classes), "total": get_task_ratio_dict(locust_classes, total=True) } console_logger.info(dumps(task_data)) sys.exit(0) # if --master is set, make sure --no-web isn't set if options.master and options.no_web: logger.error( "Locust can not run distributed with the web interface disabled (do not use --no-web and --master together)" ) sys.exit(0) if not options.no_web and not options.slave: # spawn web greenlet logger.info("Starting web monitor on port 8089") main_greenlet = gevent.spawn(web.start, locust_classes, options.hatch_rate, options.num_clients, options.num_requests, options.ramp) # enable/disable gzip in WebLocust's HTTP client WebLocust.gzip = options.gzip if not options.master and not options.slave: runners.locust_runner = LocalLocustRunner(locust_classes, options.hatch_rate, options.num_clients, options.num_requests, options.host) # spawn client spawning/hatching greenlet if options.no_web: runners.locust_runner.start_hatching(wait=True) main_greenlet = runners.locust_runner.greenlet elif options.master: runners.locust_runner = MasterLocustRunner( locust_classes, options.hatch_rate, options.num_clients, num_requests=options.num_requests, host=options.host, master_host=options.master_host) elif options.slave: runners.locust_runner = SlaveLocustRunner( locust_classes, options.hatch_rate, options.num_clients, num_requests=options.num_requests, host=options.host, master_host=options.master_host) main_greenlet = runners.locust_runner.greenlet if options.ramp: import rampstats from rampstats import on_request_success, on_report_to_master, on_slave_report if options.slave: events.report_to_master += on_report_to_master if options.master: events.slave_report += on_slave_report else: events.request_success += on_request_success if options.print_stats or (options.no_web and not options.slave): # spawn stats printing greenlet gevent.spawn(stats_printer) try: logger.info("Starting Locust %s" % version) main_greenlet.join() except KeyboardInterrupt, e: events.quitting.fire() time.sleep(0.2) print_stats(runners.locust_runner.request_stats) print_percentile_stats(runners.locust_runner.request_stats) print_error_report() logger.info("Got KeyboardInterrupt. Exiting, bye..")
def print_error_report(): if not len(global_stats.errors): return console_logger.info("Error report") console_logger.info(" %-18s %-100s" % ("# occurences", "Error")) console_logger.info("-" * (80 + STATS_NAME_WIDTH)) for error in global_stats.errors.itervalues(): console_logger.info(" %-18i %-100s" % (error.occurences, error.to_name())) console_logger.info("-" * (80 + STATS_NAME_WIDTH)) console_logger.info("")
def main(): parser, options, arguments = parse_options() # setup logging setup_logging(options.loglevel, options.logfile) logger = logging.getLogger(__name__) if options.show_version: print "Locust %s" % (version,) sys.exit(0) if os.path.isdir(options.locustfile): all_locustfiles = collect_locustfiles(options.locustfile) else: locustfile = find_locustfile(options.locustfile) if not locustfile: logger.error("Could not find any locustfile! Ensure file ends in '.py' and see --help for available options.") sys.exit(1) all_locustfiles = load_locustfile(locustfile) logger.info("All available locustfiles: {}".format(all_locustfiles)) # Use the first locustfile for the default locusts locusts = all_locustfiles.values()[0] if options.list_commands: console_logger.info("Available Locusts:") for name in locusts: console_logger.info(" " + name) sys.exit(0) if not locusts: logger.error("No Locust class found!") sys.exit(1) # make sure specified Locust exists if arguments: missing = set(arguments) - set(locusts.keys()) if missing: logger.error("Unknown Locust(s): %s\n" % (", ".join(missing))) sys.exit(1) else: names = set(arguments) & set(locusts.keys()) locust_classes = [locusts[n] for n in names] else: locust_classes = locusts.values() if options.show_task_ratio: console_logger.info("\n Task ratio per locust class") console_logger.info( "-" * 80) print_task_ratio(locust_classes) console_logger.info("\n Total task ratio") console_logger.info("-" * 80) print_task_ratio(locust_classes, total=True) sys.exit(0) if options.show_task_ratio_json: from json import dumps task_data = { "per_class": get_task_ratio_dict(locust_classes), "total": get_task_ratio_dict(locust_classes, total=True) } console_logger.info(dumps(task_data)) sys.exit(0) if options.master and options.no_web and not options.min_slaves: logger.error("When running --master and --no-web, you must specify --min-slaves to be available before starting to swarm") sys.exit(1) if options.master and options.no_web and not (options.timeout or options.num_requests): logger.error("When running --master and --no-web, you must specify either --num-request or --timeout to tell the slaves when to stop running each locustfile") sys.exit(1) if not options.no_web and not options.slave: # spawn web greenlet logger.info("Starting web monitor at %s:%s" % (options.web_host or "*", options.port)) main_greenlet = gevent.spawn(web.start, locust_classes, options) if options.slave: logger.info("Waiting for master to become available") try: runners.locust_runner = polling.poll( lambda: SlaveLocustRunner(locust_classes, options, available_locustfiles=all_locustfiles), timeout=60, step=1, ignore_exceptions=(socket.error,)) except polling.TimeoutException, e: logger.error("Failed to connect to the Locust master: %s", e.last) sys.exit(-1) main_greenlet = runners.locust_runner.greenlet
def main(): parser, options, arguments = parse_options() # setup logging setup_logging(options.loglevel, options.logfile) logger = logging.getLogger(__name__) if options.show_version: print "Locust %s" % (version, ) sys.exit(0) locustfile = find_locustfile(options.locustfile) if not locustfile: logger.error( "Could not find any locustfile! Ensure file ends in '.py' and see --help for available options." ) sys.exit(1) docstring, locusts = load_locustfile(locustfile) if options.list_commands: console_logger.info("Available Locusts:") for name in locusts: console_logger.info(" " + name) sys.exit(0) if not locusts: logger.error("No Locust class found!") sys.exit(1) # make sure specified Locust exists if arguments: missing = set(arguments) - set(locusts.keys()) if missing: logger.error("Unknown Locust(s): %s\n" % (", ".join(missing))) sys.exit(1) else: names = set(arguments) & set(locusts.keys()) locust_classes = [locusts[n] for n in names] else: locust_classes = locusts.values() if options.show_task_ratio: console_logger.info("\n Task ratio per locust class") console_logger.info("-" * 80) print_task_ratio(locust_classes) console_logger.info("\n Total task ratio") console_logger.info("-" * 80) print_task_ratio(locust_classes, total=True) sys.exit(0) if options.show_task_ratio_json: from json import dumps task_data = { "per_class": get_task_ratio_dict(locust_classes), "total": get_task_ratio_dict(locust_classes, total=True) } console_logger.info(dumps(task_data)) sys.exit(0) # if --master is set, make sure --no-web isn't set if options.master and options.no_web: logger.error( "Locust can not run distributed with the web interface disabled (do not use --no-web and --master together)" ) sys.exit(0) if not options.no_web and not options.slave: # spawn web greenlet logger.info("Starting web monitor at %s:%s" % (options.web_host or "*", options.port)) main_greenlet = gevent.spawn(web.start, locust_classes, options) if not options.master and not options.slave: runners.locust_runner = LocalLocustRunner(locust_classes, options) # spawn client spawning/hatching greenlet if options.no_web: runners.locust_runner.start_hatching(wait=True) main_greenlet = runners.locust_runner.greenlet elif options.master: runners.locust_runner = MasterLocustRunner(locust_classes, options) elif options.slave: try: runners.locust_runner = SlaveLocustRunner(locust_classes, options) main_greenlet = runners.locust_runner.greenlet except socket.error, e: logger.error("Failed to connect to the Locust master: %s", e) sys.exit(-1)
def main(): parser, options, arguments = parse_options() #print "Options:", options, dir(options) #print "Arguments:", arguments #print "largs:", parser.largs #print "rargs:", parser.rargs # setup logging setup_logging(options.loglevel, options.logfile) logger = logging.getLogger(__name__) if options.show_version: print "Locust %s" % (version) sys.exit(0) locustfile = find_locustfile(options.locustfile) if not locustfile: logger.error("Could not find any locustfile! See --help for available options.") sys.exit(1) docstring, locusts = load_locustfile(locustfile) if options.list_commands: print "Available Locusts:" for name in locusts: print " " + name sys.exit(0) if not locusts: logger.error("No Locust class found!") sys.exit(1) # make sure specified Locust exists if arguments: missing = set(arguments) - set(locusts.keys()) if missing: logger.error("Unknown Locust(s): %s\n" % (", ".join(missing))) sys.exit(1) else: names = set(arguments) & set(locusts.keys()) locust_classes = [locusts[n] for n in names] else: locust_classes = locusts.values() if options.show_task_ratio: console_logger.info("\n Task ratio per locust class") console_logger.info( "-" * 80) print_task_ratio(locust_classes) console_logger.info("\n Total task ratio") console_logger.info("-" * 80) print_task_ratio(locust_classes, total=True) sys.exit(0) if options.show_task_ratio_json: from json import dumps task_data = { "per_class": get_task_ratio_dict(locust_classes), "total": get_task_ratio_dict(locust_classes, total=True) } console_logger.info(dumps(task_data)) sys.exit(0) # if --master is set, make sure --no-web isn't set if options.master and options.no_web: logger.error("Locust can not run distributed with the web interface disabled (do not use --no-web and --master together)") sys.exit(0) if not options.no_web and not options.slave: # spawn web greenlet logger.info("Starting web monitor on port 8089") main_greenlet = gevent.spawn(web.start, locust_classes, options.hatch_rate, options.num_clients, options.num_requests, options.ramp) # enable/disable gzip in WebLocust's HTTP client WebLocust.gzip = options.gzip if not options.master and not options.slave: runners.locust_runner = LocalLocustRunner(locust_classes, options.hatch_rate, options.num_clients, options.num_requests, options.host) # spawn client spawning/hatching greenlet if options.no_web: runners.locust_runner.start_hatching(wait=True) main_greenlet = runners.locust_runner.greenlet elif options.master: runners.locust_runner = MasterLocustRunner(locust_classes, options.hatch_rate, options.num_clients, num_requests=options.num_requests, host=options.host, master_host=options.master_host) elif options.slave: runners.locust_runner = SlaveLocustRunner(locust_classes, options.hatch_rate, options.num_clients, num_requests=options.num_requests, host=options.host, master_host=options.master_host) main_greenlet = runners.locust_runner.greenlet if options.print_stats or (options.no_web and not options.slave): # spawn stats printing greenlet gevent.spawn(stats_printer) try: logger.info("Starting Locust %s" % version) main_greenlet.join() except KeyboardInterrupt, e: events.quitting.fire() time.sleep(0.2) print_stats(runners.locust_runner.request_stats) print_percentile_stats(runners.locust_runner.request_stats) print_error_report() logger.info("Got KeyboardInterrupt. Exiting, bye..")
def print_stats(stats): console_logger.info( (" %-" + str(STATS_NAME_WIDTH) + "s %7s %12s %7s %7s %7s | %7s %7s") % ('Name', '# reqs', '# fails', 'Avg', 'Min', 'Max', 'Median', 'req/s')) console_logger.info("-" * (80 + STATS_NAME_WIDTH)) total_rps = 0 total_reqs = 0 total_failures = 0 for key in sorted(stats.iterkeys()): r = stats[key] total_rps += r.current_rps total_reqs += r.num_requests total_failures += r.num_failures console_logger.info(r) console_logger.info("-" * (80 + STATS_NAME_WIDTH)) try: fail_percent = (total_failures / float(total_reqs + total_failures)) * 100 except ZeroDivisionError: fail_percent = 0 console_logger.info((" %-" + str(STATS_NAME_WIDTH) + "s %7d %12s %42.2f") % ('Total', total_reqs, "%d(%.2f%%)" % (total_failures, fail_percent), total_rps)) console_logger.info("")
def print_percentile_stats(stats): console_logger.info("Percentage of the requests completed within given times") console_logger.info((" %-" + str(STATS_NAME_WIDTH) + "s %8s %6s %6s %6s %6s %6s %6s %6s %6s %6s") % ('Name', '# reqs', '50%', '66%', '75%', '80%', '90%', '95%', '98%', '99%', '100%')) console_logger.info("-" * (80 + STATS_NAME_WIDTH)) complete_list = [] for key in sorted(stats.iterkeys()): r = stats[key] if r.response_times: console_logger.info(r.percentile()) complete_list.extend(r.create_response_times_list()) console_logger.info("-" * (80 + STATS_NAME_WIDTH)) complete_list.sort() if complete_list: console_logger.info( (" %-" + str(STATS_NAME_WIDTH) + "s %8s %6d %6d %6d %6d %6d %6d %6d %6d %6d") % ( 'Total', str(len(complete_list)), percentile(complete_list, 0.5), percentile(complete_list, 0.66), percentile(complete_list, 0.75), percentile(complete_list, 0.8), percentile(complete_list, 0.9), percentile(complete_list, 0.95), percentile(complete_list, 0.98), percentile(complete_list, 0.99), complete_list[-1] )) console_logger.info("")
def main(): parser, options, arguments = parse_options() # setup logging setup_logging(options.loglevel, options.logfile) logger = logging.getLogger(__name__) if options.show_version: print "Locust %s" % (version, ) sys.exit(0) locustfile = find_locustfile(options.locustfile) if not locustfile: logger.error( "Could not find any locustfile! See --help for available options.") sys.exit(1) docstring, locusts = load_locustfile(locustfile) if options.list_commands: print "Available Locusts:" for name in locusts: print " " + name sys.exit(0) if not locusts: logger.error("No Locust class found!") sys.exit(1) # make sure specified Locust exists if arguments: missing = set(arguments) - set(locusts.keys()) if missing: logger.error("Unknown Locust(s): %s\n" % (", ".join(missing))) sys.exit(1) else: names = set(arguments) & set(locusts.keys()) locust_classes = [locusts[n] for n in names] else: locust_classes = locusts.values() if options.show_task_ratio: console_logger.info("\n Task ratio per locust class") console_logger.info("-" * 80) print_task_ratio(locust_classes) console_logger.info("\n Total task ratio") console_logger.info("-" * 80) print_task_ratio(locust_classes, total=True) sys.exit(0) if options.show_task_ratio_json: from json import dumps task_data = { "per_class": get_task_ratio_dict(locust_classes), "total": get_task_ratio_dict(locust_classes, total=True) } console_logger.info(dumps(task_data)) sys.exit(0) # if --master is set, make sure --no-web isn't set if options.master and options.no_web: logger.error( "Locust can not run distributed with the web interface disabled (do not use --no-web and --master together)" ) sys.exit(0) if not options.no_web and not options.slave: # spawn web greenlet logger.info("Starting web monitor on port %s" % options.port) main_greenlet = gevent.spawn(web.start, locust_classes, options.hatch_rate, options.num_clients, options.num_requests, options.ramp, options.port) if not options.master and not options.slave: runners.locust_runner = LocalLocustRunner(locust_classes, options.hatch_rate, options.num_clients, options.num_requests, options.host) # spawn client spawning/hatching greenlet if options.no_web: runners.locust_runner.start_hatching(wait=True) main_greenlet = runners.locust_runner.greenlet elif options.master: runners.locust_runner = MasterLocustRunner( locust_classes, options.hatch_rate, options.num_clients, num_requests=options.num_requests, host=options.host, master_host=options.master_host) elif options.slave: runners.locust_runner = SlaveLocustRunner( locust_classes, options.hatch_rate, options.num_clients, num_requests=options.num_requests, host=options.host, master_host=options.master_host) main_greenlet = runners.locust_runner.greenlet if options.print_stats or (options.no_web and not options.slave): # spawn stats printing greenlet gevent.spawn(stats_printer) def shutdown(code=0): """ Shut down locust by firing quitting event, printing stats and exiting """ logger.info("Shutting down, bye..") events.quitting.fire() print_stats(runners.locust_runner.request_stats) print_percentile_stats(runners.locust_runner.request_stats) print_error_report() sys.exit(code) # install SIGTERM handler def sig_term_handler(): logger.info("Got SIGTERM signal") shutdown(0) gevent.signal(signal.SIGTERM, sig_term_handler) try: logger.info("Starting Locust %s" % version) main_greenlet.join() shutdown(0) except KeyboardInterrupt as e: shutdown(0)
def main(): parser, options, arguments = parse_options() # setup logging setup_logging(options.loglevel, options.logfile) logger = logging.getLogger(__name__) if options.show_version: print "Locust %s" % (version, ) sys.exit(0) if os.path.isdir(options.locustfile): all_locustfiles = collect_locustfiles(options.locustfile) else: locustfile = find_locustfile(options.locustfile) if not locustfile: logger.error( "Could not find any locustfile! Ensure file ends in '.py' and see --help for available options." ) sys.exit(1) all_locustfiles = load_locustfile(locustfile) logger.info("All available locustfiles: {}".format(all_locustfiles)) # Use the first locustfile for the default locusts locusts = all_locustfiles.values()[0] if options.list_commands: console_logger.info("Available Locusts:") for name in locusts: console_logger.info(" " + name) sys.exit(0) if not locusts: logger.error("No Locust class found!") sys.exit(1) # make sure specified Locust exists if arguments: missing = set(arguments) - set(locusts.keys()) if missing: logger.error("Unknown Locust(s): %s\n" % (", ".join(missing))) sys.exit(1) else: names = set(arguments) & set(locusts.keys()) locust_classes = [locusts[n] for n in names] else: locust_classes = locusts.values() if options.show_task_ratio: console_logger.info("\n Task ratio per locust class") console_logger.info("-" * 80) print_task_ratio(locust_classes) console_logger.info("\n Total task ratio") console_logger.info("-" * 80) print_task_ratio(locust_classes, total=True) sys.exit(0) if options.show_task_ratio_json: from json import dumps task_data = { "per_class": get_task_ratio_dict(locust_classes), "total": get_task_ratio_dict(locust_classes, total=True) } console_logger.info(dumps(task_data)) sys.exit(0) if options.master and options.no_web and not options.min_slaves: logger.error( "When running --master and --no-web, you must specify --min-slaves to be available before starting to swarm" ) sys.exit(1) if options.master and options.no_web and not (options.timeout or options.num_requests): logger.error( "When running --master and --no-web, you must specify either --num-request or --timeout to tell the slaves when to stop running each locustfile" ) sys.exit(1) if not options.no_web and not options.slave: # spawn web greenlet logger.info("Starting web monitor at %s:%s" % (options.web_host or "*", options.port)) main_greenlet = gevent.spawn(web.start, locust_classes, options) if options.slave: logger.info("Waiting for master to become available") try: runners.locust_runner = polling.poll(lambda: SlaveLocustRunner( locust_classes, options, available_locustfiles=all_locustfiles ), timeout=60, step=1, ignore_exceptions=( socket.error, )) except polling.TimeoutException, e: logger.error("Failed to connect to the Locust master: %s", e.last) sys.exit(-1) main_greenlet = runners.locust_runner.greenlet