def print_task_ratio(locusts, total=False, level=0, parent_ratio=1.0):
"""
Output table with task execution ratio info to console_logger
"""
ratio = {}
for locust in locusts:
ratio.setdefault(locust, 0)
ratio[locust] += 1
# get percentage
ratio_percent = dict(
map(lambda x: (x[0], float(x[1]) / len(locusts) * parent_ratio),
ratio.iteritems()))
for locust, ratio in ratio_percent.iteritems():
console_logger.info(" %-10s %-50s" %
(" " * level + "%-6.1f" %
(ratio * 100), " " * level + locust.__name__))
if inspect.isclass(locust):
if issubclass(locust, Locust):
if total:
print_task_ratio(locust.task_set.tasks, total, level + 1,
ratio)
else:
print_task_ratio(locust.task_set.tasks, total, level + 1)
elif issubclass(locust, TaskSet):
if total:
print_task_ratio(locust.tasks, total, level + 1, ratio)
else:
print_task_ratio(locust.tasks, total, level + 1)
def _print_task_ratio(x, level=0):
for k, v in x.iteritems():
padding = 2*" "*level
ratio = v.get('ratio', 1)
console_logger.info(" %-10s %-50s" % (padding + "%-6.1f" % (ratio*100), padding + k))
if 'tasks' in v:
_print_task_ratio(v['tasks'], level + 1)
def print_task_ratio(locusts, total=False, level=0, parent_ratio=1.0):
"""
Output table with task execution ratio info to console_logger
"""
ratio = {}
for locust in locusts:
ratio.setdefault(locust, 0)
ratio[locust] += 1
# get percentage
ratio_percent = dict(map(lambda x: (x[0], float(x[1])/len(locusts) * parent_ratio), ratio.iteritems()))
for locust, ratio in ratio_percent.iteritems():
console_logger.info(" %-10s %-50s" % (" "*level + "%-6.1f" % (ratio*100), " "*level + locust.__name__))
if inspect.isclass(locust):
if issubclass(locust, Locust):
if total:
print_task_ratio(locust.task_set.tasks, total, level+1, ratio)
else:
print_task_ratio(locust.task_set.tasks, total, level+1)
elif issubclass(locust, TaskSet):
if total:
print_task_ratio(locust.tasks, total, level+1, ratio)
else:
print_task_ratio(locust.tasks, total, level+1)
def get_final_stats_after_shutdown():
from runners import locust_runner
import os
pkg_version = os.getenv('PKG_VERSION', 'PKG_VERSION_ENV_NOT_FOUND')
ret = {pkg_version: get_all_stats(locust_runner.request_stats)}
console_logger.info("== percentile_stats = {0}".format(ret))
return ret
def local_file_append_df(
df: pd.DataFrame,
local_path: str,
**kw,
):
'''
追加写入本地文件
'''
if not os.path.exists(local_path):
mkdir(os.path.dirname(local_path))
if not os.path.exists(local_path):
df_partial = partial(
df.to_csv,
local_path,
encoding='utf-8-sig',
)
else: # * 文件存在时追加不写header
df_partial = partial(
df.to_csv,
local_path,
mode='a',
header=False,
encoding='utf-8-sig',
)
df_shape = df.shape
if df_shape[0]:
logger.info(f'{df_shape} append to -> {local_path}')
df_partial(**kw)
return
def _print_task_ratio(x, level=0):
for k, v in x.iteritems():
padding = 2 * " " * level
ratio = v.get('ratio', 1)
console_logger.info(" %-10s %-50s" % (padding + "%-6.1f" %
(ratio * 100), padding + k))
if 'tasks' in v:
_print_task_ratio(v['tasks'], level + 1)
def init_clean_file(
local_path: str,
clean_flag=0,
):
'''
清理路径下的文件
'''
try:
if not clean_flag:
logger.info(f'Init Clean File -> {local_path}')
os.remove(local_path)
except Exception:
pass
def write_df_to_csv(
total_df: pd.DataFrame,
filter_df: pd.DataFrame,
folder_name: str,
):
match_result_path = os.path.join(folder_name, 'match_result.csv')
if args.csv and os.path.exists(match_result_path):
logger.info(f'match result saved as -> {match_result_path}')
summary_path = os.path.join(folder_name, 'summary_result.csv')
if len(total_df):
local_file_append_df(total_df, summary_path, index=True)
logger.info(f'summary result saved as -> {summary_path}')
filtered_path = os.path.join(folder_name, 'filter_result.csv')
if len(filter_df):
local_file_append_df(filter_df, filtered_path, index=True)
logger.info(f'filtered result saved as -> {filtered_path}')
def download_single_file(
cos_client: CosS3Client,
bucket: str,
cos_path: str,
local_path: str,
**kw,
):
"""
下载单个文件
"""
if os.path.exists(local_path):
logger.info(f'{cos_path} already exists')
return local_path
logger.info(
f'start download file form bucket -> {bucket}, Path -> {cos_path}')
resp = cos_client.download_file(
Bucket=bucket,
Key=cos_path,
DestFilePath=local_path,
)
result = local_path if resp is None else resp
logger.info(f'success, saved as -> {result}')
return result
def print_error_report():
console_logger.info("Error report")
console_logger.info(" %-18s %-100s" % ("# occurences", "Error"))
console_logger.info("-" * (80 + STATS_NAME_WIDTH))
for error, count in RequestStats.errors.iteritems():
console_logger.info(" %-18i %-100s" % (count, error))
console_logger.info("-" * (80 + STATS_NAME_WIDTH))
console_logger.info("")
def print_stats(stats):
console_logger.info((" %-" + str(STATS_NAME_WIDTH) + "s %7s %12s %7s %7s %7s | %7s %7s") % ('Name', '# reqs', '# fails', 'Avg', 'Min', 'Max', 'Median', 'req/s'))
console_logger.info("-" * (80 + STATS_NAME_WIDTH))
total_rps = 0
total_reqs = 0
total_failures = 0
avg_response_time = []
median_response_time = []
min_response_time = None
max_response_time = None
avg_content_length = []
for key in sorted(stats.iterkeys()):
r = stats[key]
total_rps += r.current_rps
total_reqs += r.num_requests
total_failures += r.num_failures
median_response_time.append(r.median_response_time)
avg_response_time.append(r.avg_response_time)
if min_response_time is None:
min_response_time = r.min_response_time
else:
if r.min_response_time < min_response_time:
min_response_time = r.min_response_time
if max_response_time is None:
max_response_time = r.max_response_time
else:
if r.max_response_time > max_response_time:
max_response_time = r.max_response_time
avg_content_length.append(r.avg_content_length)
console_logger.info(r)
console_logger.info("-" * (80 + STATS_NAME_WIDTH))
try:
fail_percent = (total_failures/float(total_reqs))*100
except ZeroDivisionError:
fail_percent = 0
console_logger.info((" %-" + str(STATS_NAME_WIDTH) + "s %7d %12s %42.2f") % ('Total', total_reqs, "%d(%.2f%%)" % (total_failures, fail_percent), total_rps))
console_logger.info("total_reqs,total_failures,median_response_time,avg_response_time,min_response_time,max_response_time,avg_content_length,total_rps")
console_logger.info("{},{},{},{},{},{},{},{}".format(
total_reqs,
total_failures,
sum(median_response_time)/len(median_response_time),
sum(avg_response_time)/len(avg_response_time),
min_response_time,
max_response_time,
sum(avg_content_length)/len(avg_content_length),
total_rps
))
console_logger.info("")
def main():
config = load_config(args)
bucket = config.get('bucket')
cos_client = get_cos_client(**config)
global FILTER_RESULT, DIFF_RESULT, TOTAL_PREFIX_SET, MANIFEST_DATA
log_tmp_folder = 'log_tmp'
result_folder = 'result'
manifest_folder = 'manifest'
result_folder_name = os.path.join(result_folder, now_time_fmt)
mkdir(log_tmp_folder, result_folder, manifest_folder)
MANIFEST_DATA = get_manifest_data(
cos_client,
bucket,
manifest_folder,
args.manifest,
)
MANIFEST_DATA = get_mainfest_header(MANIFEST_DATA)
cos_file_list, file_schema = MANIFEST_DATA['files'], MANIFEST_DATA[
f'_{FILE_SCHEMA}']
logger.info(f'{len(cos_file_list)} csv files in total')
local_file_path = download_csv_files(
cos_client,
bucket,
cos_file_list,
log_tmp_folder,
)
if not len(local_file_path):
logger.info('Not Found Manifest File')
sys.exit()
logger_debug('The manifest file has been saved.')
prefix_level = -1
if args.prefix_level is not None:
prefix_level = int(args.prefix_level)
index = 0
file_bar = tqdm(
local_file_path,
desc='Manifest File Processing',
unit='file',
leave=False,
)
for _file in file_bar:
logger_debug(f'loading file to df -> {_file}')
dfs = pd.read_csv(
_file,
compression='gzip',
names=file_schema,
chunksize=CHUNK_SIZE,
)
for df in dfs:
# * 对数据进行预处理
filter_df, diff_df = pre_proc_df(df, prefix_level)
del df
filter_r, diff_r = proc_single_csv(
filter_df,
diff_df,
file_schema,
)
# * 写入预处理后的df
FILTER_RESULT = merge_df(FILTER_RESULT, filter_r)
DIFF_RESULT = merge_df(DIFF_RESULT, diff_r)
if args.csv:
match_result_file_name = os.path.join(
result_folder_name, 'match_result.csv') # * 生成的csv文件名
filter_df = filter_df[file_schema]
local_file_append_df(filter_df, match_result_file_name, index)
del filter_df, diff_r
index += 1
total_df, filter_df = get_manifest_result(
FILTER_RESULT,
DIFF_RESULT,
TOTAL_PREFIX_SET,
)
write_df_to_csv(
total_df,
filter_df,
result_folder_name,
)
def print_stats(stats):
console_logger.info(
(" %-" + str(STATS_NAME_WIDTH) + "s %7s %12s %7s %7s %7s | %7s %7s")
% ("Name", "# reqs", "# fails", "Avg", "Min", "Max", "Median", "req/s")
)
console_logger.info("-" * (80 + STATS_NAME_WIDTH))
total_rps = 0
total_reqs = 0
total_failures = 0
for key in sorted(stats.iterkeys()):
r = stats[key]
total_rps += r.current_rps
total_reqs += r.num_requests
total_failures += r.num_failures
console_logger.info(r)
console_logger.info("-" * (80 + STATS_NAME_WIDTH))
try:
fail_percent = (total_failures / float(total_reqs)) * 100
except ZeroDivisionError:
fail_percent = 0
console_logger.info(
(" %-" + str(STATS_NAME_WIDTH) + "s %7d %12s %42.2f")
% ("Total", total_reqs, "%d(%.2f%%)" % (total_failures, fail_percent), total_rps)
)
console_logger.info("")
def main(args=None):
parser, options, arguments = parse_options(args)
# setup logging
setup_logging(options.loglevel, options.logfile)
logger = logging.getLogger(__name__)
if options.show_version:
print "Locust %s" % (version,)
sys.exit(0)
locustfile = find_locustfile(options.locustfile)
if not locustfile:
logger.error("Could not find any locustfile! Ensure file ends in '.py' and see --help for available options.")
sys.exit(1)
docstring, locusts = load_locustfile(locustfile)
if options.list_commands:
console_logger.info("Available Locusts:")
for name in locusts:
console_logger.info(" " + name)
sys.exit(0)
if not locusts:
logger.error("No Locust class found!")
sys.exit(1)
# make sure specified Locust exists
if arguments:
missing = set(arguments) - set(locusts.keys())
if missing:
logger.error("Unknown Locust(s): %s\n" % (", ".join(missing)))
sys.exit(1)
else:
names = set(arguments) & set(locusts.keys())
locust_classes = [locusts[n] for n in names]
else:
locust_classes = locusts.values()
if options.show_task_ratio:
console_logger.info("\n Task ratio per locust class")
console_logger.info( "-" * 80)
print_task_ratio(locust_classes)
console_logger.info("\n Total task ratio")
console_logger.info("-" * 80)
print_task_ratio(locust_classes, total=True)
sys.exit(0)
if options.show_task_ratio_json:
from json import dumps
task_data = {
"per_class": get_task_ratio_dict(locust_classes),
"total": get_task_ratio_dict(locust_classes, total=True)
}
console_logger.info(dumps(task_data))
sys.exit(0)
# if --master is set, make sure --no-web isn't set
if options.master and options.no_web:
logger.error("Locust can not run distributed with the web interface disabled (do not use --no-web and --master together)")
sys.exit(0)
if not options.no_web and not options.slave:
# spawn web greenlet
logger.info("Starting web monitor at %s:%s" % (options.web_host or "*", options.port))
main_greenlet = gevent.spawn(web.start, locust_classes, options)
if not options.master and not options.slave:
runners.locust_runner = LocalLocustRunner(locust_classes, options)
# spawn client spawning/hatching greenlet
if options.no_web:
runners.locust_runner.start_hatching(wait=True)
main_greenlet = runners.locust_runner.greenlet
elif options.master:
runners.locust_runner = MasterLocustRunner(locust_classes, options)
elif options.slave:
try:
runners.locust_runner = SlaveLocustRunner(locust_classes, options)
main_greenlet = runners.locust_runner.greenlet
except socket.error, e:
logger.error("Failed to connect to the Locust master: %s", e)
sys.exit(-1)
def print_percentile_stats(stats):
console_logger.info(
"Percentage of the requests completed within given times")
console_logger.info((" %-" + str(STATS_NAME_WIDTH) +
"s %8s %6s %6s %6s %6s %6s %6s %6s %6s %6s") %
('Name', '# reqs', '50%', '66%', '75%', '80%', '90%',
'95%', '98%', '99%', '100%'))
console_logger.info("-" * (80 + STATS_NAME_WIDTH))
for key in sorted(stats.iterkeys()):
r = stats[key]
if r.response_times:
console_logger.info(r.percentile())
console_logger.info("-" * (80 + STATS_NAME_WIDTH))
total_stats = global_stats.aggregated_stats()
if total_stats.response_times:
console_logger.info(total_stats.percentile())
console_logger.info("")
def main():
parser, options, arguments = parse_options()
# setup logging
setup_logging(options.loglevel, options.logfile)
logger = logging.getLogger(__name__)
if options.show_version:
print "Locust %s" % (version,)
sys.exit(0)
locustfile = find_locustfile(options.locustfile)
if not locustfile:
logger.error("Could not find any locustfile! See --help for available options.")
sys.exit(1)
docstring, locusts = load_locustfile(locustfile)
if options.list_commands:
print "Available Locusts:"
for name in locusts:
print " " + name
sys.exit(0)
if not locusts:
logger.error("No Locust class found!")
sys.exit(1)
# make sure specified Locust exists
if arguments:
missing = set(arguments) - set(locusts.keys())
if missing:
logger.error("Unknown Locust(s): %s\n" % (", ".join(missing)))
sys.exit(1)
else:
names = set(arguments) & set(locusts.keys())
locust_classes = [locusts[n] for n in names]
else:
locust_classes = locusts.values()
if options.show_task_ratio:
console_logger.info("\n Task ratio per locust class")
console_logger.info("-" * 80)
print_task_ratio(locust_classes)
console_logger.info("\n Total task ratio")
console_logger.info("-" * 80)
print_task_ratio(locust_classes, total=True)
sys.exit(0)
if options.show_task_ratio_json:
from json import dumps
task_data = {
"per_class": get_task_ratio_dict(locust_classes),
"total": get_task_ratio_dict(locust_classes, total=True),
}
console_logger.info(dumps(task_data))
sys.exit(0)
# if --master is set, make sure --no-web isn't set
if options.master and options.no_web:
logger.error(
"Locust can not run distributed with the web interface disabled (do not use --no-web and --master together)"
)
sys.exit(0)
if not options.no_web and not options.slave:
# spawn web greenlet
logger.info("Starting web monitor on port %s" % options.port)
main_greenlet = gevent.spawn(
web.start,
locust_classes,
options.hatch_rate,
options.num_clients,
options.num_requests,
options.ramp,
options.port,
)
if not options.master and not options.slave:
runners.locust_runner = LocalLocustRunner(
locust_classes, options.hatch_rate, options.num_clients, options.num_requests, options.host
)
# spawn client spawning/hatching greenlet
if options.no_web:
runners.locust_runner.start_hatching(wait=True)
main_greenlet = runners.locust_runner.greenlet
elif options.master:
runners.locust_runner = MasterLocustRunner(
locust_classes,
options.hatch_rate,
options.num_clients,
num_requests=options.num_requests,
host=options.host,
master_host=options.master_host,
)
elif options.slave:
runners.locust_runner = SlaveLocustRunner(
locust_classes,
options.hatch_rate,
options.num_clients,
num_requests=options.num_requests,
host=options.host,
master_host=options.master_host,
)
main_greenlet = runners.locust_runner.greenlet
if options.print_stats or (options.no_web and not options.slave):
# spawn stats printing greenlet
gevent.spawn(stats_printer)
def shutdown(code=0):
"""
Shut down locust by firing quitting event, printing stats and exiting
"""
logger.info("Shutting down, bye..")
events.quitting.fire()
print_stats(runners.locust_runner.request_stats)
print_percentile_stats(runners.locust_runner.request_stats)
print_error_report()
sys.exit(code)
# install SIGTERM handler
def sig_term_handler():
logger.info("Got SIGTERM signal")
shutdown(0)
gevent.signal(signal.SIGTERM, sig_term_handler)
try:
logger.info("Starting Locust %s" % version)
main_greenlet.join()
shutdown(0)
except KeyboardInterrupt, e:
shutdown(0)
def print_total_stats(stats):
console_logger.info(
(" %-" + str(STATS_NAME_WIDTH) + "s %7s %12s %7s %7s %7s | %7s %7s") %
('Name', '# reqs', '# fails', 'Avg', 'Min', 'Max', 'Median', 'req/s'))
console_logger.info("-" * (80 + STATS_NAME_WIDTH))
current_rps = 0
total_rps = 0
total_reqs = 0
total_failures = 0
for key in sorted(stats.iterkeys()):
r = stats[key]
current_rps += r.current_rps
total_rps += r.total_rps
total_reqs += r.num_requests
total_failures += r.num_failures
console_logger.info(r)
console_logger.info("-" * (80 + STATS_NAME_WIDTH))
try:
fail_percent = (total_failures/float(total_reqs))*100
except ZeroDivisionError:
fail_percent = 0
console_logger.info(
(" %-" + str(STATS_NAME_WIDTH) + "s %7d %12s %42.2f") % ('Current', total_reqs, "%d(%.2f%%)" %
(total_failures, fail_percent), current_rps))
console_logger.info(
(" %-" + str(STATS_NAME_WIDTH) + "s %7d %12s %42.2f") % ('Total', total_reqs, "%d(%.2f%%)" %
(total_failures, fail_percent), total_rps))
console_logger.info("")
def print_percentile_stats(stats):
console_logger.info("Percentage of the requests completed within given times")
console_logger.info((" %-" + str(STATS_NAME_WIDTH) + "s %8s %6s %6s %6s %6s %6s %6s %6s %6s %6s") % ('Name', '# reqs', '50%', '66%', '75%', '80%', '90%', '95%', '98%', '99%', '100%'))
console_logger.info("-" * (80 + STATS_NAME_WIDTH))
for key in sorted(stats.iterkeys()):
r = stats[key]
if r.response_times:
console_logger.info(r.percentile())
console_logger.info("-" * (80 + STATS_NAME_WIDTH))
total_stats = global_stats.aggregated_stats()
if total_stats.response_times:
console_logger.info(total_stats.percentile())
console_logger.info("")
def print_percentile_stats(stats):
console_logger.info("Percentage of the requests completed within given times")
console_logger.info(
(" %-" + str(STATS_NAME_WIDTH) + "s %8s %6s %6s %6s %6s %6s %6s %6s %6s %6s")
% ("Name", "# reqs", "50%", "66%", "75%", "80%", "90%", "95%", "98%", "99%", "100%")
)
console_logger.info("-" * (80 + STATS_NAME_WIDTH))
for key in sorted(stats.iterkeys()):
r = stats[key]
if r.response_times:
console_logger.info(r.percentile())
console_logger.info("-" * (80 + STATS_NAME_WIDTH))
total_stats = global_stats.aggregated_stats()
if total_stats.response_times:
console_logger.info(total_stats.percentile())
console_logger.info("")
def main():
parser, options, arguments = parse_options()
#print "Options:", options, dir(options)
#print "Arguments:", arguments
#print "largs:", parser.largs
#print "rargs:", parser.rargs
# setup logging
setup_logging(options.loglevel, options.logfile)
logger = logging.getLogger(__name__)
if options.show_version:
print "Locust %s" % (version)
sys.exit(0)
locustfile = find_locustfile(options.locustfile)
if not locustfile:
logger.error(
"Could not find any locustfile! See --help for available options.")
sys.exit(1)
docstring, locusts = load_locustfile(locustfile)
if options.list_commands:
print "Available Locusts:"
for name in locusts:
print " " + name
sys.exit(0)
if not locusts:
logger.error("No Locust class found!")
sys.exit(1)
# make sure specified Locust exists
if arguments:
missing = set(arguments) - set(locusts.keys())
if missing:
logger.error("Unknown Locust(s): %s\n" % (", ".join(missing)))
sys.exit(1)
else:
names = set(arguments) & set(locusts.keys())
locust_classes = [locusts[n] for n in names]
else:
locust_classes = locusts.values()
if options.show_task_ratio:
console_logger.info("\n Task ratio per locust class")
console_logger.info("-" * 80)
print_task_ratio(locust_classes)
console_logger.info("\n Total task ratio")
console_logger.info("-" * 80)
print_task_ratio(locust_classes, total=True)
sys.exit(0)
if options.show_task_ratio_json:
from json import dumps
task_data = {
"per_class": get_task_ratio_dict(locust_classes),
"total": get_task_ratio_dict(locust_classes, total=True)
}
console_logger.info(dumps(task_data))
sys.exit(0)
# if --master is set, make sure --no-web isn't set
if options.master and options.no_web:
logger.error(
"Locust can not run distributed with the web interface disabled (do not use --no-web and --master together)"
)
sys.exit(0)
if not options.no_web and not options.slave:
# spawn web greenlet
logger.info("Starting web monitor on port 8089")
main_greenlet = gevent.spawn(web.start, locust_classes,
options.hatch_rate, options.num_clients,
options.num_requests, options.ramp)
# enable/disable gzip in WebLocust's HTTP client
WebLocust.gzip = options.gzip
if not options.master and not options.slave:
runners.locust_runner = LocalLocustRunner(locust_classes,
options.hatch_rate,
options.num_clients,
options.num_requests,
options.host)
# spawn client spawning/hatching greenlet
if options.no_web:
runners.locust_runner.start_hatching(wait=True)
main_greenlet = runners.locust_runner.greenlet
elif options.master:
runners.locust_runner = MasterLocustRunner(
locust_classes,
options.hatch_rate,
options.num_clients,
num_requests=options.num_requests,
host=options.host,
master_host=options.master_host)
elif options.slave:
runners.locust_runner = SlaveLocustRunner(
locust_classes,
options.hatch_rate,
options.num_clients,
num_requests=options.num_requests,
host=options.host,
master_host=options.master_host)
main_greenlet = runners.locust_runner.greenlet
if options.ramp:
import rampstats
from rampstats import on_request_success, on_report_to_master, on_slave_report
if options.slave:
events.report_to_master += on_report_to_master
if options.master:
events.slave_report += on_slave_report
else:
events.request_success += on_request_success
if options.print_stats or (options.no_web and not options.slave):
# spawn stats printing greenlet
gevent.spawn(stats_printer)
try:
logger.info("Starting Locust %s" % version)
main_greenlet.join()
except KeyboardInterrupt, e:
events.quitting.fire()
time.sleep(0.2)
print_stats(runners.locust_runner.request_stats)
print_percentile_stats(runners.locust_runner.request_stats)
print_error_report()
logger.info("Got KeyboardInterrupt. Exiting, bye..")
def print_error_report():
if not len(global_stats.errors):
return
console_logger.info("Error report")
console_logger.info(" %-18s %-100s" % ("# occurences", "Error"))
console_logger.info("-" * (80 + STATS_NAME_WIDTH))
for error in global_stats.errors.itervalues():
console_logger.info(" %-18i %-100s" % (error.occurences, error.to_name()))
console_logger.info("-" * (80 + STATS_NAME_WIDTH))
console_logger.info("")
def main():
parser, options, arguments = parse_options()
# setup logging
setup_logging(options.loglevel, options.logfile)
logger = logging.getLogger(__name__)
if options.show_version:
print "Locust %s" % (version,)
sys.exit(0)
if os.path.isdir(options.locustfile):
all_locustfiles = collect_locustfiles(options.locustfile)
else:
locustfile = find_locustfile(options.locustfile)
if not locustfile:
logger.error("Could not find any locustfile! Ensure file ends in '.py' and see --help for available options.")
sys.exit(1)
all_locustfiles = load_locustfile(locustfile)
logger.info("All available locustfiles: {}".format(all_locustfiles))
# Use the first locustfile for the default locusts
locusts = all_locustfiles.values()[0]
if options.list_commands:
console_logger.info("Available Locusts:")
for name in locusts:
console_logger.info(" " + name)
sys.exit(0)
if not locusts:
logger.error("No Locust class found!")
sys.exit(1)
# make sure specified Locust exists
if arguments:
missing = set(arguments) - set(locusts.keys())
if missing:
logger.error("Unknown Locust(s): %s\n" % (", ".join(missing)))
sys.exit(1)
else:
names = set(arguments) & set(locusts.keys())
locust_classes = [locusts[n] for n in names]
else:
locust_classes = locusts.values()
if options.show_task_ratio:
console_logger.info("\n Task ratio per locust class")
console_logger.info( "-" * 80)
print_task_ratio(locust_classes)
console_logger.info("\n Total task ratio")
console_logger.info("-" * 80)
print_task_ratio(locust_classes, total=True)
sys.exit(0)
if options.show_task_ratio_json:
from json import dumps
task_data = {
"per_class": get_task_ratio_dict(locust_classes),
"total": get_task_ratio_dict(locust_classes, total=True)
}
console_logger.info(dumps(task_data))
sys.exit(0)
if options.master and options.no_web and not options.min_slaves:
logger.error("When running --master and --no-web, you must specify --min-slaves to be available before starting to swarm")
sys.exit(1)
if options.master and options.no_web and not (options.timeout or options.num_requests):
logger.error("When running --master and --no-web, you must specify either --num-request or --timeout to tell the slaves when to stop running each locustfile")
sys.exit(1)
if not options.no_web and not options.slave:
# spawn web greenlet
logger.info("Starting web monitor at %s:%s" % (options.web_host or "*", options.port))
main_greenlet = gevent.spawn(web.start, locust_classes, options)
if options.slave:
logger.info("Waiting for master to become available")
try:
runners.locust_runner = polling.poll(
lambda: SlaveLocustRunner(locust_classes, options, available_locustfiles=all_locustfiles),
timeout=60,
step=1,
ignore_exceptions=(socket.error,))
except polling.TimeoutException, e:
logger.error("Failed to connect to the Locust master: %s", e.last)
sys.exit(-1)
main_greenlet = runners.locust_runner.greenlet
def main():
parser, options, arguments = parse_options()
# setup logging
setup_logging(options.loglevel, options.logfile)
logger = logging.getLogger(__name__)
if options.show_version:
print "Locust %s" % (version, )
sys.exit(0)
locustfile = find_locustfile(options.locustfile)
if not locustfile:
logger.error(
"Could not find any locustfile! Ensure file ends in '.py' and see --help for available options."
)
sys.exit(1)
docstring, locusts = load_locustfile(locustfile)
if options.list_commands:
console_logger.info("Available Locusts:")
for name in locusts:
console_logger.info(" " + name)
sys.exit(0)
if not locusts:
logger.error("No Locust class found!")
sys.exit(1)
# make sure specified Locust exists
if arguments:
missing = set(arguments) - set(locusts.keys())
if missing:
logger.error("Unknown Locust(s): %s\n" % (", ".join(missing)))
sys.exit(1)
else:
names = set(arguments) & set(locusts.keys())
locust_classes = [locusts[n] for n in names]
else:
locust_classes = locusts.values()
if options.show_task_ratio:
console_logger.info("\n Task ratio per locust class")
console_logger.info("-" * 80)
print_task_ratio(locust_classes)
console_logger.info("\n Total task ratio")
console_logger.info("-" * 80)
print_task_ratio(locust_classes, total=True)
sys.exit(0)
if options.show_task_ratio_json:
from json import dumps
task_data = {
"per_class": get_task_ratio_dict(locust_classes),
"total": get_task_ratio_dict(locust_classes, total=True)
}
console_logger.info(dumps(task_data))
sys.exit(0)
# if --master is set, make sure --no-web isn't set
if options.master and options.no_web:
logger.error(
"Locust can not run distributed with the web interface disabled (do not use --no-web and --master together)"
)
sys.exit(0)
if not options.no_web and not options.slave:
# spawn web greenlet
logger.info("Starting web monitor at %s:%s" %
(options.web_host or "*", options.port))
main_greenlet = gevent.spawn(web.start, locust_classes, options)
if not options.master and not options.slave:
runners.locust_runner = LocalLocustRunner(locust_classes, options)
# spawn client spawning/hatching greenlet
if options.no_web:
runners.locust_runner.start_hatching(wait=True)
main_greenlet = runners.locust_runner.greenlet
elif options.master:
runners.locust_runner = MasterLocustRunner(locust_classes, options)
elif options.slave:
try:
runners.locust_runner = SlaveLocustRunner(locust_classes, options)
main_greenlet = runners.locust_runner.greenlet
except socket.error, e:
logger.error("Failed to connect to the Locust master: %s", e)
sys.exit(-1)
def main():
parser, options, arguments = parse_options()
#print "Options:", options, dir(options)
#print "Arguments:", arguments
#print "largs:", parser.largs
#print "rargs:", parser.rargs
# setup logging
setup_logging(options.loglevel, options.logfile)
logger = logging.getLogger(__name__)
if options.show_version:
print "Locust %s" % (version)
sys.exit(0)
locustfile = find_locustfile(options.locustfile)
if not locustfile:
logger.error("Could not find any locustfile! See --help for available options.")
sys.exit(1)
docstring, locusts = load_locustfile(locustfile)
if options.list_commands:
print "Available Locusts:"
for name in locusts:
print " " + name
sys.exit(0)
if not locusts:
logger.error("No Locust class found!")
sys.exit(1)
# make sure specified Locust exists
if arguments:
missing = set(arguments) - set(locusts.keys())
if missing:
logger.error("Unknown Locust(s): %s\n" % (", ".join(missing)))
sys.exit(1)
else:
names = set(arguments) & set(locusts.keys())
locust_classes = [locusts[n] for n in names]
else:
locust_classes = locusts.values()
if options.show_task_ratio:
console_logger.info("\n Task ratio per locust class")
console_logger.info( "-" * 80)
print_task_ratio(locust_classes)
console_logger.info("\n Total task ratio")
console_logger.info("-" * 80)
print_task_ratio(locust_classes, total=True)
sys.exit(0)
if options.show_task_ratio_json:
from json import dumps
task_data = {
"per_class": get_task_ratio_dict(locust_classes),
"total": get_task_ratio_dict(locust_classes, total=True)
}
console_logger.info(dumps(task_data))
sys.exit(0)
# if --master is set, make sure --no-web isn't set
if options.master and options.no_web:
logger.error("Locust can not run distributed with the web interface disabled (do not use --no-web and --master together)")
sys.exit(0)
if not options.no_web and not options.slave:
# spawn web greenlet
logger.info("Starting web monitor on port 8089")
main_greenlet = gevent.spawn(web.start, locust_classes, options.hatch_rate, options.num_clients, options.num_requests, options.ramp)
# enable/disable gzip in WebLocust's HTTP client
WebLocust.gzip = options.gzip
if not options.master and not options.slave:
runners.locust_runner = LocalLocustRunner(locust_classes, options.hatch_rate, options.num_clients, options.num_requests, options.host)
# spawn client spawning/hatching greenlet
if options.no_web:
runners.locust_runner.start_hatching(wait=True)
main_greenlet = runners.locust_runner.greenlet
elif options.master:
runners.locust_runner = MasterLocustRunner(locust_classes, options.hatch_rate, options.num_clients, num_requests=options.num_requests, host=options.host, master_host=options.master_host)
elif options.slave:
runners.locust_runner = SlaveLocustRunner(locust_classes, options.hatch_rate, options.num_clients, num_requests=options.num_requests, host=options.host, master_host=options.master_host)
main_greenlet = runners.locust_runner.greenlet
if options.print_stats or (options.no_web and not options.slave):
# spawn stats printing greenlet
gevent.spawn(stats_printer)
try:
logger.info("Starting Locust %s" % version)
main_greenlet.join()
except KeyboardInterrupt, e:
events.quitting.fire()
time.sleep(0.2)
print_stats(runners.locust_runner.request_stats)
print_percentile_stats(runners.locust_runner.request_stats)
print_error_report()
logger.info("Got KeyboardInterrupt. Exiting, bye..")
def print_stats(stats):
console_logger.info(
(" %-" + str(STATS_NAME_WIDTH) + "s %7s %12s %7s %7s %7s | %7s %7s") %
('Name', '# reqs', '# fails', 'Avg', 'Min', 'Max', 'Median', 'req/s'))
console_logger.info("-" * (80 + STATS_NAME_WIDTH))
total_rps = 0
total_reqs = 0
total_failures = 0
for key in sorted(stats.iterkeys()):
r = stats[key]
total_rps += r.current_rps
total_reqs += r.num_requests
total_failures += r.num_failures
console_logger.info(r)
console_logger.info("-" * (80 + STATS_NAME_WIDTH))
try:
fail_percent = (total_failures /
float(total_reqs + total_failures)) * 100
except ZeroDivisionError:
fail_percent = 0
console_logger.info((" %-" + str(STATS_NAME_WIDTH) + "s %7d %12s %42.2f") %
('Total', total_reqs, "%d(%.2f%%)" %
(total_failures, fail_percent), total_rps))
console_logger.info("")
def print_percentile_stats(stats):
console_logger.info("Percentage of the requests completed within given times")
console_logger.info((" %-" + str(STATS_NAME_WIDTH) + "s %8s %6s %6s %6s %6s %6s %6s %6s %6s %6s") % ('Name', '# reqs', '50%', '66%', '75%', '80%', '90%', '95%', '98%', '99%', '100%'))
console_logger.info("-" * (80 + STATS_NAME_WIDTH))
complete_list = []
for key in sorted(stats.iterkeys()):
r = stats[key]
if r.response_times:
console_logger.info(r.percentile())
complete_list.extend(r.create_response_times_list())
console_logger.info("-" * (80 + STATS_NAME_WIDTH))
complete_list.sort()
if complete_list:
console_logger.info( (" %-" + str(STATS_NAME_WIDTH) + "s %8s %6d %6d %6d %6d %6d %6d %6d %6d %6d") % (
'Total',
str(len(complete_list)),
percentile(complete_list, 0.5),
percentile(complete_list, 0.66),
percentile(complete_list, 0.75),
percentile(complete_list, 0.8),
percentile(complete_list, 0.9),
percentile(complete_list, 0.95),
percentile(complete_list, 0.98),
percentile(complete_list, 0.99),
complete_list[-1]
))
console_logger.info("")
def print_error_report():
if not len(global_stats.errors):
return
console_logger.info("Error report")
console_logger.info(" %-18s %-100s" % ("# occurences", "Error"))
console_logger.info("-" * (80 + STATS_NAME_WIDTH))
for error in global_stats.errors.itervalues():
console_logger.info(" %-18i %-100s" %
(error.occurences, error.to_name()))
console_logger.info("-" * (80 + STATS_NAME_WIDTH))
console_logger.info("")
def print_error_report():
console_logger.info("Error report")
console_logger.info(" %-18s %-100s" % ("# occurences", "Error"))
console_logger.info("-" * (80 + STATS_NAME_WIDTH))
for error, count in RequestStats.errors.iteritems():
console_logger.info(" %-18i %-100s" % (count, error))
console_logger.info("-" * (80 + STATS_NAME_WIDTH))
console_logger.info("")
def main():
parser, options, arguments = parse_options()
# setup logging
setup_logging(options.loglevel, options.logfile)
logger = logging.getLogger(__name__)
if options.show_version:
print "Locust %s" % (version, )
sys.exit(0)
locustfile = find_locustfile(options.locustfile)
if not locustfile:
logger.error(
"Could not find any locustfile! See --help for available options.")
sys.exit(1)
docstring, locusts = load_locustfile(locustfile)
if options.list_commands:
print "Available Locusts:"
for name in locusts:
print " " + name
sys.exit(0)
if not locusts:
logger.error("No Locust class found!")
sys.exit(1)
# make sure specified Locust exists
if arguments:
missing = set(arguments) - set(locusts.keys())
if missing:
logger.error("Unknown Locust(s): %s\n" % (", ".join(missing)))
sys.exit(1)
else:
names = set(arguments) & set(locusts.keys())
locust_classes = [locusts[n] for n in names]
else:
locust_classes = locusts.values()
if options.show_task_ratio:
console_logger.info("\n Task ratio per locust class")
console_logger.info("-" * 80)
print_task_ratio(locust_classes)
console_logger.info("\n Total task ratio")
console_logger.info("-" * 80)
print_task_ratio(locust_classes, total=True)
sys.exit(0)
if options.show_task_ratio_json:
from json import dumps
task_data = {
"per_class": get_task_ratio_dict(locust_classes),
"total": get_task_ratio_dict(locust_classes, total=True)
}
console_logger.info(dumps(task_data))
sys.exit(0)
# if --master is set, make sure --no-web isn't set
if options.master and options.no_web:
logger.error(
"Locust can not run distributed with the web interface disabled (do not use --no-web and --master together)"
)
sys.exit(0)
if not options.no_web and not options.slave:
# spawn web greenlet
logger.info("Starting web monitor on port %s" % options.port)
main_greenlet = gevent.spawn(web.start, locust_classes,
options.hatch_rate, options.num_clients,
options.num_requests, options.ramp,
options.port)
if not options.master and not options.slave:
runners.locust_runner = LocalLocustRunner(locust_classes,
options.hatch_rate,
options.num_clients,
options.num_requests,
options.host)
# spawn client spawning/hatching greenlet
if options.no_web:
runners.locust_runner.start_hatching(wait=True)
main_greenlet = runners.locust_runner.greenlet
elif options.master:
runners.locust_runner = MasterLocustRunner(
locust_classes,
options.hatch_rate,
options.num_clients,
num_requests=options.num_requests,
host=options.host,
master_host=options.master_host)
elif options.slave:
runners.locust_runner = SlaveLocustRunner(
locust_classes,
options.hatch_rate,
options.num_clients,
num_requests=options.num_requests,
host=options.host,
master_host=options.master_host)
main_greenlet = runners.locust_runner.greenlet
if options.print_stats or (options.no_web and not options.slave):
# spawn stats printing greenlet
gevent.spawn(stats_printer)
def shutdown(code=0):
"""
Shut down locust by firing quitting event, printing stats and exiting
"""
logger.info("Shutting down, bye..")
events.quitting.fire()
print_stats(runners.locust_runner.request_stats)
print_percentile_stats(runners.locust_runner.request_stats)
print_error_report()
sys.exit(code)
# install SIGTERM handler
def sig_term_handler():
logger.info("Got SIGTERM signal")
shutdown(0)
gevent.signal(signal.SIGTERM, sig_term_handler)
try:
logger.info("Starting Locust %s" % version)
main_greenlet.join()
shutdown(0)
except KeyboardInterrupt as e:
shutdown(0)
def main():
parser, options, arguments = parse_options()
# setup logging
setup_logging(options.loglevel, options.logfile)
logger = logging.getLogger(__name__)
if options.show_version:
print "Locust %s" % (version, )
sys.exit(0)
if os.path.isdir(options.locustfile):
all_locustfiles = collect_locustfiles(options.locustfile)
else:
locustfile = find_locustfile(options.locustfile)
if not locustfile:
logger.error(
"Could not find any locustfile! Ensure file ends in '.py' and see --help for available options."
)
sys.exit(1)
all_locustfiles = load_locustfile(locustfile)
logger.info("All available locustfiles: {}".format(all_locustfiles))
# Use the first locustfile for the default locusts
locusts = all_locustfiles.values()[0]
if options.list_commands:
console_logger.info("Available Locusts:")
for name in locusts:
console_logger.info(" " + name)
sys.exit(0)
if not locusts:
logger.error("No Locust class found!")
sys.exit(1)
# make sure specified Locust exists
if arguments:
missing = set(arguments) - set(locusts.keys())
if missing:
logger.error("Unknown Locust(s): %s\n" % (", ".join(missing)))
sys.exit(1)
else:
names = set(arguments) & set(locusts.keys())
locust_classes = [locusts[n] for n in names]
else:
locust_classes = locusts.values()
if options.show_task_ratio:
console_logger.info("\n Task ratio per locust class")
console_logger.info("-" * 80)
print_task_ratio(locust_classes)
console_logger.info("\n Total task ratio")
console_logger.info("-" * 80)
print_task_ratio(locust_classes, total=True)
sys.exit(0)
if options.show_task_ratio_json:
from json import dumps
task_data = {
"per_class": get_task_ratio_dict(locust_classes),
"total": get_task_ratio_dict(locust_classes, total=True)
}
console_logger.info(dumps(task_data))
sys.exit(0)
if options.master and options.no_web and not options.min_slaves:
logger.error(
"When running --master and --no-web, you must specify --min-slaves to be available before starting to swarm"
)
sys.exit(1)
if options.master and options.no_web and not (options.timeout
or options.num_requests):
logger.error(
"When running --master and --no-web, you must specify either --num-request or --timeout to tell the slaves when to stop running each locustfile"
)
sys.exit(1)
if not options.no_web and not options.slave:
# spawn web greenlet
logger.info("Starting web monitor at %s:%s" %
(options.web_host or "*", options.port))
main_greenlet = gevent.spawn(web.start, locust_classes, options)
if options.slave:
logger.info("Waiting for master to become available")
try:
runners.locust_runner = polling.poll(lambda: SlaveLocustRunner(
locust_classes, options, available_locustfiles=all_locustfiles
),
timeout=60,
step=1,
ignore_exceptions=(
socket.error, ))
except polling.TimeoutException, e:
logger.error("Failed to connect to the Locust master: %s", e.last)
sys.exit(-1)
main_greenlet = runners.locust_runner.greenlet
