def run(settings):
"""
Run the load test programmatically.
"""
env = Environment(
user_classes=[getattr(main, settings["persona_arg"])],
host=settings["base_url"],
)
env.create_local_runner()
# start a greenlet that saves current stats to history
gevent.spawn(stats_history, env.runner)
# start the load test
env.runner.start(user_count=int(settings["users"]),
spawn_rate=int(settings["users"]))
# add a listener for failures
env.events.request_failure.add_listener(main.failure_handler)
# stop the runner at the end
duration_s = parse_timespan(settings["duration"])
gevent.spawn_later(duration_s, lambda: stop(env))
# wait for the greenlets
env.runner.greenlet.join()
def swarm():
if environment.runner.state not in (runners.STATE_STOPPED,runners.STATE_INIT):
return jsonify({'success': False,'message': '当前有任务正在执行，先停止测试再尝试'})
if not environment.runner.worker_count:
return jsonify({'success': False,'message': '没有可用的work, 不能运行测试'})
assert request.method == "POST"
# 清理下统计信息
environment.runner.stats.clear_all()
environment.runner.exceptions = {}
# 开启协程写入指标历史记录
self.reporter_running_status=True
gevent.spawn(stats_history,self)
# 开始压测任务
user_count = int(request.form["user_count"])
hatch_rate = float(request.form["hatch_rate"])
run_seconds=None
if request.form["run_time"]:
try:
run_seconds = parse_timespan(request.form["run_time"])
except ValueError:
pass
if request.form.get("host"):
environment.host = str(request.form["host"])
def stopRunAfterSecs(x):
if x>3600*6:
x=3600*6
count=0
while count<=x: # 这种方式避免长时间休眠造成问题
gevent.sleep(1)
count+=1
self.reporter_running_status = False # 结束指标历史记录
environment.runner.stop()
if run_seconds and run_seconds>=30:
gevent.spawn(stopRunAfterSecs,run_seconds)
step_user_count=None
step_duration=None
try:
step_user_count = int(request.form["step_user_count"])
step_duration = parse_timespan(str(request.form["step_duration"]))
except:
pass
if environment.step_load and step_user_count and step_duration:
environment.runner.start_stepload(user_count,hatch_rate,step_user_count,step_duration)
return jsonify(
{'success': True,'message': 'Swarming started in Step Load Mode','host': environment.host})
environment.runner.start(user_count,hatch_rate)
return jsonify({'success': True,'message': 'Swarming started','host': environment.host})
def set_run_time_in_sec(self, run_time_str):
try:
self.run_time_in_sec = parse_timespan(run_time_str)
except ValueError:
logger.error(
"Invalid format for `run_time` parameter: '%s', "
"Valid formats are: 20s, 3m, 2h, 1h20m, 3h30m10s, etc." %
run_time_str)
sys.exit(1)
except TypeError:
logger.error(
"`run_time` must be a string, not %s. Received value: % " %
(type(run_time_str), run_time_str))
sys.exit(1)
def swarm():
assert request.method == "POST"
user_count = int(request.form["user_count"])
spawn_rate = float(request.form["spawn_rate"])
if request.form.get("host"):
environment.host = str(request.form["host"])
if environment.step_load:
step_user_count = int(request.form["step_user_count"])
step_duration = parse_timespan(str(request.form["step_duration"]))
environment.runner.start_stepload(user_count, spawn_rate, step_user_count, step_duration)
return jsonify({'success': True, 'message': 'Swarming started in Step Load Mode', 'host': environment.host})
if environment.shape_class:
environment.runner.start_shape()
return jsonify({'success': True, 'message': 'Swarming started using shape class', 'host': environment.host})
environment.runner.start(user_count, spawn_rate)
return jsonify({'success': True, 'message': 'Swarming started', 'host': environment.host})
def run(self):
'''
Run the load test.
'''
if self.settings.run_time:
try:
self.settings.run_time = parse_timespan(self.settings.run_time)
except ValueError:
logger.error("Valid --run-time formats are:")
logger.error("20, 20s, 3m, 2h, 1h20m, 3h30m10s, etc.")
sys.exit(1)
self.run_time = self.settings.run_time
logger.info("Run time limit set to %s seconds" % self.run_time)
def timelimit_stop():
logger.info("Time limit reached. Stopping Locust.")
logger.info(json.dumps(self.stats()))
logger.info("Run time limit reached: {0} seconds".format(
self.run_time))
runners.locust_runner.quit()
gevent.spawn_later(self.run_time, timelimit_stop)
try:
logger.info("Starting Locust with settings {0}".format(
vars(self.settings)))
runners.locust_runner = runners.LocalLocustRunner(
self.settings.classes, self.settings)
runners.locust_runner.start_hatching(wait=True)
self.start_time = time.time()
runners.locust_runner.greenlet.join()
self.end_time = time.time()
logger.info('Locust completed {0} requests with {1} errors'.format(
runners.locust_runner.stats.num_requests,
len(runners.locust_runner.errors)))
except Exception as e:
logger.error("Locust exception {0}".format(repr(e)))
finally:
events.quitting.fire()
def test_parse_timespan(self):
self.assertEqual(7, parse_timespan("7"))
self.assertEqual(7, parse_timespan("7s"))
self.assertEqual(60, parse_timespan("1m"))
self.assertEqual(7200, parse_timespan("2h"))
self.assertEqual(3787, parse_timespan("1h3m7s"))
def main():
parser, options = parse_options()
# setup logging
if not options.skip_log_setup:
setup_logging(options.loglevel, options.logfile)
logger = logging.getLogger(__name__)
locust_path = get_locust_path()
if options.web:
web_port = options.web_port
logger.info('Running easy-locust web: 0.0.0.0:{}'.format(web_port))
init_app(port=web_port)
sys.exit(0)
if options.demo:
if not locust_path:
logger.error(
'''Cannot locate Python path, make sure it is in right place. If windows add it to sys PATH,
if linux make sure python is installed in /usr/local/lib/''')
sys.exit(1)
pt_demo_path = os.path.join(locust_path, 'demo',
'demo_pressuretest.xls')
pt_demo_path_json = os.path.join(locust_path, 'demo',
'demo_locustfile.json')
pt_new_demo = os.path.join(os.getcwd(), 'PtDemo.xls')
pt_new_demo_json = os.path.join(os.getcwd(), 'demo.json')
shutil.copyfile(pt_demo_path, pt_new_demo)
shutil.copyfile(pt_demo_path_json, pt_new_demo_json)
sys.exit(0)
if options.xlsfile:
pt_file = options.xlsfile
if not (pt_file.endswith('.xls') or pt_file.endswith('.json')):
logger.error(
"PressureTest file must be end with '.xls' or '.json' and see --help for available options."
)
sys.exit(1)
if not os.path.isfile(pt_file):
logger.error('PressureTest file is not exist, please check it.')
sys.exit(1)
_status = generate_locust_file(pt_file)
if not _status:
sys.exit(1)
sys.exit(0)
locustfile = find_locustfile(options.locustfile)
if not locustfile:
logger.error(
"Could not find any locustfile! Ensure file ends in '.py' and see --help for available options."
)
sys.exit(1)
if locustfile == "locust.py" or locustfile == "locust.xls" or locustfile == "locust.json":
logger.error(
"The locustfile must not be named `locust.py` or `locust.xls` or `locust.json`. "
"Please rename the file and try again.")
sys.exit(1)
docstring, locusts = load_locustfile(locustfile)
if options.list_commands:
console_logger.info("Available Locusts:")
for name in locusts:
console_logger.info(" " + name)
sys.exit(0)
if not locusts:
logger.error("No Locust class found!")
sys.exit(1)
# make sure specified Locust exists
if options.locust_classes:
missing = set(options.locust_classes) - set(locusts.keys())
if missing:
logger.error("Unknown Locust(s): %s\n" % (", ".join(missing)))
sys.exit(1)
else:
names = set(options.locust_classes) & set(locusts.keys())
locust_classes = [locusts[n] for n in names]
else:
# list() call is needed to consume the dict_view object in Python 3
locust_classes = list(locusts.values())
if options.show_task_ratio:
console_logger.info("\n Task ratio per locust class")
console_logger.info("-" * 80)
print_task_ratio(locust_classes)
console_logger.info("\n Total task ratio")
console_logger.info("-" * 80)
print_task_ratio(locust_classes, total=True)
sys.exit(0)
if options.show_task_ratio_json:
from json import dumps
task_data = {
"per_class": get_task_ratio_dict(locust_classes),
"total": get_task_ratio_dict(locust_classes, total=True)
}
console_logger.info(dumps(task_data))
sys.exit(0)
if options.run_time:
if not options.no_web:
logger.error(
"The --run-time argument can only be used together with --no-web"
)
sys.exit(1)
try:
options.run_time = parse_timespan(options.run_time)
except ValueError:
logger.error(
"Valid --run-time formats are: 20, 20s, 3m, 2h, 1h20m, 3h30m10s, etc."
)
sys.exit(1)
def spawn_run_time_limit_greenlet():
logger.info("Run time limit set to %s seconds" % options.run_time)
def timelimit_stop():
logger.info("Time limit reached. Stopping Locust.")
runners.locust_runner.quit()
gevent.spawn_later(options.run_time, timelimit_stop)
if options.step_time:
if not options.step_load:
logger.error(
"The --step-time argument can only be used together with --step-load"
)
sys.exit(1)
try:
options.step_time = parse_timespan(options.step_time)
except ValueError:
logger.error(
"Valid --step-time formats are: 20, 20s, 3m, 2h, 1h20m, 3h30m10s, etc."
)
sys.exit(1)
if options.master:
# Add -d for automatically run slaves
if options.distribute:
ptpy = locustfile
if options.locustfile.endswith('.xls'):
_type = 'xls'
pt_s = PtExcel(options.locustfile)
master_ip, pt_slave_info = pt_s.pt_slave()
else:
_type = 'dict'
with open(options.locustfile, 'r') as f:
_d = json.load(f, encoding='utf-8')
master_ip = _d.get('master_ip')
pt_slave_info = _d.get('slaves')
if master_ip == '':
logger.error(
'master IP cannot be None if you use --distribute')
sys.exit(1)
if options.boomer:
locust_cli_slave = 'nohup ./client_v1 --web --master-host={masteIP} > /dev/null 2>&1 &'.format(
masteIP=master_ip)
targets_dict, file_list = gen_boomer_client_json(
options.locustfile)
boomer_client_file = os.path.join(locust_path, 'boomer_client',
'client_v1')
file_list.append(boomer_client_file)
thread_pool = []
try:
for slave in pt_slave_info:
if _type == 'xls':
slave_ip, slave_username, slave_password = slave
else:
slave_ip, slave_username, slave_password = slave[
'ip'], slave['username'], slave['password']
_t = Thread(target=pt_slave_boomer,
args=(slave_ip, slave_username,
slave_password, file_list,
locust_cli_slave, targets_dict))
logger.info('Prepare slave {}'.format(slave_ip))
thread_pool.append(_t)
_t.start()
for each_t in thread_pool:
each_t.join()
file_list.pop()
for each in file_list:
os.remove(each)
except KeyboardInterrupt:
pass
except Exception as e:
logger.error(
'Something happened, collect Exceptions here: {}'.
format(e))
else:
try:
locust_cli_slave = 'nohup locust -f /root/locust_client.py --slave --master-host={masteIP} > /dev/null 2>&1 &'\
.format(masteIP=master_ip)
thread_pool = []
for slave in pt_slave_info:
if _type == 'xls':
slave_ip, slave_username, slave_password = slave
else:
slave_ip, slave_username, slave_password = slave[
'ip'], slave['username'], slave['password']
_t = Thread(target=pt_slave,
args=(slave_ip, slave_username,
slave_password, ptpy,
locust_cli_slave))
logger.info('Prepare slave {}'.format(slave_ip))
thread_pool.append(_t)
_t.start()
for each_t in thread_pool:
each_t.join()
except KeyboardInterrupt:
pass
except Exception as e:
logger.error(
'Something happened, collect Exceptions here: {}'.
format(e))
runners.locust_runner = MasterLocustRunner(locust_classes, options)
else:
runners.locust_runner = LocalLocustRunner(locust_classes, options)
# main_greenlet is pointing to runners.locust_runner.greenlet by default, it will point the web greenlet later if in web mode
main_greenlet = runners.locust_runner.greenlet
if options.no_web:
if options.master:
while len(runners.locust_runner.clients.ready
) < options.expect_slaves:
logging.info(
"Waiting for slaves to be ready, %s of %s connected",
len(runners.locust_runner.clients.ready),
options.expect_slaves)
time.sleep(1)
if options.step_time:
runners.locust_runner.start_stepload(options.num_clients,
options.hatch_rate,
options.step_clients,
options.step_time)
else:
runners.locust_runner.start_hatching(options.num_clients,
options.hatch_rate)
# make locusts are spawned
time.sleep(1)
else:
# spawn web greenlet
logger.info("Starting web monitor at http://%s:%s" %
(options.web_host or "*", options.port))
main_greenlet = gevent.spawn(web.start, locust_classes, options)
if options.run_time:
spawn_run_time_limit_greenlet()
stats_printer_greenlet = None
if not options.only_summary and (options.print_stats or options.no_web):
# spawn stats printing greenlet
stats_printer_greenlet = gevent.spawn(stats_printer)
if options.csvfilebase:
gevent.spawn(stats_writer, options.csvfilebase,
options.stats_history_enabled)
def shutdown(code=0):
"""
Shut down locust by firing quitting event, printing/writing stats and exiting
"""
logger.info("Shutting down (exit code %s), bye." % code)
if stats_printer_greenlet is not None:
stats_printer_greenlet.kill(block=False)
logger.info("Cleaning up runner...")
if runners.locust_runner is not None:
runners.locust_runner.quit()
logger.info("Running teardowns...")
events.quitting.fire(reverse=True)
print_stats(runners.locust_runner.stats, current=False)
print_percentile_stats(runners.locust_runner.stats)
if options.csvfilebase:
write_stat_csvs(options.csvfilebase, options.stats_history_enabled)
print_error_report()
sys.exit(code)
# install SIGTERM handler
def sig_term_handler():
logger.info("Got SIGTERM signal")
shutdown(0)
gevent.signal(signal.SIGTERM, sig_term_handler)
try:
logger.info("Starting Locust %s" % version)
main_greenlet.join()
code = 0
if len(runners.locust_runner.errors) or len(
runners.locust_runner.exceptions):
code = options.exit_code_on_error
shutdown(code=code)
except KeyboardInterrupt:
shutdown(0)
def main():
# find specified locustfile and make sure it exists, using a very simplified
# command line parser that is only used to parse the -f option
locustfile = parse_locustfile_option()
# import the locustfile
docstring, user_classes, shape_class = load_locustfile(locustfile)
# parse all command line options
options = parse_options()
if options.slave or options.expect_slaves:
sys.stderr.write(
"The --slave/--expect-slaves parameters have been renamed --worker/--expect-workers\n"
)
sys.exit(1)
if options.hatch_rate:
sys.stderr.write(
"[DEPRECATED] The --hatch-rate parameter has been renamed --spawn-rate\n"
)
options.spawn_rate = options.hatch_rate
# setup logging
if not options.skip_log_setup:
if options.loglevel.upper() in [
"DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"
]:
setup_logging(options.loglevel, options.logfile)
else:
sys.stderr.write(
"Invalid --loglevel. Valid values are: DEBUG/INFO/WARNING/ERROR/CRITICAL\n"
)
sys.exit(1)
logger = logging.getLogger(__name__)
greenlet_exception_handler = greenlet_exception_logger(logger)
if options.list_commands:
print("Available Users:")
for name in user_classes:
print(" " + name)
sys.exit(0)
if not user_classes:
logger.error("No User class found!")
sys.exit(1)
# make sure specified User exists
if options.user_classes:
missing = set(options.user_classes) - set(user_classes.keys())
if missing:
logger.error("Unknown User(s): %s\n" % (", ".join(missing)))
sys.exit(1)
else:
names = set(options.user_classes) & set(user_classes.keys())
user_classes = [user_classes[n] for n in names]
else:
# list() call is needed to consume the dict_view object in Python 3
user_classes = list(user_classes.values())
try:
import resource
if resource.getrlimit(resource.RLIMIT_NOFILE)[0] < 10000:
# Increasing the limit to 10000 within a running process should work on at least MacOS.
# It does not work on all OS:es, but we should be no worse off for trying.
resource.setrlimit(resource.RLIMIT_NOFILE,
[10000, resource.RLIM_INFINITY])
except:
logger.warning(
"System open file limit setting is not high enough for load testing, and the OS didn't allow locust to increase it by itself. See https://github.com/locustio/locust/wiki/Installation#increasing-maximum-number-of-open-files-limit for more info."
)
# create locust Environment
environment = create_environment(user_classes,
options,
events=locust.events,
shape_class=shape_class)
if shape_class and (options.num_users or options.spawn_rate
or options.step_load):
logger.error(
"The specified locustfile contains a shape class but a conflicting argument was specified: users, spawn-rate or step-load"
)
sys.exit(1)
if options.show_task_ratio:
print("\n Task ratio per User class")
print("-" * 80)
print_task_ratio(user_classes)
print("\n Total task ratio")
print("-" * 80)
print_task_ratio(user_classes, total=True)
sys.exit(0)
if options.show_task_ratio_json:
from json import dumps
task_data = {
"per_class": get_task_ratio_dict(user_classes),
"total": get_task_ratio_dict(user_classes, total=True)
}
print(dumps(task_data))
sys.exit(0)
if options.step_time:
if not options.step_load:
logger.error(
"The --step-time argument can only be used together with --step-load"
)
sys.exit(1)
if options.worker:
logger.error(
"--step-time should be specified on the master node, and not on worker nodes"
)
sys.exit(1)
try:
options.step_time = parse_timespan(options.step_time)
except ValueError:
logger.error(
"Valid --step-time formats are: 20, 20s, 3m, 2h, 1h20m, 3h30m10s, etc."
)
sys.exit(1)
if options.master:
runner = environment.create_master_runner(
master_bind_host=options.master_bind_host,
master_bind_port=options.master_bind_port,
)
elif options.worker:
try:
runner = environment.create_worker_runner(options.master_host,
options.master_port)
except socket.error as e:
logger.error("Failed to connect to the Locust master: %s", e)
sys.exit(-1)
else:
runner = environment.create_local_runner()
# main_greenlet is pointing to runners.greenlet by default, it will point the web greenlet later if in web mode
main_greenlet = runner.greenlet
if options.run_time:
if not options.headless:
logger.error(
"The --run-time argument can only be used together with --headless"
)
sys.exit(1)
if options.worker:
logger.error(
"--run-time should be specified on the master node, and not on worker nodes"
)
sys.exit(1)
try:
options.run_time = parse_timespan(options.run_time)
except ValueError:
logger.error(
"Valid --run-time formats are: 20, 20s, 3m, 2h, 1h20m, 3h30m10s, etc."
)
sys.exit(1)
def spawn_run_time_limit_greenlet():
logger.info("Run time limit set to %s seconds" % options.run_time)
def timelimit_stop():
logger.info("Time limit reached. Stopping Locust.")
runner.quit()
gevent.spawn_later(
options.run_time,
timelimit_stop).link_exception(greenlet_exception_handler)
if options.csv_prefix:
stats_csv_writer = StatsCSVFileWriter(environment,
stats.PERCENTILES_TO_REPORT,
options.csv_prefix,
options.stats_history_enabled)
else:
stats_csv_writer = StatsCSV(environment, stats.PERCENTILES_TO_REPORT)
# start Web UI
if not options.headless and not options.worker:
# spawn web greenlet
protocol = "https" if options.tls_cert and options.tls_key else "http"
try:
if options.web_host == "*":
# special check for "*" so that we're consistent with --master-bind-host
web_host = ''
else:
web_host = options.web_host
if web_host:
logger.info("Starting web interface at %s://%s:%s" %
(protocol, web_host, options.web_port))
else:
logger.info(
"Starting web interface at %s://0.0.0.0:%s (accepting connections from all network interfaces)"
% (protocol, options.web_port))
web_ui = environment.create_web_ui(
host=web_host,
port=options.web_port,
auth_credentials=options.web_auth,
tls_cert=options.tls_cert,
tls_key=options.tls_key,
stats_csv_writer=stats_csv_writer,
)
except AuthCredentialsError:
logger.error(
"Credentials supplied with --web-auth should have the format: username:password"
)
sys.exit(1)
else:
main_greenlet = web_ui.greenlet
else:
web_ui = None
# Fire locust init event which can be used by end-users' code to run setup code that
# need access to the Environment, Runner or WebUI
environment.events.init.fire(environment=environment,
runner=runner,
web_ui=web_ui)
if options.headless:
# headless mode
if options.master:
# wait for worker nodes to connect
while len(runner.clients.ready) < options.expect_workers:
logging.info(
"Waiting for workers to be ready, %s of %s connected",
len(runner.clients.ready), options.expect_workers)
time.sleep(1)
if not options.worker:
# apply headless mode defaults
if options.num_users is None:
options.num_users = 1
if options.spawn_rate is None:
options.spawn_rate = 1
if options.step_users is None:
options.step_users = 1
# start the test
if options.step_time:
runner.start_stepload(options.num_users, options.spawn_rate,
options.step_users, options.step_time)
if environment.shape_class:
environment.runner.start_shape()
else:
runner.start(options.num_users, options.spawn_rate)
if options.run_time:
spawn_run_time_limit_greenlet()
stats_printer_greenlet = None
if not options.only_summary and (options.print_stats or
(options.headless
and not options.worker)):
# spawn stats printing greenlet
stats_printer_greenlet = gevent.spawn(stats_printer(runner.stats))
stats_printer_greenlet.link_exception(greenlet_exception_handler)
if options.csv_prefix:
gevent.spawn(stats_csv_writer.stats_writer).link_exception(
greenlet_exception_handler)
gevent.spawn(stats_history, runner)
def shutdown():
"""
Shut down locust by firing quitting event, printing/writing stats and exiting
"""
logger.info("Running teardowns...")
environment.events.quitting.fire(environment=environment, reverse=True)
# determine the process exit code
if log.unhandled_greenlet_exception:
code = 2
elif environment.process_exit_code is not None:
code = environment.process_exit_code
elif len(runner.errors) or len(runner.exceptions):
code = options.exit_code_on_error
else:
code = 0
logger.info("Shutting down (exit code %s), bye." % code)
if stats_printer_greenlet is not None:
stats_printer_greenlet.kill(block=False)
logger.info("Cleaning up runner...")
if runner is not None:
runner.quit()
print_stats(runner.stats, current=False)
print_percentile_stats(runner.stats)
print_error_report(runner.stats)
sys.exit(code)
# install SIGTERM handler
def sig_term_handler():
logger.info("Got SIGTERM signal")
shutdown()
gevent.signal_handler(signal.SIGTERM, sig_term_handler)
try:
logger.info("Starting Locust %s" % version)
main_greenlet.join()
shutdown()
except KeyboardInterrupt as e:
shutdown()