def launch(classname, n_clients, run_time=180):
"""
Launches the tests
:param: classname: class inherited from HttpLocust defining the test
:param: n_clients: Number of concurrent users
:param: run_time: Stop testing after the specified amount of seconds
"""
base_url = os.environ.get('KOLIBRI_BASE_URL', 'http://127.0.0.1:8000')
options = Namespace(**{
'host': base_url,
'num_clients': n_clients,
'hatch_rate': n_clients, # this way it will be 1 request per second per client
'num_requests': 9999999, # obsolete, discontinued in new locust versions
'run_time': run_time,
'no_web': True,
'no_reset_stats': True,
'csvfilebase': os.path.join(get_or_create_output_dir(), get_csv_filename()),
'expect_slaves': 1,
'master_host': '127.0.0.1',
'master_port': 5557,
'master_bind_host': '*',
'master_bind_port': 5557
})
setup_logging('INFO', None)
if classname.__module__ == '__main__':
test_path = 'scenarios/{}.py'.format(get_test_calling())
else:
test_path = 'scenarios/{}.py'.format(classname.__module__)
locust_executable = spawn.find_executable('locust')
slave_args = [locust_executable, '--slave', '-f', test_path]
for slave in range(n_clients):
subprocess.Popen(slave_args, env={'KOLIBRI_BASE_URL': base_url})
if '127.0.0.1' in base_url or 'localhost' in base_url:
time.sleep(1)
else:
time.sleep(5)
runners.locust_runner = MasterLocustRunner([classname], options)
while len(runners.locust_runner.clients.ready) < options.expect_slaves:
time.sleep(1)
# spawn client spawning/hatching greenlets:
runners.locust_runner.start_hatching(options.num_clients, options.hatch_rate)
main_greenlet = runners.locust_runner.greenlet
spawn_run_time_limit_greenlet(options)
if options.csvfilebase:
gevent.spawn(stats_writer, options.csvfilebase)
try:
main_greenlet.join()
code = 0
if len(runners.locust_runner.errors):
code = 1
shutdown(options, code=code)
except KeyboardInterrupt:
shutdown(options, 0)
def home_page(self):
with self.client.get("", catch_response=True,
name="T00_HomePage") as response:
if "Welcome to JPetStore 6" in response.text and response.elapsed.total_seconds(
) < 2.0:
response.success()
log.setup_logging("Testing")
else:
response.failure(
"Home page took too long to load and/or text check has failed."
)
def main():
args = ['--no-web', '-c', CONF.client, '--hatch-rate', CONF.client]
sys.argv.extend(args)
parser, options, arguments = parse_options()
setup_logging(options.loglevel, options.logfile)
file_dir = CONF.test_file_dir
test_cases = []
for f in CONF.test_file_names.split(','):
path = os.path.join(file_dir, f)
cases = load_testcase(path)
test_cases.append(cases)
for case in test_cases:
for k, v in case.iteritems():
run(v, options)
gevent.signal(signal.SIGTERM, sig_term_handler)
def locust_executor(host,
port,
collection_name,
connection_type="single",
run_params=None):
m = MilvusClient(host=host, port=port, collection_name=collection_name)
MyUser.tasks = {}
tasks = run_params["tasks"]
for op, weight in tasks.items():
task = {eval("Tasks." + op): weight}
MyUser.tasks.update(task)
logger.error(MyUser.tasks)
# MyUser.tasks = {Tasks.query: 1, Tasks.flush: 1}
MyUser.client = MilvusTask(host=host,
port=port,
collection_name=collection_name,
connection_type=connection_type,
m=m)
env = Environment(events=events, user_classes=[MyUser])
runner = env.create_local_runner()
# setup logging
# setup_logging("WARNING", "/dev/null")
setup_logging("WARNING", "/dev/null")
greenlet_exception_logger(logger=logger)
gevent.spawn(stats_printer(env.stats))
# env.create_web_ui("127.0.0.1", 8089)
# gevent.spawn(stats_printer(env.stats), env, "test", full_history=True)
# events.init.fire(environment=env, runner=runner)
clients_num = run_params["clients_num"]
spawn_rate = run_params["spawn_rate"]
during_time = run_params["during_time"]
runner.start(clients_num, spawn_rate=spawn_rate)
gevent.spawn_later(during_time, lambda: runner.quit())
runner.greenlet.join()
print_stats(env.stats)
result = {
"rps": round(env.stats.total.current_rps, 1),
"fail_ratio": env.stats.total.fail_ratio,
"max_response_time": round(env.stats.total.max_response_time, 1),
"min_response_time": round(env.stats.total.avg_response_time, 1)
}
runner.stop()
return result
def __init__(self, core, cfg, cfg_updater):
AbstractPlugin.__init__(self, core, cfg, cfg_updater)
self.core = core
self._locustrunner = None
self._locustclasses = None
self._options = None
self._user_count = 0
self._state = ''
self._locuststats = ''
self._locustslaves = None
self.stats_reader = None
self.reader = None
self.host = None
self.web_host = ''
self.port = 8089
self.locustfile = 'locustfile'
self.master = False
self.slave = False
self.master_host = "127.0.0.1"
self.master_port = 5557
self.master_bind_host = "*"
self.master_bind_port = 5557
self.expect_slaves = 0
self.no_web = True
self.num_clients = int(1)
self.hatch_rate = float(1)
self.num_requests = None
self.run_time = None
self.loglevel = 'INFO'
self.logfile = None
self.csvfilebase = None
self.csvappend = True
self.print_stats = True
self.only_summary = True
self.list_commands = False
self.show_task_ratio = False
self.show_task_ratio_json = False
self.show_version = True
self.locustlog_level = 'INFO'
self.cfg = cfg
# setup logging
ll.setup_logging(self.loglevel, self.logfile)
def launch(classname, base_url, n_clients, rate, n_requests=None, timeout=600):
"""
Launches the tests
:param: classname: class inherited from HttpLocust defining the test
:param: base_url: server and port of the url to request
:param: n_clients: Number of concurrent users
:param: rate: The rate per second in which clients are spawned
:param: n_requests: Total number of requests to be done
:param: timeout: Stop testing after the specified amount of seconds
"""
options = Namespace(**{
'host': base_url,
'num_clients': n_clients,
'hatch_rate': rate,
'num_requests': n_requests if n_requests else n_clients * 10,
'run_time': timeout,
'no_web': True,
'no_reset_stats': True,
'csvfilebase': os.path.join(get_or_create_output_dir(), get_test_calling())
})
setup_logging('INFO', None)
runners.locust_runner = LocalLocustRunner([classname], options)
# spawn client spawning/hatching greenlets:
runners.locust_runner.start_hatching(wait=True)
main_greenlet = runners.locust_runner.greenlet
spawn_run_time_limit_greenlet(options)
if options.csvfilebase:
gevent.spawn(stats_writer, options.csvfilebase)
try:
main_greenlet.join()
code = 0
if len(runners.locust_runner.errors):
code = 1
shutdown(options, code=code)
except KeyboardInterrupt:
shutdown(options, 0)
import gevent
from locust import HttpLocust, TaskSet, task, between
from locust.runners import LocalLocustRunner
from locust.env import Environment
from locust.stats import stats_printer
from locust.log import setup_logging
from locust.web import WebUI
setup_logging("INFO", None)
class User(HttpLocust):
wait_time = between(1, 3)
host = "https://docs.locust.io"
class task_set(TaskSet):
@task
def my_task(self):
self.client.get("/")
@task
def task_404(self):
self.client.get("/non-existing-path")
# setup Environment and Runner
env = Environment()
runner = LocalLocustRunner(environment=env, locust_classes=[User])
# start a WebUI instance
web_ui = WebUI(environment=env)
gevent.spawn(lambda: web_ui.start("127.0.0.1", 8089))
import sys
import gevent
from locust.env import Environment
from locust.event import EventHook
from locust.log import setup_logging
from locust.stats import stats_printer
from locustfile import StudioDesktopBrowserUser
setup_logging("DEBUG", None)
def error_output(*args, **kwargs):
print("Error: {}, {}".format(args, kwargs))
failure_hook = EventHook()
failure_hook.add_listener(error_output)
# setup Environment and Runner
env = Environment(user_classes=[StudioDesktopBrowserUser])
env.events.request_failure = failure_hook
env.create_local_runner()
# start a WebUI instance
env.create_web_ui("127.0.0.1", 8089)
# start a greenlet that periodically outputs the current stats
gevent.spawn(stats_printer(env.stats))
# start the test
env.runner.start(10, hatch_rate=10)
def main():
# 改成直接使用满足boomer中的类
user_classes = {"name": Dummy}
user_classes = list(user_classes.values())
# 解析命令行参数
options = parse_options()
print(options)
# 设置logging
if not options.skip_log_setup:
if options.loglevel.upper() in [
"DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"
]:
setup_logging(options.loglevel, options.logfile)
else:
sys.stderr.write(
"非法参数--log-level. 合法值: DEBUG/INFO/WARNING/ERROR/CRITICAL\n")
sys.exit(1)
logger = logging.getLogger(__name__)
logger.warning(
"这里是locust hazard改造的web-ui专用版本，只支持以下命令：\n" +
"\t--master-host(必须)、--master-bind-port、--web-port、--web-auth、--tls-cert、--tls-key、--log-level"
)
logger.info("options.master_host=" + options.master_host)
if not options.master_host:
sys.stdout.write("请提供--master-host参数，以便通知压测机\n")
exit(0)
try:
import resource
if resource.getrlimit(resource.RLIMIT_NOFILE)[0] < 10000:
# Increasing the limit to 10000 within a running process should work on at least MacOS.
# It does not work on all OS:es, but we should be no worse off for trying.
resource.setrlimit(resource.RLIMIT_NOFILE,
[10000, resource.RLIM_INFINITY])
except:
logger.warning(
"System open file limit setting is not high enough for load testing, and the OS wouldnt allow locust to increase it by itself.\n"
+
"\tSee https://docs.locust.io/en/stable/installation.html#increasing-maximum-number-of-open-files-limit for more info."
)
# create locust Environment
environment = create_environment(user_classes,
options,
events=locust.events)
# 只使用master模式运行
runner = environment.create_master_runner(
master_bind_host="*", master_bind_port=options.master_bind_port)
runner.state = runners.STATE_STOPPED
stats_csv_writer = StatsCSV(environment, stats.PERCENTILES_TO_REPORT)
# 开启web-ui服务
web_host = "0.0.0.0"
protocol = "https" if options.tls_cert and options.tls_key else "http"
logger.info("Starting web interface at %s://%s:%s" %
(protocol, '127.0.0.1', options.web_port))
try:
web_ui = WebUI(
environment,
host=web_host,
port=options.web_port,
masterHost=options.master_host,
auth_credentials=options.web_auth,
tls_cert=options.tls_cert,
tls_key=options.tls_key,
stats_csv_writer=stats_csv_writer,
)
except AuthCredentialsError:
logger.error(
"Credentials supplied with --web-auth should have the format: username:password"
)
sys.exit(1)
else:
main_greenlet = web_ui.greenlet
# Fire locust init event which can be used by end-users' code to run setup code that
# need access to the Environment, Runner or WebUI
environment.events.init.fire(environment=environment,
runner=runner,
web_ui=web_ui)
stats_printer_greenlet = None # 指标打印协程
def shutdown():
"""
Shut down locust by firing quitting event, printing/writing stats and exiting
"""
logger.info("Running teardowns...")
environment.events.quitting.fire(environment=environment, reverse=True)
# determine the process exit code
if log.unhandled_greenlet_exception:
code = 2
elif environment.process_exit_code is not None:
code = environment.process_exit_code
elif len(runner.errors) or len(runner.exceptions):
code = options.exit_code_on_error
else:
code = 0
logger.info("Shutting down (exit code %s), bye." % code)
if stats_printer_greenlet is not None:
stats_printer_greenlet.kill(block=False)
logger.info("Cleaning up runner...")
if runner is not None:
runner.quit()
print_stats(runner.stats, current=False)
print_percentile_stats(runner.stats)
print_error_report(runner.stats)
sys.exit(code)
# install SIGTERM handler
def sig_term_handler():
logger.info("Got SIGTERM signal, shutdown.......")
shutdown()
gevent.signal_handler(signal.SIGTERM, sig_term_handler)
try:
logger.info("Starting Locust %s" % version)
main_greenlet.join()
logger.info("The main_greenlet finished, shutdown.......")
shutdown()
except KeyboardInterrupt as e:
logger.error("Got the KeyboardInterrupt, shutdown.......")
shutdown()
def main():
parser, options = parse_options()
# setup logging
if not options.skip_log_setup:
setup_logging(options.loglevel, options.logfile)
logger = logging.getLogger(__name__)
locust_path = get_locust_path()
if options.web:
web_port = options.web_port
logger.info('Running easy-locust web: 0.0.0.0:{}'.format(web_port))
init_app(port=web_port)
sys.exit(0)
if options.demo:
if not locust_path:
logger.error(
'''Cannot locate Python path, make sure it is in right place. If windows add it to sys PATH,
if linux make sure python is installed in /usr/local/lib/''')
sys.exit(1)
pt_demo_path = os.path.join(locust_path, 'demo',
'demo_pressuretest.xls')
pt_demo_path_json = os.path.join(locust_path, 'demo',
'demo_locustfile.json')
pt_new_demo = os.path.join(os.getcwd(), 'PtDemo.xls')
pt_new_demo_json = os.path.join(os.getcwd(), 'demo.json')
shutil.copyfile(pt_demo_path, pt_new_demo)
shutil.copyfile(pt_demo_path_json, pt_new_demo_json)
sys.exit(0)
if options.xlsfile:
pt_file = options.xlsfile
if not (pt_file.endswith('.xls') or pt_file.endswith('.json')):
logger.error(
"PressureTest file must be end with '.xls' or '.json' and see --help for available options."
)
sys.exit(1)
if not os.path.isfile(pt_file):
logger.error('PressureTest file is not exist, please check it.')
sys.exit(1)
_status = generate_locust_file(pt_file)
if not _status:
sys.exit(1)
sys.exit(0)
locustfile = find_locustfile(options.locustfile)
if not locustfile:
logger.error(
"Could not find any locustfile! Ensure file ends in '.py' and see --help for available options."
)
sys.exit(1)
if locustfile == "locust.py" or locustfile == "locust.xls" or locustfile == "locust.json":
logger.error(
"The locustfile must not be named `locust.py` or `locust.xls` or `locust.json`. "
"Please rename the file and try again.")
sys.exit(1)
docstring, locusts = load_locustfile(locustfile)
if options.list_commands:
console_logger.info("Available Locusts:")
for name in locusts:
console_logger.info(" " + name)
sys.exit(0)
if not locusts:
logger.error("No Locust class found!")
sys.exit(1)
# make sure specified Locust exists
if options.locust_classes:
missing = set(options.locust_classes) - set(locusts.keys())
if missing:
logger.error("Unknown Locust(s): %s\n" % (", ".join(missing)))
sys.exit(1)
else:
names = set(options.locust_classes) & set(locusts.keys())
locust_classes = [locusts[n] for n in names]
else:
# list() call is needed to consume the dict_view object in Python 3
locust_classes = list(locusts.values())
if options.show_task_ratio:
console_logger.info("\n Task ratio per locust class")
console_logger.info("-" * 80)
print_task_ratio(locust_classes)
console_logger.info("\n Total task ratio")
console_logger.info("-" * 80)
print_task_ratio(locust_classes, total=True)
sys.exit(0)
if options.show_task_ratio_json:
from json import dumps
task_data = {
"per_class": get_task_ratio_dict(locust_classes),
"total": get_task_ratio_dict(locust_classes, total=True)
}
console_logger.info(dumps(task_data))
sys.exit(0)
if options.run_time:
if not options.no_web:
logger.error(
"The --run-time argument can only be used together with --no-web"
)
sys.exit(1)
try:
options.run_time = parse_timespan(options.run_time)
except ValueError:
logger.error(
"Valid --run-time formats are: 20, 20s, 3m, 2h, 1h20m, 3h30m10s, etc."
)
sys.exit(1)
def spawn_run_time_limit_greenlet():
logger.info("Run time limit set to %s seconds" % options.run_time)
def timelimit_stop():
logger.info("Time limit reached. Stopping Locust.")
runners.locust_runner.quit()
gevent.spawn_later(options.run_time, timelimit_stop)
if options.step_time:
if not options.step_load:
logger.error(
"The --step-time argument can only be used together with --step-load"
)
sys.exit(1)
try:
options.step_time = parse_timespan(options.step_time)
except ValueError:
logger.error(
"Valid --step-time formats are: 20, 20s, 3m, 2h, 1h20m, 3h30m10s, etc."
)
sys.exit(1)
if options.master:
# Add -d for automatically run slaves
if options.distribute:
ptpy = locustfile
if options.locustfile.endswith('.xls'):
_type = 'xls'
pt_s = PtExcel(options.locustfile)
master_ip, pt_slave_info = pt_s.pt_slave()
else:
_type = 'dict'
with open(options.locustfile, 'r') as f:
_d = json.load(f, encoding='utf-8')
master_ip = _d.get('master_ip')
pt_slave_info = _d.get('slaves')
if master_ip == '':
logger.error(
'master IP cannot be None if you use --distribute')
sys.exit(1)
if options.boomer:
locust_cli_slave = 'nohup ./client_v1 --web --master-host={masteIP} > /dev/null 2>&1 &'.format(
masteIP=master_ip)
targets_dict, file_list = gen_boomer_client_json(
options.locustfile)
boomer_client_file = os.path.join(locust_path, 'boomer_client',
'client_v1')
file_list.append(boomer_client_file)
thread_pool = []
try:
for slave in pt_slave_info:
if _type == 'xls':
slave_ip, slave_username, slave_password = slave
else:
slave_ip, slave_username, slave_password = slave[
'ip'], slave['username'], slave['password']
_t = Thread(target=pt_slave_boomer,
args=(slave_ip, slave_username,
slave_password, file_list,
locust_cli_slave, targets_dict))
logger.info('Prepare slave {}'.format(slave_ip))
thread_pool.append(_t)
_t.start()
for each_t in thread_pool:
each_t.join()
file_list.pop()
for each in file_list:
os.remove(each)
except KeyboardInterrupt:
pass
except Exception as e:
logger.error(
'Something happened, collect Exceptions here: {}'.
format(e))
else:
try:
locust_cli_slave = 'nohup locust -f /root/locust_client.py --slave --master-host={masteIP} > /dev/null 2>&1 &'\
.format(masteIP=master_ip)
thread_pool = []
for slave in pt_slave_info:
if _type == 'xls':
slave_ip, slave_username, slave_password = slave
else:
slave_ip, slave_username, slave_password = slave[
'ip'], slave['username'], slave['password']
_t = Thread(target=pt_slave,
args=(slave_ip, slave_username,
slave_password, ptpy,
locust_cli_slave))
logger.info('Prepare slave {}'.format(slave_ip))
thread_pool.append(_t)
_t.start()
for each_t in thread_pool:
each_t.join()
except KeyboardInterrupt:
pass
except Exception as e:
logger.error(
'Something happened, collect Exceptions here: {}'.
format(e))
runners.locust_runner = MasterLocustRunner(locust_classes, options)
else:
runners.locust_runner = LocalLocustRunner(locust_classes, options)
# main_greenlet is pointing to runners.locust_runner.greenlet by default, it will point the web greenlet later if in web mode
main_greenlet = runners.locust_runner.greenlet
if options.no_web:
if options.master:
while len(runners.locust_runner.clients.ready
) < options.expect_slaves:
logging.info(
"Waiting for slaves to be ready, %s of %s connected",
len(runners.locust_runner.clients.ready),
options.expect_slaves)
time.sleep(1)
if options.step_time:
runners.locust_runner.start_stepload(options.num_clients,
options.hatch_rate,
options.step_clients,
options.step_time)
else:
runners.locust_runner.start_hatching(options.num_clients,
options.hatch_rate)
# make locusts are spawned
time.sleep(1)
else:
# spawn web greenlet
logger.info("Starting web monitor at http://%s:%s" %
(options.web_host or "*", options.port))
main_greenlet = gevent.spawn(web.start, locust_classes, options)
if options.run_time:
spawn_run_time_limit_greenlet()
stats_printer_greenlet = None
if not options.only_summary and (options.print_stats or options.no_web):
# spawn stats printing greenlet
stats_printer_greenlet = gevent.spawn(stats_printer)
if options.csvfilebase:
gevent.spawn(stats_writer, options.csvfilebase,
options.stats_history_enabled)
def shutdown(code=0):
"""
Shut down locust by firing quitting event, printing/writing stats and exiting
"""
logger.info("Shutting down (exit code %s), bye." % code)
if stats_printer_greenlet is not None:
stats_printer_greenlet.kill(block=False)
logger.info("Cleaning up runner...")
if runners.locust_runner is not None:
runners.locust_runner.quit()
logger.info("Running teardowns...")
events.quitting.fire(reverse=True)
print_stats(runners.locust_runner.stats, current=False)
print_percentile_stats(runners.locust_runner.stats)
if options.csvfilebase:
write_stat_csvs(options.csvfilebase, options.stats_history_enabled)
print_error_report()
sys.exit(code)
# install SIGTERM handler
def sig_term_handler():
logger.info("Got SIGTERM signal")
shutdown(0)
gevent.signal(signal.SIGTERM, sig_term_handler)
try:
logger.info("Starting Locust %s" % version)
main_greenlet.join()
code = 0
if len(runners.locust_runner.errors) or len(
runners.locust_runner.exceptions):
code = options.exit_code_on_error
shutdown(code=code)
except KeyboardInterrupt:
shutdown(0)
import sys
from datetime import datetime
import tarfile
import pathlib
import subprocess
import constants as c
from model import MeasurementResult
from model import BackendMetrics
from model import SystemState
from model import MeasurementSequenceResult
logging.basicConfig(level=logging.INFO)
# setup Locust logging
setup_logging('INFO', None)
class User(HttpUser):
# constant_pacing for an adaptive time that ensures the task runs (at most) once every X seconds
wait_time = constant_pacing(5)
host = c.BACKEND_HOST
@task
def submit_job_task(self):
headers = {
'Content-Type': 'application/json',
'Accept': 'application/json'
}
data = {'input': 48}
self.client.post('/api/v1/jobs', json=data, headers=headers)
def main(user_params=None, common_param=None):
parser, options, arguments = parse_options()
# setup logging
setup_logging(options.loglevel, options.logfile)
logger = logging.getLogger(__name__)
if options.show_version:
print("Locust %s" % (version, ))
sys.exit(0)
locustfile = find_locustfile(options.locustfile)
if not locustfile:
logger.error(
"Could not find any locustfile! Ensure file ends in '.py' and see --help for available options."
)
sys.exit(1)
if locustfile == "locust.py":
logger.error(
"The locustfile must not be named `locust.py`. Please rename the file and try again."
)
sys.exit(1)
docstring, locusts = load_locustfile(locustfile)
if options.list_commands:
console_logger.info("Available Locusts:")
for name in locusts:
console_logger.info(" " + name)
sys.exit(0)
if not locusts:
logger.error("No Locust class found!")
sys.exit(1)
# make sure specified Locust exists
if arguments:
missing = set(arguments) - set(locusts.keys())
if missing:
logger.error("Unknown Locust(s): %s\n" % (", ".join(missing)))
sys.exit(1)
else:
names = set(arguments) & set(locusts.keys())
locust_classes = [locusts[n] for n in names]
else:
# list() call is needed to consume the dict_view object in Python 3
locust_classes = list(locusts.values())
if options.show_task_ratio:
console_logger.info("\n Task ratio per locust class")
console_logger.info("-" * 80)
print_task_ratio(locust_classes)
console_logger.info("\n Total task ratio")
console_logger.info("-" * 80)
print_task_ratio(locust_classes, total=True)
sys.exit(0)
if options.show_task_ratio_json:
from json import dumps
task_data = {
"per_class": get_task_ratio_dict(locust_classes),
"total": get_task_ratio_dict(locust_classes, total=True)
}
console_logger.info(dumps(task_data))
sys.exit(0)
if not options.no_web and not options.slave:
# spawn web greenlet
logger.info("Starting web monitor at %s:%s" %
(options.web_host or "*", options.port))
main_greenlet = gevent.spawn(web.start, locust_classes, options)
if not options.master and not options.slave:
if user_params:
runners.locust_runner = ParameterizableLocustRunner(
locust_classes, options, user_params, common_param)
else:
runners.locust_runner = LocalLocustRunner(locust_classes, options)
# spawn client spawning/hatching greenlet
if options.no_web:
runners.locust_runner.start_hatching(wait=True)
main_greenlet = runners.locust_runner.greenlet
elif options.master:
runners.locust_runner = MasterLocustRunner(locust_classes, options)
if options.no_web:
while len(runners.locust_runner.clients.ready
) < options.expect_slaves:
logging.info(
"Waiting for slaves to be ready, %s of %s connected",
len(runners.locust_runner.clients.ready),
options.expect_slaves)
time.sleep(1)
runners.locust_runner.start_hatching(options.num_clients,
options.hatch_rate)
main_greenlet = runners.locust_runner.greenlet
elif options.slave:
try:
runners.locust_runner = SlaveLocustRunner(locust_classes, options)
main_greenlet = runners.locust_runner.greenlet
except socket.error as e:
logger.error("Failed to connect to the Locust master: %s", e)
sys.exit(-1)
if not options.only_summary and (options.print_stats or
(options.no_web and not options.slave)):
# spawn stats printing greenlet
gevent.spawn(stats_printer)
if options.csvfilebase:
gevent.spawn(stats_writer, options.csvfilebase)
def shutdown(code=0):
"""
Shut down locust by firing quitting event, printing/writing stats and exiting
"""
logger.info("Shutting down (exit code %s), bye." % code)
events.quitting.fire()
print_stats(runners.locust_runner.request_stats)
print_percentile_stats(runners.locust_runner.request_stats)
if options.csvfilebase:
write_stat_csvs(options.csvfilebase)
print_error_report()
sys.exit(code)
# install SIGTERM handler
def sig_term_handler():
logger.info("Got SIGTERM signal")
shutdown(0)
gevent.signal(gevent.signal.SIGTERM, sig_term_handler)
try:
logger.info("Starting Locust %s" % version)
main_greenlet.join()
code = 0
if len(runners.locust_runner.errors):
code = 1
shutdown(code=code)
except KeyboardInterrupt as e:
shutdown(0)
if args.process_name is None:
return f'{os.path.dirname(os.path.realpath(__file__))}{get_slash()}Data{get_slash()}{data_time_str}_{args.total_user}_{args.process_pid}'
else:
return f'{os.path.dirname(os.path.realpath(__file__))}{get_slash()}Data{get_slash()}{data_time_str}_{args.total_user}_{args.process_name}'
def tell_when_this_done(total_monitor_time):
complete_part = 0
while True:
time.sleep(total_monitor_time / 10)
complete_part += 10
logger.info(f'{complete_part}% Done')
if __name__ == '__main__':
setup_logging('INFO')
data_time_str = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
args = config_argument_parser()
logger = logging.getLogger(__name__)
locust_runner = LocustRunner(args.target_server_address, AccessLogUser)
can_draw_process = False
try:
process_monitor = ProcessMonitor(
pid=args.process_pid,
name=args.process_name,
csv_file=get_default_process_monitor_csv_prefix(
args.process_pid, args.process_name, args.total_user))
process_monitor_task = threading.Thread(target=run_process_monitor,
args=(process_monitor, args))
process_monitor_task.start()
can_draw_process = True
parser.add_argument('-ho', '--host', default='https://localhost:5001/graphql')
parser.add_argument('--wait-time-min', default=1, type=int, help='Default minimum wait time between tasks')
parser.add_argument('--wait-time-max', default=10, type=int, help='Default maximum wait time between tasks')
parser.add_argument('-w', '--web', action='store_true', help='Enable Web UI')
parser.add_argument('-wh', '--web-host', default='127.0.0.1', help='Address to host the Web UI')
parser.add_argument('-wp', '--web-port', default=8089, type=int, help='Port to host the Web UI')
parser.add_argument('-u', '--number-of-users', default=10, type=int, help='Number of Users to start')
parser.add_argument('-uh', '--user-hatch-rate', default=5, type=int, help='Number of Users to spawn per second')
parser.add_argument('-d', '--duration', default=60, type=int, help='Duration of the load test (in seconds)')
parser.add_argument('-o', '--output_directory', default='results', type=str, help='Output directory to put results')
parser.add_argument('-f', '--print_final_statistics', action='store_true', default=True, help='Print the final statistics before exit')
parser.add_argument('-v', '--verbosity', default="INFO", type=str, help='Set the logging level', choices=['DEBUG', 'INFO', 'WARN', 'ERROR'])
args = parser.parse_args()
setup_logging(args.verbosity)
logger = logging.getLogger('covid.load.testing')
logger.info(args)
print(args)
logger.info('Initialising Load Testing')
class User(GraphQLLocust):
host = args.host
tasks = [CountryBehaviour, TimeseriesBehaviour]
wait_time = between(args.wait_time_min, args.wait_time_max)
logger.info('Setting up Environment and Runner')
env = Environment(user_classes=[User], events=events)
env.create_local_runner()
import logging from locust.log import setup_logging # setup logging loglevel = 'INFO' logfile = None setup_logging(loglevel, logfile) logger = logging.getLogger(__name__)
def main():
# find specified locustfile and make sure it exists, using a very simplified
# command line parser that is only used to parse the -f option
locustfile = parse_locustfile_option()
# import the locustfile
docstring, user_classes, shape_class = load_locustfile(locustfile)
# parse all command line options
options = parse_options()
if options.slave or options.expect_slaves:
sys.stderr.write(
"The --slave/--expect-slaves parameters have been renamed --worker/--expect-workers\n"
)
sys.exit(1)
if options.hatch_rate:
sys.stderr.write(
"[DEPRECATED] The --hatch-rate parameter has been renamed --spawn-rate\n"
)
options.spawn_rate = options.hatch_rate
# setup logging
if not options.skip_log_setup:
if options.loglevel.upper() in [
"DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"
]:
setup_logging(options.loglevel, options.logfile)
else:
sys.stderr.write(
"Invalid --loglevel. Valid values are: DEBUG/INFO/WARNING/ERROR/CRITICAL\n"
)
sys.exit(1)
logger = logging.getLogger(__name__)
greenlet_exception_handler = greenlet_exception_logger(logger)
if options.list_commands:
print("Available Users:")
for name in user_classes:
print(" " + name)
sys.exit(0)
if not user_classes:
logger.error("No User class found!")
sys.exit(1)
# make sure specified User exists
if options.user_classes:
missing = set(options.user_classes) - set(user_classes.keys())
if missing:
logger.error("Unknown User(s): %s\n" % (", ".join(missing)))
sys.exit(1)
else:
names = set(options.user_classes) & set(user_classes.keys())
user_classes = [user_classes[n] for n in names]
else:
# list() call is needed to consume the dict_view object in Python 3
user_classes = list(user_classes.values())
try:
import resource
if resource.getrlimit(resource.RLIMIT_NOFILE)[0] < 10000:
# Increasing the limit to 10000 within a running process should work on at least MacOS.
# It does not work on all OS:es, but we should be no worse off for trying.
resource.setrlimit(resource.RLIMIT_NOFILE,
[10000, resource.RLIM_INFINITY])
except:
logger.warning(
"System open file limit setting is not high enough for load testing, and the OS didn't allow locust to increase it by itself. See https://github.com/locustio/locust/wiki/Installation#increasing-maximum-number-of-open-files-limit for more info."
)
# create locust Environment
environment = create_environment(user_classes,
options,
events=locust.events,
shape_class=shape_class)
if shape_class and (options.num_users or options.spawn_rate
or options.step_load):
logger.error(
"The specified locustfile contains a shape class but a conflicting argument was specified: users, spawn-rate or step-load"
)
sys.exit(1)
if options.show_task_ratio:
print("\n Task ratio per User class")
print("-" * 80)
print_task_ratio(user_classes)
print("\n Total task ratio")
print("-" * 80)
print_task_ratio(user_classes, total=True)
sys.exit(0)
if options.show_task_ratio_json:
from json import dumps
task_data = {
"per_class": get_task_ratio_dict(user_classes),
"total": get_task_ratio_dict(user_classes, total=True)
}
print(dumps(task_data))
sys.exit(0)
if options.step_time:
if not options.step_load:
logger.error(
"The --step-time argument can only be used together with --step-load"
)
sys.exit(1)
if options.worker:
logger.error(
"--step-time should be specified on the master node, and not on worker nodes"
)
sys.exit(1)
try:
options.step_time = parse_timespan(options.step_time)
except ValueError:
logger.error(
"Valid --step-time formats are: 20, 20s, 3m, 2h, 1h20m, 3h30m10s, etc."
)
sys.exit(1)
if options.master:
runner = environment.create_master_runner(
master_bind_host=options.master_bind_host,
master_bind_port=options.master_bind_port,
)
elif options.worker:
try:
runner = environment.create_worker_runner(options.master_host,
options.master_port)
except socket.error as e:
logger.error("Failed to connect to the Locust master: %s", e)
sys.exit(-1)
else:
runner = environment.create_local_runner()
# main_greenlet is pointing to runners.greenlet by default, it will point the web greenlet later if in web mode
main_greenlet = runner.greenlet
if options.run_time:
if not options.headless:
logger.error(
"The --run-time argument can only be used together with --headless"
)
sys.exit(1)
if options.worker:
logger.error(
"--run-time should be specified on the master node, and not on worker nodes"
)
sys.exit(1)
try:
options.run_time = parse_timespan(options.run_time)
except ValueError:
logger.error(
"Valid --run-time formats are: 20, 20s, 3m, 2h, 1h20m, 3h30m10s, etc."
)
sys.exit(1)
def spawn_run_time_limit_greenlet():
logger.info("Run time limit set to %s seconds" % options.run_time)
def timelimit_stop():
logger.info("Time limit reached. Stopping Locust.")
runner.quit()
gevent.spawn_later(
options.run_time,
timelimit_stop).link_exception(greenlet_exception_handler)
if options.csv_prefix:
stats_csv_writer = StatsCSVFileWriter(environment,
stats.PERCENTILES_TO_REPORT,
options.csv_prefix,
options.stats_history_enabled)
else:
stats_csv_writer = StatsCSV(environment, stats.PERCENTILES_TO_REPORT)
# start Web UI
if not options.headless and not options.worker:
# spawn web greenlet
protocol = "https" if options.tls_cert and options.tls_key else "http"
try:
if options.web_host == "*":
# special check for "*" so that we're consistent with --master-bind-host
web_host = ''
else:
web_host = options.web_host
if web_host:
logger.info("Starting web interface at %s://%s:%s" %
(protocol, web_host, options.web_port))
else:
logger.info(
"Starting web interface at %s://0.0.0.0:%s (accepting connections from all network interfaces)"
% (protocol, options.web_port))
web_ui = environment.create_web_ui(
host=web_host,
port=options.web_port,
auth_credentials=options.web_auth,
tls_cert=options.tls_cert,
tls_key=options.tls_key,
stats_csv_writer=stats_csv_writer,
)
except AuthCredentialsError:
logger.error(
"Credentials supplied with --web-auth should have the format: username:password"
)
sys.exit(1)
else:
main_greenlet = web_ui.greenlet
else:
web_ui = None
# Fire locust init event which can be used by end-users' code to run setup code that
# need access to the Environment, Runner or WebUI
environment.events.init.fire(environment=environment,
runner=runner,
web_ui=web_ui)
if options.headless:
# headless mode
if options.master:
# wait for worker nodes to connect
while len(runner.clients.ready) < options.expect_workers:
logging.info(
"Waiting for workers to be ready, %s of %s connected",
len(runner.clients.ready), options.expect_workers)
time.sleep(1)
if not options.worker:
# apply headless mode defaults
if options.num_users is None:
options.num_users = 1
if options.spawn_rate is None:
options.spawn_rate = 1
if options.step_users is None:
options.step_users = 1
# start the test
if options.step_time:
runner.start_stepload(options.num_users, options.spawn_rate,
options.step_users, options.step_time)
if environment.shape_class:
environment.runner.start_shape()
else:
runner.start(options.num_users, options.spawn_rate)
if options.run_time:
spawn_run_time_limit_greenlet()
stats_printer_greenlet = None
if not options.only_summary and (options.print_stats or
(options.headless
and not options.worker)):
# spawn stats printing greenlet
stats_printer_greenlet = gevent.spawn(stats_printer(runner.stats))
stats_printer_greenlet.link_exception(greenlet_exception_handler)
if options.csv_prefix:
gevent.spawn(stats_csv_writer.stats_writer).link_exception(
greenlet_exception_handler)
gevent.spawn(stats_history, runner)
def shutdown():
"""
Shut down locust by firing quitting event, printing/writing stats and exiting
"""
logger.info("Running teardowns...")
environment.events.quitting.fire(environment=environment, reverse=True)
# determine the process exit code
if log.unhandled_greenlet_exception:
code = 2
elif environment.process_exit_code is not None:
code = environment.process_exit_code
elif len(runner.errors) or len(runner.exceptions):
code = options.exit_code_on_error
else:
code = 0
logger.info("Shutting down (exit code %s), bye." % code)
if stats_printer_greenlet is not None:
stats_printer_greenlet.kill(block=False)
logger.info("Cleaning up runner...")
if runner is not None:
runner.quit()
print_stats(runner.stats, current=False)
print_percentile_stats(runner.stats)
print_error_report(runner.stats)
sys.exit(code)
# install SIGTERM handler
def sig_term_handler():
logger.info("Got SIGTERM signal")
shutdown()
gevent.signal_handler(signal.SIGTERM, sig_term_handler)
try:
logger.info("Starting Locust %s" % version)
main_greenlet.join()
shutdown()
except KeyboardInterrupt as e:
shutdown()
