def run(self):
"""
Run the load test.
"""
if self.settings.run_time:
self.set_run_time_in_sec(run_time_str=self.settings.run_time)
logger.info("Run time limit set to %s seconds" %
self.run_time_in_sec)
def timelimit_stop():
logger.info(
"Run time limit reached: %s seconds. Stopping Locust Runner."
% self.run_time_in_sec)
self.env.runner.quit()
self.end_time = time.time()
logger.info("Locust completed %s requests with %s errors" %
(self.env.runner.stats.num_requests,
len(self.env.runner.errors)))
logger.info(json.dumps(self.stats()))
gevent.spawn_later(self.run_time_in_sec, timelimit_stop)
try:
logger.info("Starting Locust with settings %s " %
vars(self.settings))
self.env = Environment(
user_classes=self.settings.classes,
host=self.settings.host,
tags=self.settings.tags,
exclude_tags=self.settings.exclude_tags,
reset_stats=self.settings.reset_stats,
step_load=self.settings.step_load,
stop_timeout=self.settings.stop_timeout,
)
self.env.create_local_runner()
gevent.spawn(stats_printer(self.env.stats))
self.env.runner.start(user_count=self.settings.num_users,
hatch_rate=self.settings.hatch_rate)
self.start_time = time.time()
self.env.runner.greenlet.join()
except Exception as e:
logger.error("Locust exception {0}".format(repr(e)))
finally:
self.env.events.quitting.fire()
def locust_executor(host,
port,
collection_name,
connection_type="single",
run_params=None):
m = MilvusClient(host=host, port=port, collection_name=collection_name)
MyUser.tasks = {}
MyUser.op_info = run_params["op_info"]
MyUser.params = {}
tasks = run_params["tasks"]
for op, value in tasks.items():
task = {eval("Tasks." + op): value["weight"]}
MyUser.tasks.update(task)
MyUser.params[op] = value["params"] if "params" in value else None
logger.info(MyUser.tasks)
MyUser.tasks = {Tasks.load: 1, Tasks.flush: 1}
MyUser.client = MilvusTask(host=host,
port=port,
collection_name=collection_name,
connection_type=connection_type,
m=m)
# MyUser.info = m.get_info(collection_name)
env = Environment(events=events, user_classes=[MyUser])
runner = env.create_local_runner()
# setup logging
# setup_logging("WARNING", "/dev/null")
# greenlet_exception_logger(logger=logger)
gevent.spawn(stats_printer(env.stats))
# env.create_web_ui("127.0.0.1", 8089)
# gevent.spawn(stats_printer(env.stats), env, "test", full_history=True)
# events.init.fire(environment=env, runner=runner)
clients_num = run_params["clients_num"]
spawn_rate = run_params["spawn_rate"]
during_time = run_params["during_time"]
runner.start(clients_num, spawn_rate=spawn_rate)
gevent.spawn_later(during_time, lambda: runner.quit())
runner.greenlet.join()
print_stats(env.stats)
result = {
"rps": round(env.stats.total.current_rps, 1),
"fail_ratio": env.stats.total.fail_ratio,
"max_response_time": round(env.stats.total.max_response_time, 1),
"avg_response_time": round(env.stats.total.avg_response_time, 1)
}
runner.stop()
return result
def web_ui():
env.create_web_ui("127.0.0.1", 8089)
# start a greenlet that periodically outputs the current stats
gevent.spawn(stats_printer(env.stats))
# start a greenlet that save current stats to history
gevent.spawn(stats_history, env.runner)
# start the test
env.runner.start(1, spawn_rate=10)
# in 60 seconds stop the runner
gevent.spawn_later(30, lambda: env.runner.quit())
# wait for the greenlets
env.runner.greenlet.join()
# stop the web server for good measures
env.web_ui.stop()
def run_locust(test_file=None, directory=None, profile=None):
from locust.env import Environment
from locust.stats import stats_printer
import gevent
env = Environment(user_classes=[DocStorageUser])
env.create_local_runner()
setattr(env, "source_file", test_file or SOURCE_FILE)
setattr(env, "working_directory", directory or WORKING_DIRECTORY)
setattr(env, "aws_profile", profile)
env.runner.start(CONCURRENT_USERS, hatch_rate=1)
env.runner.greenlet.join()
# start a greenlet that periodically outputs the current stats
gevent.spawn(stats_printer(env.stats))
# in 60 seconds stop the runner
gevent.spawn_later(60, lambda: env.runner.quit())
# wait for the greenlets
env.runner.greenlet.join()
def locust_run(c, r, t, base, path, webui):
class User(HttpUser):
wait_time = constant(1)
host = base
@task
def my_task(self):
self.client.get(path)
env = Environment(user_classes=[User])
env.create_local_runner()
if webui:
env.create_web_ui("127.0.0.1", 8089)
webbrowser.open('http://127.0.0.1:8089')
gevent.spawn(stats_printer(env.stats))
gevent.spawn(stats_history, env.runner)
env.runner.start(c, spawn_rate=c)
gevent.spawn_later(t, lambda: env.runner.quit())
env.runner.greenlet.join()
if webui:
env.web_ui.stop()
class task_set(TaskSet):
@task
def my_task(self):
self.client.get("/")
@task
def task_404(self):
self.client.get("/non-existing-path")
# setup Environment and Runner
env = Environment()
runner = LocalLocustRunner(environment=env, locust_classes=[User])
# start a WebUI instance
web_ui = WebUI(environment=env)
gevent.spawn(lambda: web_ui.start("127.0.0.1", 8089))
# TODO: fix
#def on_request_success(request_type, name, response_time, response_length, **kwargs):
# report_to_grafana("%_%s" % (request_type, name), response_time)
#env.events.request_succes.add_listener(on_request_success)
# start a greenlet that periodically outputs the current stats
gevent.spawn(stats_printer(runner.stats))
# start the test
runner.start(1, hatch_rate=10)
# wait for the greenlets (indefinitely)
runner.greenlet.join()
self.client.get("/non-existing-path")
@task
def my_task(self):
self.client.get("/api/Url/1")
# setup Environment and Runner
env = Environment(user_classes=[User])
env.create_local_runner()
# start a WebUI instance
env.create_web_ui("127.0.0.1", 8089)
# start a greenlet that periodically outputs the current stats
gevent.spawn(stats_printer(env.stats))
# start a greenlet that save current stats to history
gevent.spawn(stats_history, env.runner)
# start the test
env.runner.start(1, spawn_rate=100)
# in 60 seconds stop the runner
gevent.spawn_later(60, lambda: env.runner.quit())
# wait for the greenlets
env.runner.greenlet.join()
# stop the web server for good measures
env.web_ui.stop()
def locust_executor(host,
port,
collection_name,
connection_type="single",
run_params=None):
m = MilvusClient(host=host, port=port, collection_name=collection_name)
MyUser.tasks = {}
MyUser.op_info = run_params["op_info"]
MyUser.params = {}
tasks = run_params["tasks"]
for op, value in tasks.items():
task = {eval("Tasks." + op): value["weight"]}
MyUser.tasks.update(task)
MyUser.params[op] = value["params"] if "params" in value else None
logger.info(MyUser.tasks)
MyUser.values = {
"ids": [random.randint(1000000, 10000000) for _ in range(nb)],
"get_ids": [random.randint(1, 10000000) for _ in range(nb)],
"X": utils.generate_vectors(nq, MyUser.op_info["dimension"])
}
# MyUser.tasks = {Tasks.query: 1, Tasks.flush: 1}
MyUser.client = MilvusTask(host=host,
port=port,
collection_name=collection_name,
connection_type=connection_type,
m=m)
if "load_shape" in run_params and run_params["load_shape"]:
test = StepLoadShape()
test.init(run_params["step_time"], run_params["step_load"],
run_params["spawn_rate"], run_params["during_time"])
env = Environment(events=events,
user_classes=[MyUser],
shape_class=test)
runner = env.create_local_runner()
env.runner.start_shape()
else:
env = Environment(events=events, user_classes=[MyUser])
runner = env.create_local_runner()
# setup logging
# setup_logging("WARNING", "/dev/null")
# greenlet_exception_logger(logger=logger)
gevent.spawn(stats_printer(env.stats))
# env.create_web_ui("127.0.0.1", 8089)
# gevent.spawn(stats_printer(env.stats), env, "test", full_history=True)
# events.init.fire(environment=env, runner=runner)
clients_num = run_params[
"clients_num"] if "clients_num" in run_params else 0
step_load = run_params["step_load"] if "step_load" in run_params else 0
step_time = run_params["step_time"] if "step_time" in run_params else 0
spawn_rate = run_params["spawn_rate"]
during_time = run_params["during_time"]
runner.start(clients_num, spawn_rate=spawn_rate)
gevent.spawn_later(during_time, lambda: runner.quit())
runner.greenlet.join()
print_stats(env.stats)
result = {
"rps": round(env.stats.total.current_rps,
1), # Number of interface requests per second
"fail_ratio":
env.stats.total.fail_ratio, # Interface request failure rate
"max_response_time": round(env.stats.total.max_response_time,
1), # Maximum interface response time
"avg_response_time": round(env.stats.total.avg_response_time,
1) # ratio of average response time
}
runner.stop()
return result
def main(nosave, fault_profile, measurement_count, load_duration,
locust_user_count, locust_spawn_rate, cluster_type, comment):
check_working_dir()
# Save the start time of the measurement sequence
start_time = datetime.now().strftime("%m-%d-%Y_%H-%M-%S.%f")
# lint helm chart
try:
logging.info('Linting Helm chart...')
subprocess.check_output(['helm', 'lint', '../charts/kubedepend-chaos'])
logging.info('Linting Helm chart finished OK')
except subprocess.CalledProcessError as error:
logging.error('Helm lint failed, exiting...')
exit()
# assembling helm value options according to fault profile
helm_value_sets = assemble_helm_set_options(fault_profile)
# filter out emtpy strings
helm_command = [x for x in HELM_COMMAND_FIX_PART + helm_value_sets if x]
# Save current stack into archive
if not nosave:
save_helm_chart(helm_command=helm_command)
archive_stack(start_time)
sequence_result = MeasurementSequenceResult(
start_time=start_time,
fault_profile=fault_profile,
cluster_type=cluster_type,
load_duration=load_duration,
locust_user_count=locust_user_count,
locust_spawn_rate=locust_spawn_rate,
comment=comment)
for i in range(measurement_count):
logging.info(f'Start measurement #{i + 1}')
logging.info('Waiting for stable system state...')
wait_for_stable_state()
# Initialize measurement result
measurement_result = MeasurementResult()
# Setup Locust objects
# setup Environment and Runner
env = Environment(user_classes=[User])
env.create_local_runner()
# start a greenlet that periodically outputs the current stats
gevent.spawn(stats_printer(env.stats))
# start a greenlet that save current stats to history
gevent.spawn(stats_history, env.runner)
logging.info('Creating chaos objects...')
subprocess.run(helm_command)
logging.info('Chaos objects applied.')
logging.info('Generating load...')
# start the test
env.runner.start(user_count=locust_user_count,
spawn_rate=locust_spawn_rate)
# in 'duartion' seconds stop the runner
gevent.spawn_later(load_duration, lambda: env.runner.quit())
# wait for the greenlets
env.runner.greenlet.join()
logging.info('Load generation finished')
# get dependability metrics
metrics = get_dependability_metrics(load_duration)
# add metrics to measurement result
measurement_result.backend_metrics = metrics
# end measurement (fill end_time attribute)
measurement_result.end()
# add measurement result to sequence result
sequence_result.add_measurement_result(measurement_result)
# save dependability metrics
logging.info('Deleting chaos objects...')
subprocess.run(
['helm', 'delete', 'kubedepend-chaos', '-n', 'chaos-testing'])
logging.info('Chaos objects deleted.')
if not nosave:
sequence_result.save_results('results/results.csv')
# TODO stop running greenlets (stats)
logging.info('Test finished')
def main():
# find specified locustfile and make sure it exists, using a very simplified
# command line parser that is only used to parse the -f option
locustfile = parse_locustfile_option()
# import the locustfile
docstring, user_classes, shape_class = load_locustfile(locustfile)
# parse all command line options
options = parse_options()
if options.slave or options.expect_slaves:
sys.stderr.write(
"The --slave/--expect-slaves parameters have been renamed --worker/--expect-workers\n"
)
sys.exit(1)
if options.hatch_rate:
sys.stderr.write(
"[DEPRECATED] The --hatch-rate parameter has been renamed --spawn-rate\n"
)
options.spawn_rate = options.hatch_rate
# setup logging
if not options.skip_log_setup:
if options.loglevel.upper() in [
"DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"
]:
setup_logging(options.loglevel, options.logfile)
else:
sys.stderr.write(
"Invalid --loglevel. Valid values are: DEBUG/INFO/WARNING/ERROR/CRITICAL\n"
)
sys.exit(1)
logger = logging.getLogger(__name__)
greenlet_exception_handler = greenlet_exception_logger(logger)
if options.list_commands:
print("Available Users:")
for name in user_classes:
print(" " + name)
sys.exit(0)
if not user_classes:
logger.error("No User class found!")
sys.exit(1)
# make sure specified User exists
if options.user_classes:
missing = set(options.user_classes) - set(user_classes.keys())
if missing:
logger.error("Unknown User(s): %s\n" % (", ".join(missing)))
sys.exit(1)
else:
names = set(options.user_classes) & set(user_classes.keys())
user_classes = [user_classes[n] for n in names]
else:
# list() call is needed to consume the dict_view object in Python 3
user_classes = list(user_classes.values())
try:
import resource
if resource.getrlimit(resource.RLIMIT_NOFILE)[0] < 10000:
# Increasing the limit to 10000 within a running process should work on at least MacOS.
# It does not work on all OS:es, but we should be no worse off for trying.
resource.setrlimit(resource.RLIMIT_NOFILE,
[10000, resource.RLIM_INFINITY])
except:
logger.warning(
"System open file limit setting is not high enough for load testing, and the OS didn't allow locust to increase it by itself. See https://github.com/locustio/locust/wiki/Installation#increasing-maximum-number-of-open-files-limit for more info."
)
# create locust Environment
environment = create_environment(user_classes,
options,
events=locust.events,
shape_class=shape_class)
if shape_class and (options.num_users or options.spawn_rate
or options.step_load):
logger.error(
"The specified locustfile contains a shape class but a conflicting argument was specified: users, spawn-rate or step-load"
)
sys.exit(1)
if options.show_task_ratio:
print("\n Task ratio per User class")
print("-" * 80)
print_task_ratio(user_classes)
print("\n Total task ratio")
print("-" * 80)
print_task_ratio(user_classes, total=True)
sys.exit(0)
if options.show_task_ratio_json:
from json import dumps
task_data = {
"per_class": get_task_ratio_dict(user_classes),
"total": get_task_ratio_dict(user_classes, total=True)
}
print(dumps(task_data))
sys.exit(0)
if options.step_time:
if not options.step_load:
logger.error(
"The --step-time argument can only be used together with --step-load"
)
sys.exit(1)
if options.worker:
logger.error(
"--step-time should be specified on the master node, and not on worker nodes"
)
sys.exit(1)
try:
options.step_time = parse_timespan(options.step_time)
except ValueError:
logger.error(
"Valid --step-time formats are: 20, 20s, 3m, 2h, 1h20m, 3h30m10s, etc."
)
sys.exit(1)
if options.master:
runner = environment.create_master_runner(
master_bind_host=options.master_bind_host,
master_bind_port=options.master_bind_port,
)
elif options.worker:
try:
runner = environment.create_worker_runner(options.master_host,
options.master_port)
except socket.error as e:
logger.error("Failed to connect to the Locust master: %s", e)
sys.exit(-1)
else:
runner = environment.create_local_runner()
# main_greenlet is pointing to runners.greenlet by default, it will point the web greenlet later if in web mode
main_greenlet = runner.greenlet
if options.run_time:
if not options.headless:
logger.error(
"The --run-time argument can only be used together with --headless"
)
sys.exit(1)
if options.worker:
logger.error(
"--run-time should be specified on the master node, and not on worker nodes"
)
sys.exit(1)
try:
options.run_time = parse_timespan(options.run_time)
except ValueError:
logger.error(
"Valid --run-time formats are: 20, 20s, 3m, 2h, 1h20m, 3h30m10s, etc."
)
sys.exit(1)
def spawn_run_time_limit_greenlet():
logger.info("Run time limit set to %s seconds" % options.run_time)
def timelimit_stop():
logger.info("Time limit reached. Stopping Locust.")
runner.quit()
gevent.spawn_later(
options.run_time,
timelimit_stop).link_exception(greenlet_exception_handler)
if options.csv_prefix:
stats_csv_writer = StatsCSVFileWriter(environment,
stats.PERCENTILES_TO_REPORT,
options.csv_prefix,
options.stats_history_enabled)
else:
stats_csv_writer = StatsCSV(environment, stats.PERCENTILES_TO_REPORT)
# start Web UI
if not options.headless and not options.worker:
# spawn web greenlet
protocol = "https" if options.tls_cert and options.tls_key else "http"
try:
if options.web_host == "*":
# special check for "*" so that we're consistent with --master-bind-host
web_host = ''
else:
web_host = options.web_host
if web_host:
logger.info("Starting web interface at %s://%s:%s" %
(protocol, web_host, options.web_port))
else:
logger.info(
"Starting web interface at %s://0.0.0.0:%s (accepting connections from all network interfaces)"
% (protocol, options.web_port))
web_ui = environment.create_web_ui(
host=web_host,
port=options.web_port,
auth_credentials=options.web_auth,
tls_cert=options.tls_cert,
tls_key=options.tls_key,
stats_csv_writer=stats_csv_writer,
)
except AuthCredentialsError:
logger.error(
"Credentials supplied with --web-auth should have the format: username:password"
)
sys.exit(1)
else:
main_greenlet = web_ui.greenlet
else:
web_ui = None
# Fire locust init event which can be used by end-users' code to run setup code that
# need access to the Environment, Runner or WebUI
environment.events.init.fire(environment=environment,
runner=runner,
web_ui=web_ui)
if options.headless:
# headless mode
if options.master:
# wait for worker nodes to connect
while len(runner.clients.ready) < options.expect_workers:
logging.info(
"Waiting for workers to be ready, %s of %s connected",
len(runner.clients.ready), options.expect_workers)
time.sleep(1)
if not options.worker:
# apply headless mode defaults
if options.num_users is None:
options.num_users = 1
if options.spawn_rate is None:
options.spawn_rate = 1
if options.step_users is None:
options.step_users = 1
# start the test
if options.step_time:
runner.start_stepload(options.num_users, options.spawn_rate,
options.step_users, options.step_time)
if environment.shape_class:
environment.runner.start_shape()
else:
runner.start(options.num_users, options.spawn_rate)
if options.run_time:
spawn_run_time_limit_greenlet()
stats_printer_greenlet = None
if not options.only_summary and (options.print_stats or
(options.headless
and not options.worker)):
# spawn stats printing greenlet
stats_printer_greenlet = gevent.spawn(stats_printer(runner.stats))
stats_printer_greenlet.link_exception(greenlet_exception_handler)
if options.csv_prefix:
gevent.spawn(stats_csv_writer.stats_writer).link_exception(
greenlet_exception_handler)
gevent.spawn(stats_history, runner)
def shutdown():
"""
Shut down locust by firing quitting event, printing/writing stats and exiting
"""
logger.info("Running teardowns...")
environment.events.quitting.fire(environment=environment, reverse=True)
# determine the process exit code
if log.unhandled_greenlet_exception:
code = 2
elif environment.process_exit_code is not None:
code = environment.process_exit_code
elif len(runner.errors) or len(runner.exceptions):
code = options.exit_code_on_error
else:
code = 0
logger.info("Shutting down (exit code %s), bye." % code)
if stats_printer_greenlet is not None:
stats_printer_greenlet.kill(block=False)
logger.info("Cleaning up runner...")
if runner is not None:
runner.quit()
print_stats(runner.stats, current=False)
print_percentile_stats(runner.stats)
print_error_report(runner.stats)
sys.exit(code)
# install SIGTERM handler
def sig_term_handler():
logger.info("Got SIGTERM signal")
shutdown()
gevent.signal_handler(signal.SIGTERM, sig_term_handler)
try:
logger.info("Starting Locust %s" % version)
main_greenlet.join()
shutdown()
except KeyboardInterrupt as e:
shutdown()
