def locust_executor(host,
port,
collection_name,
connection_type="single",
run_params=None):
m = MilvusClient(host=host, port=port, collection_name=collection_name)
MyUser.tasks = {}
tasks = run_params["tasks"]
for op, weight in tasks.items():
task = {eval("Tasks." + op): weight}
MyUser.tasks.update(task)
logger.error(MyUser.tasks)
# MyUser.tasks = {Tasks.query: 1, Tasks.flush: 1}
MyUser.client = MilvusTask(host=host,
port=port,
collection_name=collection_name,
connection_type=connection_type,
m=m)
env = Environment(events=events, user_classes=[MyUser])
runner = env.create_local_runner()
# setup logging
# setup_logging("WARNING", "/dev/null")
setup_logging("WARNING", "/dev/null")
greenlet_exception_logger(logger=logger)
gevent.spawn(stats_printer(env.stats))
# env.create_web_ui("127.0.0.1", 8089)
# gevent.spawn(stats_printer(env.stats), env, "test", full_history=True)
# events.init.fire(environment=env, runner=runner)
clients_num = run_params["clients_num"]
spawn_rate = run_params["spawn_rate"]
during_time = run_params["during_time"]
runner.start(clients_num, spawn_rate=spawn_rate)
gevent.spawn_later(during_time, lambda: runner.quit())
runner.greenlet.join()
print_stats(env.stats)
result = {
"rps": round(env.stats.total.current_rps, 1),
"fail_ratio": env.stats.total.fail_ratio,
"max_response_time": round(env.stats.total.max_response_time, 1),
"min_response_time": round(env.stats.total.avg_response_time, 1)
}
runner.stop()
return result
def test_greenlet_exception_logger(self, mocked_stderr):
def thread():
raise ValueError("Boom!?")
logger = getLogger("greenlet_test_logger")
g = gevent.spawn(thread)
g.link_exception(greenlet_exception_logger(logger))
g.join()
self.assertEqual(1, len(self.mocked_log.critical))
msg = self.mocked_log.critical[0]
self.assertIn("Unhandled exception in greenlet: ", msg["message"])
self.assertTrue(isinstance(msg["exc_info"][1], ValueError))
self.assertIn("Boom!?", str(msg["exc_info"][1]))
import logging
import sys
TARGET_APP_DIR = os.getenv('TARGET_DIR' , './app')
LOCUST_RUN_TIME = os.getenv('LOCUST_RUN_TIME' , 20)
LOCUST_USER_COUNT = os.getenv('LOCUST_USER_COUNT' , 10)
LOCUST_SPAWN_RATE = os.getenv('LOCUST_SPAWN_RATE' , 5)
LOCUST_SPAWN_RATE = os.getenv('LOCUST_SPAWN_RATE' , 5)
setup_logging("INFO", None)
logger = logging.getLogger()
greenlet_exception_handler = greenlet_exception_logger(logger)
options = parse_options()
from app.OnceUser import WebsiteUser
env = Environment(user_classes=[WebsiteUser], host="http://www.google.com", reset_stats=True, parsed_options=options)
if os.getenv('OnceUser.py'):
pass
else:
pass
#raise ValueError('test was not loaded')
stats_csv_writer = StatsCSVFileWriter(env, stats.PERCENTILES_TO_REPORT, './app/reports', options.stats_history_enabled)
print(type(stats_csv_writer))
env.create_local_runner()
def main():
# find specified locustfile and make sure it exists, using a very simplified
# command line parser that is only used to parse the -f option
locustfile = parse_locustfile_option()
# import the locustfile
docstring, user_classes, shape_class = load_locustfile(locustfile)
# parse all command line options
options = parse_options()
if options.slave or options.expect_slaves:
sys.stderr.write(
"The --slave/--expect-slaves parameters have been renamed --worker/--expect-workers\n"
)
sys.exit(1)
if options.hatch_rate:
sys.stderr.write(
"[DEPRECATED] The --hatch-rate parameter has been renamed --spawn-rate\n"
)
options.spawn_rate = options.hatch_rate
# setup logging
if not options.skip_log_setup:
if options.loglevel.upper() in [
"DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"
]:
setup_logging(options.loglevel, options.logfile)
else:
sys.stderr.write(
"Invalid --loglevel. Valid values are: DEBUG/INFO/WARNING/ERROR/CRITICAL\n"
)
sys.exit(1)
logger = logging.getLogger(__name__)
greenlet_exception_handler = greenlet_exception_logger(logger)
if options.list_commands:
print("Available Users:")
for name in user_classes:
print(" " + name)
sys.exit(0)
if not user_classes:
logger.error("No User class found!")
sys.exit(1)
# make sure specified User exists
if options.user_classes:
missing = set(options.user_classes) - set(user_classes.keys())
if missing:
logger.error("Unknown User(s): %s\n" % (", ".join(missing)))
sys.exit(1)
else:
names = set(options.user_classes) & set(user_classes.keys())
user_classes = [user_classes[n] for n in names]
else:
# list() call is needed to consume the dict_view object in Python 3
user_classes = list(user_classes.values())
try:
import resource
if resource.getrlimit(resource.RLIMIT_NOFILE)[0] < 10000:
# Increasing the limit to 10000 within a running process should work on at least MacOS.
# It does not work on all OS:es, but we should be no worse off for trying.
resource.setrlimit(resource.RLIMIT_NOFILE,
[10000, resource.RLIM_INFINITY])
except:
logger.warning(
"System open file limit setting is not high enough for load testing, and the OS didn't allow locust to increase it by itself. See https://github.com/locustio/locust/wiki/Installation#increasing-maximum-number-of-open-files-limit for more info."
)
# create locust Environment
environment = create_environment(user_classes,
options,
events=locust.events,
shape_class=shape_class)
if shape_class and (options.num_users or options.spawn_rate
or options.step_load):
logger.error(
"The specified locustfile contains a shape class but a conflicting argument was specified: users, spawn-rate or step-load"
)
sys.exit(1)
if options.show_task_ratio:
print("\n Task ratio per User class")
print("-" * 80)
print_task_ratio(user_classes)
print("\n Total task ratio")
print("-" * 80)
print_task_ratio(user_classes, total=True)
sys.exit(0)
if options.show_task_ratio_json:
from json import dumps
task_data = {
"per_class": get_task_ratio_dict(user_classes),
"total": get_task_ratio_dict(user_classes, total=True)
}
print(dumps(task_data))
sys.exit(0)
if options.step_time:
if not options.step_load:
logger.error(
"The --step-time argument can only be used together with --step-load"
)
sys.exit(1)
if options.worker:
logger.error(
"--step-time should be specified on the master node, and not on worker nodes"
)
sys.exit(1)
try:
options.step_time = parse_timespan(options.step_time)
except ValueError:
logger.error(
"Valid --step-time formats are: 20, 20s, 3m, 2h, 1h20m, 3h30m10s, etc."
)
sys.exit(1)
if options.master:
runner = environment.create_master_runner(
master_bind_host=options.master_bind_host,
master_bind_port=options.master_bind_port,
)
elif options.worker:
try:
runner = environment.create_worker_runner(options.master_host,
options.master_port)
except socket.error as e:
logger.error("Failed to connect to the Locust master: %s", e)
sys.exit(-1)
else:
runner = environment.create_local_runner()
# main_greenlet is pointing to runners.greenlet by default, it will point the web greenlet later if in web mode
main_greenlet = runner.greenlet
if options.run_time:
if not options.headless:
logger.error(
"The --run-time argument can only be used together with --headless"
)
sys.exit(1)
if options.worker:
logger.error(
"--run-time should be specified on the master node, and not on worker nodes"
)
sys.exit(1)
try:
options.run_time = parse_timespan(options.run_time)
except ValueError:
logger.error(
"Valid --run-time formats are: 20, 20s, 3m, 2h, 1h20m, 3h30m10s, etc."
)
sys.exit(1)
def spawn_run_time_limit_greenlet():
logger.info("Run time limit set to %s seconds" % options.run_time)
def timelimit_stop():
logger.info("Time limit reached. Stopping Locust.")
runner.quit()
gevent.spawn_later(
options.run_time,
timelimit_stop).link_exception(greenlet_exception_handler)
if options.csv_prefix:
stats_csv_writer = StatsCSVFileWriter(environment,
stats.PERCENTILES_TO_REPORT,
options.csv_prefix,
options.stats_history_enabled)
else:
stats_csv_writer = StatsCSV(environment, stats.PERCENTILES_TO_REPORT)
# start Web UI
if not options.headless and not options.worker:
# spawn web greenlet
protocol = "https" if options.tls_cert and options.tls_key else "http"
try:
if options.web_host == "*":
# special check for "*" so that we're consistent with --master-bind-host
web_host = ''
else:
web_host = options.web_host
if web_host:
logger.info("Starting web interface at %s://%s:%s" %
(protocol, web_host, options.web_port))
else:
logger.info(
"Starting web interface at %s://0.0.0.0:%s (accepting connections from all network interfaces)"
% (protocol, options.web_port))
web_ui = environment.create_web_ui(
host=web_host,
port=options.web_port,
auth_credentials=options.web_auth,
tls_cert=options.tls_cert,
tls_key=options.tls_key,
stats_csv_writer=stats_csv_writer,
)
except AuthCredentialsError:
logger.error(
"Credentials supplied with --web-auth should have the format: username:password"
)
sys.exit(1)
else:
main_greenlet = web_ui.greenlet
else:
web_ui = None
# Fire locust init event which can be used by end-users' code to run setup code that
# need access to the Environment, Runner or WebUI
environment.events.init.fire(environment=environment,
runner=runner,
web_ui=web_ui)
if options.headless:
# headless mode
if options.master:
# wait for worker nodes to connect
while len(runner.clients.ready) < options.expect_workers:
logging.info(
"Waiting for workers to be ready, %s of %s connected",
len(runner.clients.ready), options.expect_workers)
time.sleep(1)
if not options.worker:
# apply headless mode defaults
if options.num_users is None:
options.num_users = 1
if options.spawn_rate is None:
options.spawn_rate = 1
if options.step_users is None:
options.step_users = 1
# start the test
if options.step_time:
runner.start_stepload(options.num_users, options.spawn_rate,
options.step_users, options.step_time)
if environment.shape_class:
environment.runner.start_shape()
else:
runner.start(options.num_users, options.spawn_rate)
if options.run_time:
spawn_run_time_limit_greenlet()
stats_printer_greenlet = None
if not options.only_summary and (options.print_stats or
(options.headless
and not options.worker)):
# spawn stats printing greenlet
stats_printer_greenlet = gevent.spawn(stats_printer(runner.stats))
stats_printer_greenlet.link_exception(greenlet_exception_handler)
if options.csv_prefix:
gevent.spawn(stats_csv_writer.stats_writer).link_exception(
greenlet_exception_handler)
gevent.spawn(stats_history, runner)
def shutdown():
"""
Shut down locust by firing quitting event, printing/writing stats and exiting
"""
logger.info("Running teardowns...")
environment.events.quitting.fire(environment=environment, reverse=True)
# determine the process exit code
if log.unhandled_greenlet_exception:
code = 2
elif environment.process_exit_code is not None:
code = environment.process_exit_code
elif len(runner.errors) or len(runner.exceptions):
code = options.exit_code_on_error
else:
code = 0
logger.info("Shutting down (exit code %s), bye." % code)
if stats_printer_greenlet is not None:
stats_printer_greenlet.kill(block=False)
logger.info("Cleaning up runner...")
if runner is not None:
runner.quit()
print_stats(runner.stats, current=False)
print_percentile_stats(runner.stats)
print_error_report(runner.stats)
sys.exit(code)
# install SIGTERM handler
def sig_term_handler():
logger.info("Got SIGTERM signal")
shutdown()
gevent.signal_handler(signal.SIGTERM, sig_term_handler)
try:
logger.info("Starting Locust %s" % version)
main_greenlet.join()
shutdown()
except KeyboardInterrupt as e:
shutdown()