def test_ungraceful_close_in_greenlet(self):
group = Group()
UnaryCallWithSleep = self._channel.unary_unary(
_UNARY_CALL_METHOD_WITH_SLEEP,
request_serializer=messages_pb2.SimpleRequest.SerializeToString,
response_deserializer=messages_pb2.SimpleResponse.FromString,
)
greenlet = group.spawn(self._run_client, UnaryCallWithSleep)
# release loop so that greenlet can take control
gevent.sleep()
group.killone(greenlet)
self.assertFalse(self._unhandled_exception, "Unhandled GreenletExit")
def test_kill_greenlet_with_generic_exception(self):
group = Group()
UnaryCallWithSleep = self._channel.unary_unary(
_UNARY_CALL_METHOD_WITH_SLEEP,
request_serializer=messages_pb2.SimpleRequest.SerializeToString,
response_deserializer=messages_pb2.SimpleResponse.FromString,
)
greenlet = group.spawn(self._run_client, UnaryCallWithSleep)
# release loop so that greenlet can take control
gevent.sleep()
group.killone(greenlet, exception=Exception)
self.assertFalse(self._unhandled_exception, "Unhandled exception")
self.assertRaises(Exception, greenlet.get)
def test_graceful_close_in_greenlet(self):
group = Group()
stub = test_pb2_grpc.TestServiceStub(self._channel)
greenlet = group.spawn(self._run_client, stub.UnaryCall)
# release loop so that greenlet can take control
gevent.sleep()
self._channel.close()
group.killone(greenlet)
self.assertFalse(self._unhandled_exception, "Unhandled GreenletExit")
try:
greenlet.get()
except Exception as e: # pylint: disable=broad-except
self.fail(f"Unexpected exception in greenlet: {e}")
class LocustRunner(object):
def __init__(self, locust_classes, options):
self.options = options
self.locust_classes = locust_classes
self.hatch_rate = options.hatch_rate
self.num_clients = options.num_clients
self.host = options.host
self.locusts = Group()
self.greenlet = self.locusts
self.state = STATE_INIT
self.hatching_greenlet = None
self.exceptions = {}
self.stats = global_stats
# register listener that resets stats when hatching is complete
def on_hatch_complete(user_count):
self.state = STATE_RUNNING
if self.options.reset_stats:
logger.info("Resetting stats\n")
self.stats.reset_all()
events.hatch_complete += on_hatch_complete
@property
def request_stats(self):
return self.stats.entries
@property
def errors(self):
return self.stats.errors
@property
def user_count(self):
return len(self.locusts)
def weight_locusts(self, amount, stop_timeout=None):
"""
Distributes the amount of locusts for each WebLocust-class according to it's weight
returns a list "bucket" with the weighted locusts
"""
bucket = []
weight_sum = sum((locust.weight for locust in self.locust_classes
if locust.task_set))
for locust in self.locust_classes:
if not locust.task_set:
warnings.warn(
"Notice: Found Locust class (%s) got no task_set. Skipping..."
% locust.__name__)
continue
if self.host is not None:
locust.host = self.host
if stop_timeout is not None:
locust.stop_timeout = stop_timeout
# create locusts depending on weight
percent = locust.weight / float(weight_sum)
num_locusts = int(round(amount * percent))
bucket.extend([locust for x in xrange(0, num_locusts)])
return bucket
def spawn_locusts(self, spawn_count=None, stop_timeout=None, wait=False):
if spawn_count is None:
spawn_count = self.num_clients
bucket = self.weight_locusts(spawn_count, stop_timeout)
spawn_count = len(bucket)
if self.state == STATE_INIT or self.state == STATE_STOPPED:
self.state = STATE_HATCHING
self.num_clients = spawn_count
else:
self.num_clients += spawn_count
logger.info(
"Hatching and swarming %i clients at the rate %g clients/s..." %
(spawn_count, self.hatch_rate))
occurrence_count = dict([(l.__name__, 0) for l in self.locust_classes])
def hatch():
sleep_time = 1.0 / self.hatch_rate
while True:
if not bucket:
logger.info("All locusts hatched: %s" % ", ".join([
"%s: %d" % (name, count)
for name, count in six.iteritems(occurrence_count)
]))
events.hatch_complete.fire(user_count=self.num_clients)
return
locust = bucket.pop(random.randint(0, len(bucket) - 1))
occurrence_count[locust.__name__] += 1
def start_locust(_):
try:
locust().run(runner=self)
except GreenletExit:
pass
new_locust = self.locusts.spawn(start_locust, locust)
if len(self.locusts) % 10 == 0:
logger.debug("%i locusts hatched" % len(self.locusts))
gevent.sleep(sleep_time)
hatch()
if wait:
self.locusts.join()
logger.info("All locusts dead\n")
def kill_locusts(self, kill_count):
"""
Kill a kill_count of weighted locusts from the Group() object in self.locusts
"""
bucket = self.weight_locusts(kill_count)
kill_count = len(bucket)
self.num_clients -= kill_count
logger.info("Killing %i locusts" % kill_count)
dying = []
for g in self.locusts:
for l in bucket:
if l == g.args[0]:
dying.append(g)
bucket.remove(l)
break
total_dying = len(dying)
logger.info("Dying %i locusts" % total_dying)
logger.info("Total of %i clients before" % self.num_clients)
for g in dying:
try:
self.locusts.killone(g, timeout=5)
logger.info("Locust killed")
except:
logger.info("Error on locust kill")
logger.info("Total of %i clients after" % self.num_clients)
events.hatch_complete.fire(user_count=self.num_clients)
def start_hatching(self, locust_count=None, hatch_rate=None, wait=False):
if self.state != STATE_RUNNING and self.state != STATE_HATCHING:
self.stats.clear_all()
self.stats.start_time = time()
self.exceptions = {}
events.locust_start_hatching.fire()
# Dynamically changing the locust count
if self.state != STATE_INIT and self.state != STATE_STOPPED:
self.state = STATE_HATCHING
if self.num_clients > locust_count:
# Kill some locusts
kill_count = self.num_clients - locust_count
self.kill_locusts(kill_count)
elif self.num_clients < locust_count:
# Spawn some locusts
if hatch_rate:
self.hatch_rate = hatch_rate
spawn_count = locust_count - self.num_clients
self.spawn_locusts(spawn_count=spawn_count)
else:
events.hatch_complete.fire(user_count=self.num_clients)
else:
if hatch_rate:
self.hatch_rate = hatch_rate
if locust_count is not None:
self.spawn_locusts(locust_count, wait=wait)
else:
self.spawn_locusts(wait=wait)
def stop(self):
# if we are currently hatching locusts we need to kill the hatching greenlet first
if self.hatching_greenlet and not self.hatching_greenlet.ready():
self.hatching_greenlet.kill(block=True)
self.locusts.kill(block=True)
self.state = STATE_STOPPED
events.locust_stop_hatching.fire()
def quit(self):
self.stop()
self.greenlet.kill(block=True)
def log_exception(self, node_id, msg, formatted_tb):
key = hash(formatted_tb)
row = self.exceptions.setdefault(key, {
"count": 0,
"msg": msg,
"traceback": formatted_tb,
"nodes": set()
})
row["count"] += 1
row["nodes"].add(node_id)
self.exceptions[key] = row
class LocustRunner(object):
def __init__(self, locust_classes, options):
self.options = options
self.locust_classes = locust_classes
self.hatch_rate = options.hatch_rate
self.host = options.host
self.locusts = Group()
self.greenlet = Group()
self.state = STATE_INIT
self.hatching_greenlet = None
self.stepload_greenlet = None
self.current_cpu_usage = 0
self.cpu_warning_emitted = False
self.greenlet.spawn(self.monitor_cpu)
self.exceptions = {}
self.stats = global_stats
self.step_load = options.step_load
# register listener that resets stats when hatching is complete
def on_hatch_complete(user_count):
self.state = STATE_RUNNING
if self.options.reset_stats:
logger.info("Resetting stats\n")
self.stats.reset_all()
events.hatch_complete += on_hatch_complete
def __del__(self):
# don't leave any stray greenlets if runner is removed
if len(self.greenlet) > 0:
self.greenlet.kill(block=False)
@property
def request_stats(self):
return self.stats.entries
@property
def errors(self):
return self.stats.errors
@property
def user_count(self):
return len(self.locusts)
def cpu_log_warning(self):
"""Called at the end of the test to repeat the warning & return the status"""
if self.cpu_warning_emitted:
logger.warning(
"Loadgen CPU usage was too high at some point during the test! See https://docs.locust.io/en/stable/running-locust-distributed.html for how to distribute the load over multiple CPU cores or machines"
)
return True
return False
def weight_locusts(self, amount):
"""
Distributes the amount of locusts for each WebLocust-class according to it's weight
returns a list "bucket" with the weighted locusts
"""
bucket = []
weight_sum = sum((locust.weight for locust in self.locust_classes
if locust.task_set))
residuals = {}
for locust in self.locust_classes:
if not locust.task_set:
warnings.warn(
"Notice: Found Locust class (%s) got no task_set. Skipping..."
% locust.__name__)
continue
if self.host is not None:
locust.host = self.host
# create locusts depending on weight
percent = locust.weight / float(weight_sum)
num_locusts = int(round(amount * percent))
bucket.extend([locust for x in range(num_locusts)])
# used to keep track of the amount of rounding was done if we need
# to add/remove some instances from bucket
residuals[locust] = amount * percent - round(amount * percent)
if len(bucket) < amount:
# We got too few locust classes in the bucket, so we need to create a few extra locusts,
# and we do this by iterating over each of the Locust classes - starting with the one
# where the residual from the rounding was the largest - and creating one of each until
# we get the correct amount
for locust in [
l for l, r in sorted(
residuals.items(), key=lambda x: x[1], reverse=True)
][:amount - len(bucket)]:
bucket.append(locust)
elif len(bucket) > amount:
# We've got too many locusts due to rounding errors so we need to remove some
for locust in [
l for l, r in sorted(residuals.items(), key=lambda x: x[1])
][:len(bucket) - amount]:
bucket.remove(locust)
return bucket
def spawn_locusts(self, spawn_count, wait=False):
bucket = self.weight_locusts(spawn_count)
spawn_count = len(bucket)
if self.state == STATE_INIT or self.state == STATE_STOPPED:
self.state = STATE_HATCHING
existing_count = len(self.locusts)
logger.info(
"Hatching and swarming %i users at the rate %g users/s (%i users already running)..."
% (spawn_count, self.hatch_rate, existing_count))
occurrence_count = dict([(l.__name__, 0) for l in self.locust_classes])
def hatch():
sleep_time = 1.0 / self.hatch_rate
while True:
if not bucket:
logger.info(
"All locusts hatched: %s (%i already running)" % (
", ".join([
"%s: %d" % (name, count)
for name, count in occurrence_count.items()
]),
existing_count,
))
events.hatch_complete.fire(user_count=len(self.locusts))
return
locust = bucket.pop(random.randint(0, len(bucket) - 1))
occurrence_count[locust.__name__] += 1
new_locust = locust()
def start_locust(_):
try:
new_locust.run(runner=self)
except GreenletExit:
pass
self.locusts.spawn(start_locust, new_locust)
if len(self.locusts) % 10 == 0:
logger.debug("%i locusts hatched" % len(self.locusts))
if bucket:
gevent.sleep(sleep_time)
hatch()
if wait:
self.locusts.join()
logger.info("All locusts dead\n")
def kill_locusts(self, kill_count):
"""
Kill a kill_count of weighted locusts from the Group() object in self.locusts
"""
bucket = self.weight_locusts(kill_count)
kill_count = len(bucket)
logger.info("Killing %i locusts" % kill_count)
dying = []
for g in self.locusts:
for l in bucket:
if l == type(g.args[0]):
dying.append(g)
bucket.remove(l)
break
self.kill_locust_greenlets(dying)
events.hatch_complete.fire(user_count=self.user_count)
def kill_locust_greenlets(self, greenlets):
"""
Kill running locust greenlets. If options.stop_timeout is set, we try to stop the
Locust users gracefully
"""
if self.options.stop_timeout:
dying = Group()
for g in greenlets:
locust = g.args[0]
if locust._state == LOCUST_STATE_WAITING:
self.locusts.killone(g)
else:
locust._state = LOCUST_STATE_STOPPING
dying.add(g)
if not dying.join(timeout=self.options.stop_timeout):
logger.info(
"Not all locusts finished their tasks & terminated in %s seconds. Killing them..."
% self.options.stop_timeout)
dying.kill(block=True)
else:
for g in greenlets:
self.locusts.killone(g)
def monitor_cpu(self):
process = psutil.Process()
while True:
self.current_cpu_usage = process.cpu_percent()
if self.current_cpu_usage > 90 and not self.cpu_warning_emitted:
logging.warning(
"Loadgen CPU usage above 90%! This may constrain your throughput and may even give inconsistent response time measurements! See https://docs.locust.io/en/stable/running-locust-distributed.html for how to distribute the load over multiple CPU cores or machines"
)
self.cpu_warning_emitted = True
gevent.sleep(CPU_MONITOR_INTERVAL)
def start_hatching(self, locust_count, hatch_rate, wait=False):
if self.state != STATE_RUNNING and self.state != STATE_HATCHING:
self.stats.clear_all()
self.exceptions = {}
self.cpu_warning_emitted = False
self.slave_cpu_warning_emitted = False
events.locust_start_hatching.fire()
# Dynamically changing the locust count
if self.state != STATE_INIT and self.state != STATE_STOPPED:
self.state = STATE_HATCHING
if self.user_count > locust_count:
# Kill some locusts
kill_count = self.user_count - locust_count
self.kill_locusts(kill_count)
elif self.user_count < locust_count:
# Spawn some locusts
self.hatch_rate = hatch_rate
spawn_count = locust_count - self.user_count
self.spawn_locusts(spawn_count=spawn_count)
else:
events.hatch_complete.fire(user_count=self.user_count)
else:
self.hatch_rate = hatch_rate
self.spawn_locusts(locust_count, wait=wait)
def start_stepload(self, locust_count, hatch_rate, step_locust_count,
step_duration):
if locust_count < step_locust_count:
logger.error(
"Invalid parameters: total locust count of %d is smaller than step locust count of %d"
% (locust_count, step_locust_count))
return
self.total_clients = locust_count
self.hatch_rate = hatch_rate
self.step_clients_growth = step_locust_count
self.step_duration = step_duration
if self.stepload_greenlet:
logger.info(
"There is an ongoing swarming in Step Load mode, will stop it now."
)
self.stepload_greenlet.kill()
logger.info(
"Start a new swarming in Step Load mode: total locust count of %d, hatch rate of %d, step locust count of %d, step duration of %d "
% (locust_count, hatch_rate, step_locust_count, step_duration))
self.state = STATE_INIT
self.stepload_greenlet = self.greenlet.spawn(self.stepload_worker)
self.stepload_greenlet.link_exception(callback=self.noop)
def stepload_worker(self):
current_num_clients = 0
while self.state == STATE_INIT or self.state == STATE_HATCHING or self.state == STATE_RUNNING:
current_num_clients += self.step_clients_growth
if current_num_clients > int(self.total_clients):
logger.info('Step Load is finished.')
break
self.start_hatching(current_num_clients, self.hatch_rate)
logger.info('Step loading: start hatch job of %d locust.' %
(current_num_clients))
gevent.sleep(self.step_duration)
def stop(self):
# if we are currently hatching locusts we need to kill the hatching greenlet first
if self.hatching_greenlet and not self.hatching_greenlet.ready():
self.hatching_greenlet.kill(block=True)
self.kill_locust_greenlets([g for g in self.locusts])
self.state = STATE_STOPPED
events.locust_stop_hatching.fire()
def quit(self):
self.stop()
self.greenlet.kill(block=True)
def log_exception(self, node_id, msg, formatted_tb):
key = hash(formatted_tb)
row = self.exceptions.setdefault(key, {
"count": 0,
"msg": msg,
"traceback": formatted_tb,
"nodes": set()
})
row["count"] += 1
row["nodes"].add(node_id)
self.exceptions[key] = row
def noop(self, *args, **kwargs):
""" Used to link() greenlets to in order to be compatible with gevent 1.0 """
pass
class LocustRunner(object):
def __init__(self, locust_classes, hatch_rate, num_clients, num_requests=None, host=None):
self.locust_classes = locust_classes
self.hatch_rate = hatch_rate
self.num_clients = num_clients
self.num_requests = num_requests
self.host = host
self.locusts = Group()
self.state = STATE_INIT
self.hatching_greenlet = None
self.exceptions = {}
self.stats = global_stats
# register listener that resets stats when hatching is complete
def on_hatch_complete(count):
self.state = STATE_RUNNING
logger.info("Resetting stats\n")
self.stats.reset_all()
events.hatch_complete += on_hatch_complete
@property
def request_stats(self):
return self.stats.entries
@property
def errors(self):
return self.stats.errors
@property
def user_count(self):
return len(self.locusts)
def weight_locusts(self, amount, stop_timeout = None):
"""
Distributes the amount of locusts for each WebLocust-class according to it's weight
returns a list "bucket" with the weighted locusts
"""
bucket = []
weight_sum = sum((locust.weight for locust in self.locust_classes if locust.task_set))
for locust in self.locust_classes:
if not locust.task_set:
warnings.warn("Notice: Found Locust class (%s) got no task_set. Skipping..." % locust.__name__)
continue
if self.host is not None:
locust.host = self.host
if stop_timeout is not None:
locust.stop_timeout = stop_timeout
# create locusts depending on weight
percent = locust.weight / float(weight_sum)
num_locusts = int(round(amount * percent))
bucket.extend([locust for x in xrange(0, num_locusts)])
return bucket
def spawn_locusts(self, spawn_count=None, stop_timeout=None, wait=False):
if spawn_count is None:
spawn_count = self.num_clients
if self.num_requests is not None:
self.stats.max_requests = self.num_requests
bucket = self.weight_locusts(spawn_count, stop_timeout)
spawn_count = len(bucket)
if self.state == STATE_INIT or self.state == STATE_STOPPED:
self.state = STATE_HATCHING
self.num_clients = spawn_count
else:
self.num_clients += spawn_count
logger.info("Hatching and swarming %i clients at the rate %g clients/s..." % (spawn_count, self.hatch_rate))
occurence_count = dict([(l.__name__, 0) for l in self.locust_classes])
def hatch():
sleep_time = 1.0 / self.hatch_rate
while True:
if not bucket:
logger.info("All locusts hatched: %s" % ", ".join(["%s: %d" % (name, count) for name, count in occurence_count.iteritems()]))
events.hatch_complete.fire(self.num_clients)
return
locust = bucket.pop(random.randint(0, len(bucket)-1))
occurence_count[locust.__name__] += 1
def start_locust(_):
try:
locust().run()
except GreenletExit:
pass
new_locust = self.locusts.spawn(start_locust, locust)
if len(self.locusts) % 10 == 0:
logger.debug("%i locusts hatched" % len(self.locusts))
gevent.sleep(sleep_time)
hatch()
if wait:
self.locusts.join()
logger.info("All locusts dead\n")
def kill_locusts(self, kill_count):
"""
Kill a kill_count of weighted locusts from the Group() object in self.locusts
"""
bucket = self.weight_locusts(kill_count)
kill_count = len(bucket)
self.num_clients -= kill_count
logger.info("Killing %i locusts" % kill_count)
dying = []
for g in self.locusts:
for l in bucket:
if l == g.args[0]:
dying.append(g)
bucket.remove(l)
break
for g in dying:
self.locusts.killone(g)
events.hatch_complete.fire(self.num_clients)
def start_hatching(self, locust_count=None, hatch_rate=None, wait=False):
if self.state != STATE_RUNNING and self.state != STATE_HATCHING:
self.stats.clear_all()
self.stats.start_time = time()
self.exceptions = {}
# Dynamically changing the locust count
if self.state != STATE_INIT and self.state != STATE_STOPPED:
self.state = STATE_HATCHING
if self.num_clients > locust_count:
# Kill some locusts
kill_count = self.num_clients - locust_count
self.kill_locusts(kill_count)
elif self.num_clients < locust_count:
# Spawn some locusts
if hatch_rate:
self.hatch_rate = hatch_rate
spawn_count = locust_count - self.num_clients
self.spawn_locusts(spawn_count=spawn_count)
else:
if hatch_rate:
self.hatch_rate = hatch_rate
if locust_count:
self.spawn_locusts(locust_count, wait=wait)
else:
self.spawn_locusts(wait=wait)
def stop(self):
# if we are currently hatching locusts we need to kill the hatching greenlet first
if self.hatching_greenlet and not self.hatching_greenlet.ready():
self.hatching_greenlet.kill(block=True)
self.locusts.kill(block=True)
self.state = STATE_STOPPED
def log_exception(self, node_id, msg, formatted_tb):
key = hash(formatted_tb)
row = self.exceptions.setdefault(key, {"count": 0, "msg": msg, "traceback": formatted_tb, "nodes": set()})
row["count"] += 1
row["nodes"].add(node_id)
self.exceptions[key] = row
class LocustRunner(object):
def __init__(self, locust_classes, hatch_rate, num_clients, num_requests=None, host=None):
self.locust_classes = locust_classes
self.hatch_rate = hatch_rate
self.num_clients = num_clients
self.num_requests = num_requests
self.host = host
self.locusts = Group()
self.state = STATE_INIT
self.hatching_greenlet = None
# register listener that resets stats when hatching is complete
def on_hatch_complete(count):
self.state = STATE_RUNNING
print "Resetting stats\n"
RequestStats.reset_all()
events.hatch_complete += on_hatch_complete
@property
def request_stats(self):
return RequestStats.requests
@property
def errors(self):
return RequestStats.errors
@property
def user_count(self):
return len(self.locusts)
def weight_locusts(self, amount, stop_timeout = None):
"""
Distributes the amount of locusts for each WebLocust-class according to it's weight
returns a list "bucket" with the weighted locusts
"""
bucket = []
weight_sum = sum((locust.weight for locust in self.locust_classes))
for locust in self.locust_classes:
if not locust.tasks:
warnings.warn("Notice: Found locust (%s) got no tasks. Skipping..." % locust.__name__)
continue
if self.host is not None:
locust.host = self.host
if stop_timeout is not None:
locust.stop_timeout = stop_timeout
# create locusts depending on weight
percent = locust.weight / float(weight_sum)
num_locusts = int(round(amount * percent))
bucket.extend([locust for x in xrange(0, num_locusts)])
return bucket
def spawn_locusts(self, spawn_count=None, stop_timeout=None, wait=False):
if spawn_count is None:
spawn_count = self.num_clients
if self.num_requests is not None:
RequestStats.global_max_requests = self.num_requests
bucket = self.weight_locusts(spawn_count, stop_timeout)
spawn_count = len(bucket)
if self.state == STATE_INIT or self.state == STATE_STOPPED:
self.state = STATE_HATCHING
self.num_clients = spawn_count
else:
self.num_clients += spawn_count
print "\nHatching and swarming %i clients at the rate %g clients/s...\n" % (spawn_count, self.hatch_rate)
occurence_count = dict([(l.__name__, 0) for l in self.locust_classes])
def hatch():
sleep_time = 1.0 / self.hatch_rate
while True:
if not bucket:
print "All locusts hatched: %s" % ", ".join(["%s: %d" % (name, count) for name, count in occurence_count.iteritems()])
events.hatch_complete.fire(self.num_clients)
return
locust = bucket.pop(random.randint(0, len(bucket)-1))
occurence_count[locust.__name__] += 1
def start_locust(_):
try:
locust()()
except RescheduleTaskImmediately:
pass
except GreenletExit:
pass
new_locust = self.locusts.spawn(start_locust, locust)
if len(self.locusts) % 10 == 0:
print "%i locusts hatched" % len(self.locusts)
gevent.sleep(sleep_time)
hatch()
if wait:
self.locusts.join()
print "All locusts dead\n"
print_stats(self.request_stats)
print_percentile_stats(self.request_stats) #TODO use an event listener, or such, for this?
def kill_locusts(self, kill_count):
"""
Kill a kill_count of weighted locusts from the Group() object in self.locusts
"""
bucket = self.weight_locusts(kill_count)
kill_count = len(bucket)
self.num_clients -= kill_count
print "killing locusts:", kill_count
dying = []
for g in self.locusts:
for l in bucket:
if l == g.args[0]:
dying.append(g)
bucket.remove(l)
break
for g in dying:
self.locusts.killone(g)
events.hatch_complete.fire(self.num_clients)
def start_hatching(self, locust_count=None, hatch_rate=None, wait=False):
print "start hatching", locust_count, hatch_rate, self.state
if self.state != STATE_RUNNING and self.state != STATE_HATCHING:
RequestStats.clear_all()
RequestStats.global_start_time = time()
# Dynamically changing the locust count
if self.state != STATE_INIT and self.state != STATE_STOPPED:
self.state = STATE_HATCHING
if self.num_clients > locust_count:
# Kill some locusts
kill_count = self.num_clients - locust_count
self.kill_locusts(kill_count)
elif self.num_clients < locust_count:
# Spawn some locusts
if hatch_rate:
self.hatch_rate = hatch_rate
spawn_count = locust_count - self.num_clients
self.spawn_locusts(spawn_count=spawn_count)
else:
if hatch_rate:
self.hatch_rate = hatch_rate
if locust_count:
self.spawn_locusts(locust_count, wait=wait)
else:
self.spawn_locusts(wait=wait)
def stop(self):
# if we are currently hatching locusts we need to kill the hatching greenlet first
if self.hatching_greenlet and not self.hatching_greenlet.ready():
self.hatching_greenlet.kill(block=True)
self.locusts.kill(block=True)
self.state = STATE_STOPPED
def start_ramping(self, hatch_rate=None, max_locusts=1000, hatch_stride=100,
percent=0.95, response_time_limit=2000, acceptable_fail=0.05,
precision=200, start_count=0, calibration_time=15):
from rampstats import current_percentile
if hatch_rate:
self.hatch_rate = hatch_rate
def ramp_down_help(clients, hatch_stride):
print "ramping down..."
hatch_stride = max(hatch_stride/2, precision)
clients -= hatch_stride
self.start_hatching(clients, self.hatch_rate)
return clients, hatch_stride
def ramp_up(clients, hatch_stride, boundery_found=False):
while True:
if self.state != STATE_HATCHING:
if self.num_clients >= max_locusts:
print "ramp up stopped due to max locusts limit reached:", max_locusts
client, hatch_stride = ramp_down_help(clients, hatch_stride)
return ramp_down(clients, hatch_stride)
gevent.sleep(calibration_time)
fail_ratio = RequestStats.sum_stats().fail_ratio
if fail_ratio > acceptable_fail:
print "ramp up stopped due to acceptable fail ratio %d%% exceeded with fail ratio %d%%" % (acceptable_fail*100, fail_ratio*100)
client, hatch_stride = ramp_down_help(clients, hatch_stride)
return ramp_down(clients, hatch_stride)
p = current_percentile(percent)
if p >= response_time_limit:
print "ramp up stopped due to percentile response times getting high:", p
client, hatch_stride = ramp_down_help(clients, hatch_stride)
return ramp_down(clients, hatch_stride)
if boundery_found and hatch_stride <= precision:
print "sweet spot found, ramping stopped!"
return
print "ramping up..."
if boundery_found:
hatch_stride = max((hatch_stride/2),precision)
clients += hatch_stride
self.start_hatching(clients, self.hatch_rate)
gevent.sleep(1)
def ramp_down(clients, hatch_stride):
while True:
if self.state != STATE_HATCHING:
if self.num_clients < max_locusts:
gevent.sleep(calibration_time)
fail_ratio = RequestStats.sum_stats().fail_ratio
if fail_ratio <= acceptable_fail:
p = current_percentile(percent)
if p <= response_time_limit:
if hatch_stride <= precision:
print "sweet spot found, ramping stopped!"
return
print "ramping up..."
hatch_stride = max((hatch_stride/2),precision)
clients += hatch_stride
self.start_hatching(clients, self.hatch_rate)
return ramp_up(clients, hatch_stride, True)
print "ramping down..."
hatch_stride = max((hatch_stride/2),precision)
clients -= hatch_stride
if clients > 0:
self.start_hatching(clients, self.hatch_rate)
else:
print "WARNING: no responses met the ramping thresholds, check your ramp configuration, locustfile and \"--host\" address"
print "ramping stopped!"
return
gevent.sleep(1)
if start_count > self.num_clients:
self.start_hatching(start_count, hatch_rate)
ramp_up(start_count, hatch_stride)
class LocustRunner(object):
def __init__(self, locust_classes, hatch_rate, num_clients, num_requests=None, host=None):
self.locust_classes = locust_classes
self.hatch_rate = hatch_rate
self.num_clients = num_clients
self.num_requests = num_requests
self.host = host
self.locusts = Group()
self.state = STATE_INIT
self.hatching_greenlet = None
# register listener that resets stats when hatching is complete
def on_hatch_complete(count):
self.state = STATE_RUNNING
logger.info("Resetting stats\n")
RequestStats.reset_all()
events.hatch_complete += on_hatch_complete
@property
def request_stats(self):
return RequestStats.requests
@property
def errors(self):
return RequestStats.errors
@property
def user_count(self):
return len(self.locusts)
def weight_locusts(self, amount, stop_timeout = None):
"""
Distributes the amount of locusts for each WebLocust-class according to it's weight
returns a list "bucket" with the weighted locusts
"""
bucket = []
weight_sum = sum((locust.weight for locust in self.locust_classes if locust.tasks))
for locust in self.locust_classes:
if not locust.tasks:
warnings.warn("Notice: Found locust (%s) got no tasks. Skipping..." % locust.__name__)
continue
if self.host is not None:
locust.host = self.host
if stop_timeout is not None:
locust.stop_timeout = stop_timeout
# create locusts depending on weight
percent = locust.weight / float(weight_sum)
num_locusts = int(round(amount * percent))
bucket.extend([locust for x in xrange(0, num_locusts)])
return bucket
def spawn_locusts(self, spawn_count=None, stop_timeout=None, wait=False):
if spawn_count is None:
spawn_count = self.num_clients
if self.num_requests is not None:
RequestStats.global_max_requests = self.num_requests
bucket = self.weight_locusts(spawn_count, stop_timeout)
spawn_count = len(bucket)
if self.state == STATE_INIT or self.state == STATE_STOPPED:
self.state = STATE_HATCHING
self.num_clients = spawn_count
else:
self.num_clients += spawn_count
logger.info("Hatching and swarming %i clients at the rate %g clients/s..." % (spawn_count, self.hatch_rate))
occurence_count = dict([(l.__name__, 0) for l in self.locust_classes])
def hatch():
sleep_time = 1.0 / self.hatch_rate
while True:
if not bucket:
logger.info("All locusts hatched: %s" % ", ".join(["%s: %d" % (name, count) for name, count in occurence_count.iteritems()]))
events.hatch_complete.fire(self.num_clients)
return
locust = bucket.pop(random.randint(0, len(bucket)-1))
occurence_count[locust.__name__] += 1
def start_locust(_):
try:
locust()()
except RescheduleTaskImmediately:
pass
except GreenletExit:
pass
new_locust = self.locusts.spawn(start_locust, locust)
if len(self.locusts) % 10 == 0:
logger.debug("%i locusts hatched" % len(self.locusts))
gevent.sleep(sleep_time)
hatch()
if wait:
self.locusts.join()
logger.info("All locusts dead\n")
print_stats(self.request_stats)
print_percentile_stats(self.request_stats) #TODO use an event listener, or such, for this?
def kill_locusts(self, kill_count):
"""
Kill a kill_count of weighted locusts from the Group() object in self.locusts
"""
bucket = self.weight_locusts(kill_count)
kill_count = len(bucket)
self.num_clients -= kill_count
logger.debug("killing locusts: %i", kill_count)
dying = []
for g in self.locusts:
for l in bucket:
if l == g.args[0]:
dying.append(g)
bucket.remove(l)
break
for g in dying:
self.locusts.killone(g)
events.hatch_complete.fire(self.num_clients)
def start_hatching(self, locust_count=None, hatch_rate=None, wait=False):
if self.state != STATE_RUNNING and self.state != STATE_HATCHING:
RequestStats.clear_all()
RequestStats.global_start_time = time()
# Dynamically changing the locust count
if self.state != STATE_INIT and self.state != STATE_STOPPED:
self.state = STATE_HATCHING
if self.num_clients > locust_count:
# Kill some locusts
kill_count = self.num_clients - locust_count
self.kill_locusts(kill_count)
elif self.num_clients < locust_count:
# Spawn some locusts
if hatch_rate:
self.hatch_rate = hatch_rate
spawn_count = locust_count - self.num_clients
self.spawn_locusts(spawn_count=spawn_count)
else:
if hatch_rate:
self.hatch_rate = hatch_rate
if locust_count:
self.spawn_locusts(locust_count, wait=wait)
else:
self.spawn_locusts(wait=wait)
def stop(self):
# if we are currently hatching locusts we need to kill the hatching greenlet first
if self.hatching_greenlet and not self.hatching_greenlet.ready():
self.hatching_greenlet.kill(block=True)
self.locusts.kill(block=True)
self.state = STATE_STOPPED
def start_ramping(self, hatch_rate=None, max_locusts=1000, hatch_stride=100,
percent=0.95, response_time_limit=2000, acceptable_fail=0.05,
precision=200, start_count=0, calibration_time=15):
from rampstats import current_percentile
if hatch_rate:
self.hatch_rate = hatch_rate
def ramp_down_help(clients, hatch_stride):
print "ramping down..."
hatch_stride = max(hatch_stride/2, precision)
clients -= hatch_stride
self.start_hatching(clients, self.hatch_rate)
return clients, hatch_stride
def ramp_up(clients, hatch_stride, boundery_found=False):
while True:
if self.state != STATE_HATCHING:
if self.num_clients >= max_locusts:
print "ramp up stopped due to max locusts limit reached:", max_locusts
client, hatch_stride = ramp_down_help(clients, hatch_stride)
return ramp_down(clients, hatch_stride)
gevent.sleep(calibration_time)
fail_ratio = RequestStats.sum_stats().fail_ratio
if fail_ratio > acceptable_fail:
print "ramp up stopped due to acceptable fail ratio %d%% exceeded with fail ratio %d%%" % (acceptable_fail*100, fail_ratio*100)
client, hatch_stride = ramp_down_help(clients, hatch_stride)
return ramp_down(clients, hatch_stride)
p = current_percentile(percent)
if p >= response_time_limit:
print "ramp up stopped due to percentile response times getting high:", p
client, hatch_stride = ramp_down_help(clients, hatch_stride)
return ramp_down(clients, hatch_stride)
if boundery_found and hatch_stride <= precision:
print "sweet spot found, ramping stopped!"
return
print "ramping up..."
if boundery_found:
hatch_stride = max((hatch_stride/2),precision)
clients += hatch_stride
self.start_hatching(clients, self.hatch_rate)
gevent.sleep(1)
def ramp_down(clients, hatch_stride):
while True:
if self.state != STATE_HATCHING:
if self.num_clients < max_locusts:
gevent.sleep(calibration_time)
fail_ratio = RequestStats.sum_stats().fail_ratio
if fail_ratio <= acceptable_fail:
p = current_percentile(percent)
if p <= response_time_limit:
if hatch_stride <= precision:
print "sweet spot found, ramping stopped!"
return
print "ramping up..."
hatch_stride = max((hatch_stride/2),precision)
clients += hatch_stride
self.start_hatching(clients, self.hatch_rate)
return ramp_up(clients, hatch_stride, True)
print "ramping down..."
hatch_stride = max((hatch_stride/2),precision)
clients -= hatch_stride
if clients > 0:
self.start_hatching(clients, self.hatch_rate)
else:
print "WARNING: no responses met the ramping thresholds, check your ramp configuration, locustfile and \"--host\" address"
print "ramping stopped!"
return
gevent.sleep(1)
if start_count > self.num_clients:
self.start_hatching(start_count, hatch_rate)
ramp_up(start_count, hatch_stride)
class LocustRunner(object):
def __init__(self, locust_classes, options):
self.options = options
self.locust_classes = locust_classes
self.hatch_rate = options.hatch_rate
self.num_clients = options.num_clients
self.host = options.host
self.locusts = Group()
self.greenlet = self.locusts
self.state = STATE_INIT
self.hatching_greenlet = None
self.exceptions = {}
self.stats = global_stats
# register listener that resets stats when hatching is complete
def on_hatch_complete(user_count):
self.state = STATE_RUNNING
if self.options.reset_stats:
logger.info("Resetting stats\n")
self.stats.reset_all()
events.hatch_complete += on_hatch_complete
@property
def request_stats(self):
return self.stats.entries
@property
def errors(self):
return self.stats.errors
@property
def user_count(self):
return len(self.locusts)
def weight_locusts(self, amount):
"""
Distributes the amount of locusts for each WebLocust-class according to it's weight
returns a list "bucket" with the weighted locusts
"""
bucket = []
weight_sum = sum((locust.weight for locust in self.locust_classes
if locust.task_set))
residuals = {}
for locust in self.locust_classes:
if not locust.task_set:
warnings.warn(
"Notice: Found Locust class (%s) got no task_set. Skipping..."
% locust.__name__)
continue
if self.host is not None:
locust.host = self.host
# create locusts depending on weight
percent = locust.weight / float(weight_sum)
num_locusts = int(round(amount * percent))
bucket.extend([locust for x in xrange(0, num_locusts)])
# used to keep track of the amount of rounding was done if we need
# to add/remove some instances from bucket
residuals[locust] = amount * percent - round(amount * percent)
if len(bucket) < amount:
# We got too few locust classes in the bucket, so we need to create a few extra locusts,
# and we do this by iterating over each of the Locust classes - starting with the one
# where the residual from the rounding was the largest - and creating one of each until
# we get the correct amount
for locust in [
l for l, r in sorted(
residuals.items(), key=lambda x: x[1], reverse=True)
][:amount - len(bucket)]:
bucket.append(locust)
elif len(bucket) > amount:
# We've got too many locusts due to rounding errors so we need to remove some
for locust in [
l for l, r in sorted(residuals.items(), key=lambda x: x[1])
][:len(bucket) - amount]:
bucket.remove(locust)
return bucket
def spawn_locusts(self, spawn_count=None, wait=False):
if spawn_count is None:
spawn_count = self.num_clients
bucket = self.weight_locusts(spawn_count)
spawn_count = len(bucket)
if self.state == STATE_INIT or self.state == STATE_STOPPED:
self.state = STATE_HATCHING
self.num_clients = spawn_count
else:
self.num_clients += spawn_count
logger.info(
"Hatching and swarming %i clients at the rate %g clients/s..." %
(spawn_count, self.hatch_rate))
occurrence_count = dict([(l.__name__, 0) for l in self.locust_classes])
def hatch():
sleep_time = 1.0 / self.hatch_rate
while True:
if not bucket:
logger.info("All locusts hatched: %s" % ", ".join([
"%s: %d" % (name, count)
for name, count in six.iteritems(occurrence_count)
]))
events.hatch_complete.fire(user_count=self.num_clients)
return
locust = bucket.pop(random.randint(0, len(bucket) - 1))
occurrence_count[locust.__name__] += 1
new_locust = locust()
def start_locust(_):
try:
new_locust.run(runner=self)
except GreenletExit:
pass
self.locusts.spawn(start_locust, new_locust)
if len(self.locusts) % 10 == 0:
logger.debug("%i locusts hatched" % len(self.locusts))
gevent.sleep(sleep_time)
hatch()
if wait:
self.locusts.join()
logger.info("All locusts dead\n")
def kill_locusts(self, kill_count):
"""
Kill a kill_count of weighted locusts from the Group() object in self.locusts
"""
bucket = self.weight_locusts(kill_count)
kill_count = len(bucket)
self.num_clients -= kill_count
logger.info("Killing %i locusts" % kill_count)
dying = []
for g in self.locusts:
for l in bucket:
if l == type(g.args[0]):
dying.append(g)
bucket.remove(l)
break
self.kill_locust_greenlets(dying)
events.hatch_complete.fire(user_count=self.num_clients)
def kill_locust_greenlets(self, greenlets):
"""
Kill running locust greenlets. If options.stop_timeout is set, we try to stop the
Locust users gracefully
"""
if self.options.stop_timeout:
dying = Group()
for g in greenlets:
locust = g.args[0]
if locust._state == LOCUST_STATE_WAITING:
self.locusts.killone(g)
else:
locust._state = LOCUST_STATE_STOPPING
dying.add(g)
if not dying.join(timeout=self.options.stop_timeout):
logger.info(
"Not all locusts finished their tasks & terminated in %s seconds. Killing them..."
% self.options.stop_timeout)
dying.kill(block=True)
else:
for g in greenlets:
self.locusts.killone(g)
def start_hatching(self, locust_count=None, hatch_rate=None, wait=False):
if self.state != STATE_RUNNING and self.state != STATE_HATCHING:
self.stats.clear_all()
self.exceptions = {}
events.locust_start_hatching.fire()
# Dynamically changing the locust count
if self.state != STATE_INIT and self.state != STATE_STOPPED:
self.state = STATE_HATCHING
if self.num_clients > locust_count:
# Kill some locusts
kill_count = self.num_clients - locust_count
self.kill_locusts(kill_count)
elif self.num_clients < locust_count:
# Spawn some locusts
if hatch_rate:
self.hatch_rate = hatch_rate
spawn_count = locust_count - self.num_clients
self.spawn_locusts(spawn_count=spawn_count)
else:
events.hatch_complete.fire(user_count=self.num_clients)
else:
if hatch_rate:
self.hatch_rate = hatch_rate
if locust_count is not None:
self.spawn_locusts(locust_count, wait=wait)
else:
self.spawn_locusts(wait=wait)
def stop(self):
# if we are currently hatching locusts we need to kill the hatching greenlet first
if self.hatching_greenlet and not self.hatching_greenlet.ready():
self.hatching_greenlet.kill(block=True)
self.kill_locust_greenlets([g for g in self.locusts])
self.state = STATE_STOPPED
events.locust_stop_hatching.fire()
def quit(self):
self.stop()
self.greenlet.kill(block=True)
def log_exception(self, node_id, msg, formatted_tb):
key = hash(formatted_tb)
row = self.exceptions.setdefault(key, {
"count": 0,
"msg": msg,
"traceback": formatted_tb,
"nodes": set()
})
row["count"] += 1
row["nodes"].add(node_id)
self.exceptions[key] = row
