def test_ungraceful_close_in_greenlet(self): group = Group() UnaryCallWithSleep = self._channel.unary_unary( _UNARY_CALL_METHOD_WITH_SLEEP, request_serializer=messages_pb2.SimpleRequest.SerializeToString, response_deserializer=messages_pb2.SimpleResponse.FromString, ) greenlet = group.spawn(self._run_client, UnaryCallWithSleep) # release loop so that greenlet can take control gevent.sleep() group.killone(greenlet) self.assertFalse(self._unhandled_exception, "Unhandled GreenletExit")
def test_kill_greenlet_with_generic_exception(self): group = Group() UnaryCallWithSleep = self._channel.unary_unary( _UNARY_CALL_METHOD_WITH_SLEEP, request_serializer=messages_pb2.SimpleRequest.SerializeToString, response_deserializer=messages_pb2.SimpleResponse.FromString, ) greenlet = group.spawn(self._run_client, UnaryCallWithSleep) # release loop so that greenlet can take control gevent.sleep() group.killone(greenlet, exception=Exception) self.assertFalse(self._unhandled_exception, "Unhandled exception") self.assertRaises(Exception, greenlet.get)
def test_graceful_close_in_greenlet(self): group = Group() stub = test_pb2_grpc.TestServiceStub(self._channel) greenlet = group.spawn(self._run_client, stub.UnaryCall) # release loop so that greenlet can take control gevent.sleep() self._channel.close() group.killone(greenlet) self.assertFalse(self._unhandled_exception, "Unhandled GreenletExit") try: greenlet.get() except Exception as e: # pylint: disable=broad-except self.fail(f"Unexpected exception in greenlet: {e}")
class LocustRunner(object): def __init__(self, locust_classes, options): self.options = options self.locust_classes = locust_classes self.hatch_rate = options.hatch_rate self.num_clients = options.num_clients self.host = options.host self.locusts = Group() self.greenlet = self.locusts self.state = STATE_INIT self.hatching_greenlet = None self.exceptions = {} self.stats = global_stats # register listener that resets stats when hatching is complete def on_hatch_complete(user_count): self.state = STATE_RUNNING if self.options.reset_stats: logger.info("Resetting stats\n") self.stats.reset_all() events.hatch_complete += on_hatch_complete @property def request_stats(self): return self.stats.entries @property def errors(self): return self.stats.errors @property def user_count(self): return len(self.locusts) def weight_locusts(self, amount, stop_timeout=None): """ Distributes the amount of locusts for each WebLocust-class according to it's weight returns a list "bucket" with the weighted locusts """ bucket = [] weight_sum = sum((locust.weight for locust in self.locust_classes if locust.task_set)) for locust in self.locust_classes: if not locust.task_set: warnings.warn( "Notice: Found Locust class (%s) got no task_set. Skipping..." % locust.__name__) continue if self.host is not None: locust.host = self.host if stop_timeout is not None: locust.stop_timeout = stop_timeout # create locusts depending on weight percent = locust.weight / float(weight_sum) num_locusts = int(round(amount * percent)) bucket.extend([locust for x in xrange(0, num_locusts)]) return bucket def spawn_locusts(self, spawn_count=None, stop_timeout=None, wait=False): if spawn_count is None: spawn_count = self.num_clients bucket = self.weight_locusts(spawn_count, stop_timeout) spawn_count = len(bucket) if self.state == STATE_INIT or self.state == STATE_STOPPED: self.state = STATE_HATCHING self.num_clients = spawn_count else: self.num_clients += spawn_count logger.info( "Hatching and swarming %i clients at the rate %g clients/s..." % (spawn_count, self.hatch_rate)) occurrence_count = dict([(l.__name__, 0) for l in self.locust_classes]) def hatch(): sleep_time = 1.0 / self.hatch_rate while True: if not bucket: logger.info("All locusts hatched: %s" % ", ".join([ "%s: %d" % (name, count) for name, count in six.iteritems(occurrence_count) ])) events.hatch_complete.fire(user_count=self.num_clients) return locust = bucket.pop(random.randint(0, len(bucket) - 1)) occurrence_count[locust.__name__] += 1 def start_locust(_): try: locust().run(runner=self) except GreenletExit: pass new_locust = self.locusts.spawn(start_locust, locust) if len(self.locusts) % 10 == 0: logger.debug("%i locusts hatched" % len(self.locusts)) gevent.sleep(sleep_time) hatch() if wait: self.locusts.join() logger.info("All locusts dead\n") def kill_locusts(self, kill_count): """ Kill a kill_count of weighted locusts from the Group() object in self.locusts """ bucket = self.weight_locusts(kill_count) kill_count = len(bucket) self.num_clients -= kill_count logger.info("Killing %i locusts" % kill_count) dying = [] for g in self.locusts: for l in bucket: if l == g.args[0]: dying.append(g) bucket.remove(l) break total_dying = len(dying) logger.info("Dying %i locusts" % total_dying) logger.info("Total of %i clients before" % self.num_clients) for g in dying: try: self.locusts.killone(g, timeout=5) logger.info("Locust killed") except: logger.info("Error on locust kill") logger.info("Total of %i clients after" % self.num_clients) events.hatch_complete.fire(user_count=self.num_clients) def start_hatching(self, locust_count=None, hatch_rate=None, wait=False): if self.state != STATE_RUNNING and self.state != STATE_HATCHING: self.stats.clear_all() self.stats.start_time = time() self.exceptions = {} events.locust_start_hatching.fire() # Dynamically changing the locust count if self.state != STATE_INIT and self.state != STATE_STOPPED: self.state = STATE_HATCHING if self.num_clients > locust_count: # Kill some locusts kill_count = self.num_clients - locust_count self.kill_locusts(kill_count) elif self.num_clients < locust_count: # Spawn some locusts if hatch_rate: self.hatch_rate = hatch_rate spawn_count = locust_count - self.num_clients self.spawn_locusts(spawn_count=spawn_count) else: events.hatch_complete.fire(user_count=self.num_clients) else: if hatch_rate: self.hatch_rate = hatch_rate if locust_count is not None: self.spawn_locusts(locust_count, wait=wait) else: self.spawn_locusts(wait=wait) def stop(self): # if we are currently hatching locusts we need to kill the hatching greenlet first if self.hatching_greenlet and not self.hatching_greenlet.ready(): self.hatching_greenlet.kill(block=True) self.locusts.kill(block=True) self.state = STATE_STOPPED events.locust_stop_hatching.fire() def quit(self): self.stop() self.greenlet.kill(block=True) def log_exception(self, node_id, msg, formatted_tb): key = hash(formatted_tb) row = self.exceptions.setdefault(key, { "count": 0, "msg": msg, "traceback": formatted_tb, "nodes": set() }) row["count"] += 1 row["nodes"].add(node_id) self.exceptions[key] = row
class LocustRunner(object): def __init__(self, locust_classes, options): self.options = options self.locust_classes = locust_classes self.hatch_rate = options.hatch_rate self.host = options.host self.locusts = Group() self.greenlet = Group() self.state = STATE_INIT self.hatching_greenlet = None self.stepload_greenlet = None self.current_cpu_usage = 0 self.cpu_warning_emitted = False self.greenlet.spawn(self.monitor_cpu) self.exceptions = {} self.stats = global_stats self.step_load = options.step_load # register listener that resets stats when hatching is complete def on_hatch_complete(user_count): self.state = STATE_RUNNING if self.options.reset_stats: logger.info("Resetting stats\n") self.stats.reset_all() events.hatch_complete += on_hatch_complete def __del__(self): # don't leave any stray greenlets if runner is removed if len(self.greenlet) > 0: self.greenlet.kill(block=False) @property def request_stats(self): return self.stats.entries @property def errors(self): return self.stats.errors @property def user_count(self): return len(self.locusts) def cpu_log_warning(self): """Called at the end of the test to repeat the warning & return the status""" if self.cpu_warning_emitted: logger.warning( "Loadgen CPU usage was too high at some point during the test! See https://docs.locust.io/en/stable/running-locust-distributed.html for how to distribute the load over multiple CPU cores or machines" ) return True return False def weight_locusts(self, amount): """ Distributes the amount of locusts for each WebLocust-class according to it's weight returns a list "bucket" with the weighted locusts """ bucket = [] weight_sum = sum((locust.weight for locust in self.locust_classes if locust.task_set)) residuals = {} for locust in self.locust_classes: if not locust.task_set: warnings.warn( "Notice: Found Locust class (%s) got no task_set. Skipping..." % locust.__name__) continue if self.host is not None: locust.host = self.host # create locusts depending on weight percent = locust.weight / float(weight_sum) num_locusts = int(round(amount * percent)) bucket.extend([locust for x in range(num_locusts)]) # used to keep track of the amount of rounding was done if we need # to add/remove some instances from bucket residuals[locust] = amount * percent - round(amount * percent) if len(bucket) < amount: # We got too few locust classes in the bucket, so we need to create a few extra locusts, # and we do this by iterating over each of the Locust classes - starting with the one # where the residual from the rounding was the largest - and creating one of each until # we get the correct amount for locust in [ l for l, r in sorted( residuals.items(), key=lambda x: x[1], reverse=True) ][:amount - len(bucket)]: bucket.append(locust) elif len(bucket) > amount: # We've got too many locusts due to rounding errors so we need to remove some for locust in [ l for l, r in sorted(residuals.items(), key=lambda x: x[1]) ][:len(bucket) - amount]: bucket.remove(locust) return bucket def spawn_locusts(self, spawn_count, wait=False): bucket = self.weight_locusts(spawn_count) spawn_count = len(bucket) if self.state == STATE_INIT or self.state == STATE_STOPPED: self.state = STATE_HATCHING existing_count = len(self.locusts) logger.info( "Hatching and swarming %i users at the rate %g users/s (%i users already running)..." % (spawn_count, self.hatch_rate, existing_count)) occurrence_count = dict([(l.__name__, 0) for l in self.locust_classes]) def hatch(): sleep_time = 1.0 / self.hatch_rate while True: if not bucket: logger.info( "All locusts hatched: %s (%i already running)" % ( ", ".join([ "%s: %d" % (name, count) for name, count in occurrence_count.items() ]), existing_count, )) events.hatch_complete.fire(user_count=len(self.locusts)) return locust = bucket.pop(random.randint(0, len(bucket) - 1)) occurrence_count[locust.__name__] += 1 new_locust = locust() def start_locust(_): try: new_locust.run(runner=self) except GreenletExit: pass self.locusts.spawn(start_locust, new_locust) if len(self.locusts) % 10 == 0: logger.debug("%i locusts hatched" % len(self.locusts)) if bucket: gevent.sleep(sleep_time) hatch() if wait: self.locusts.join() logger.info("All locusts dead\n") def kill_locusts(self, kill_count): """ Kill a kill_count of weighted locusts from the Group() object in self.locusts """ bucket = self.weight_locusts(kill_count) kill_count = len(bucket) logger.info("Killing %i locusts" % kill_count) dying = [] for g in self.locusts: for l in bucket: if l == type(g.args[0]): dying.append(g) bucket.remove(l) break self.kill_locust_greenlets(dying) events.hatch_complete.fire(user_count=self.user_count) def kill_locust_greenlets(self, greenlets): """ Kill running locust greenlets. If options.stop_timeout is set, we try to stop the Locust users gracefully """ if self.options.stop_timeout: dying = Group() for g in greenlets: locust = g.args[0] if locust._state == LOCUST_STATE_WAITING: self.locusts.killone(g) else: locust._state = LOCUST_STATE_STOPPING dying.add(g) if not dying.join(timeout=self.options.stop_timeout): logger.info( "Not all locusts finished their tasks & terminated in %s seconds. Killing them..." % self.options.stop_timeout) dying.kill(block=True) else: for g in greenlets: self.locusts.killone(g) def monitor_cpu(self): process = psutil.Process() while True: self.current_cpu_usage = process.cpu_percent() if self.current_cpu_usage > 90 and not self.cpu_warning_emitted: logging.warning( "Loadgen CPU usage above 90%! This may constrain your throughput and may even give inconsistent response time measurements! See https://docs.locust.io/en/stable/running-locust-distributed.html for how to distribute the load over multiple CPU cores or machines" ) self.cpu_warning_emitted = True gevent.sleep(CPU_MONITOR_INTERVAL) def start_hatching(self, locust_count, hatch_rate, wait=False): if self.state != STATE_RUNNING and self.state != STATE_HATCHING: self.stats.clear_all() self.exceptions = {} self.cpu_warning_emitted = False self.slave_cpu_warning_emitted = False events.locust_start_hatching.fire() # Dynamically changing the locust count if self.state != STATE_INIT and self.state != STATE_STOPPED: self.state = STATE_HATCHING if self.user_count > locust_count: # Kill some locusts kill_count = self.user_count - locust_count self.kill_locusts(kill_count) elif self.user_count < locust_count: # Spawn some locusts self.hatch_rate = hatch_rate spawn_count = locust_count - self.user_count self.spawn_locusts(spawn_count=spawn_count) else: events.hatch_complete.fire(user_count=self.user_count) else: self.hatch_rate = hatch_rate self.spawn_locusts(locust_count, wait=wait) def start_stepload(self, locust_count, hatch_rate, step_locust_count, step_duration): if locust_count < step_locust_count: logger.error( "Invalid parameters: total locust count of %d is smaller than step locust count of %d" % (locust_count, step_locust_count)) return self.total_clients = locust_count self.hatch_rate = hatch_rate self.step_clients_growth = step_locust_count self.step_duration = step_duration if self.stepload_greenlet: logger.info( "There is an ongoing swarming in Step Load mode, will stop it now." ) self.stepload_greenlet.kill() logger.info( "Start a new swarming in Step Load mode: total locust count of %d, hatch rate of %d, step locust count of %d, step duration of %d " % (locust_count, hatch_rate, step_locust_count, step_duration)) self.state = STATE_INIT self.stepload_greenlet = self.greenlet.spawn(self.stepload_worker) self.stepload_greenlet.link_exception(callback=self.noop) def stepload_worker(self): current_num_clients = 0 while self.state == STATE_INIT or self.state == STATE_HATCHING or self.state == STATE_RUNNING: current_num_clients += self.step_clients_growth if current_num_clients > int(self.total_clients): logger.info('Step Load is finished.') break self.start_hatching(current_num_clients, self.hatch_rate) logger.info('Step loading: start hatch job of %d locust.' % (current_num_clients)) gevent.sleep(self.step_duration) def stop(self): # if we are currently hatching locusts we need to kill the hatching greenlet first if self.hatching_greenlet and not self.hatching_greenlet.ready(): self.hatching_greenlet.kill(block=True) self.kill_locust_greenlets([g for g in self.locusts]) self.state = STATE_STOPPED events.locust_stop_hatching.fire() def quit(self): self.stop() self.greenlet.kill(block=True) def log_exception(self, node_id, msg, formatted_tb): key = hash(formatted_tb) row = self.exceptions.setdefault(key, { "count": 0, "msg": msg, "traceback": formatted_tb, "nodes": set() }) row["count"] += 1 row["nodes"].add(node_id) self.exceptions[key] = row def noop(self, *args, **kwargs): """ Used to link() greenlets to in order to be compatible with gevent 1.0 """ pass
class LocustRunner(object): def __init__(self, locust_classes, hatch_rate, num_clients, num_requests=None, host=None): self.locust_classes = locust_classes self.hatch_rate = hatch_rate self.num_clients = num_clients self.num_requests = num_requests self.host = host self.locusts = Group() self.state = STATE_INIT self.hatching_greenlet = None self.exceptions = {} self.stats = global_stats # register listener that resets stats when hatching is complete def on_hatch_complete(count): self.state = STATE_RUNNING logger.info("Resetting stats\n") self.stats.reset_all() events.hatch_complete += on_hatch_complete @property def request_stats(self): return self.stats.entries @property def errors(self): return self.stats.errors @property def user_count(self): return len(self.locusts) def weight_locusts(self, amount, stop_timeout = None): """ Distributes the amount of locusts for each WebLocust-class according to it's weight returns a list "bucket" with the weighted locusts """ bucket = [] weight_sum = sum((locust.weight for locust in self.locust_classes if locust.task_set)) for locust in self.locust_classes: if not locust.task_set: warnings.warn("Notice: Found Locust class (%s) got no task_set. Skipping..." % locust.__name__) continue if self.host is not None: locust.host = self.host if stop_timeout is not None: locust.stop_timeout = stop_timeout # create locusts depending on weight percent = locust.weight / float(weight_sum) num_locusts = int(round(amount * percent)) bucket.extend([locust for x in xrange(0, num_locusts)]) return bucket def spawn_locusts(self, spawn_count=None, stop_timeout=None, wait=False): if spawn_count is None: spawn_count = self.num_clients if self.num_requests is not None: self.stats.max_requests = self.num_requests bucket = self.weight_locusts(spawn_count, stop_timeout) spawn_count = len(bucket) if self.state == STATE_INIT or self.state == STATE_STOPPED: self.state = STATE_HATCHING self.num_clients = spawn_count else: self.num_clients += spawn_count logger.info("Hatching and swarming %i clients at the rate %g clients/s..." % (spawn_count, self.hatch_rate)) occurence_count = dict([(l.__name__, 0) for l in self.locust_classes]) def hatch(): sleep_time = 1.0 / self.hatch_rate while True: if not bucket: logger.info("All locusts hatched: %s" % ", ".join(["%s: %d" % (name, count) for name, count in occurence_count.iteritems()])) events.hatch_complete.fire(self.num_clients) return locust = bucket.pop(random.randint(0, len(bucket)-1)) occurence_count[locust.__name__] += 1 def start_locust(_): try: locust().run() except GreenletExit: pass new_locust = self.locusts.spawn(start_locust, locust) if len(self.locusts) % 10 == 0: logger.debug("%i locusts hatched" % len(self.locusts)) gevent.sleep(sleep_time) hatch() if wait: self.locusts.join() logger.info("All locusts dead\n") def kill_locusts(self, kill_count): """ Kill a kill_count of weighted locusts from the Group() object in self.locusts """ bucket = self.weight_locusts(kill_count) kill_count = len(bucket) self.num_clients -= kill_count logger.info("Killing %i locusts" % kill_count) dying = [] for g in self.locusts: for l in bucket: if l == g.args[0]: dying.append(g) bucket.remove(l) break for g in dying: self.locusts.killone(g) events.hatch_complete.fire(self.num_clients) def start_hatching(self, locust_count=None, hatch_rate=None, wait=False): if self.state != STATE_RUNNING and self.state != STATE_HATCHING: self.stats.clear_all() self.stats.start_time = time() self.exceptions = {} # Dynamically changing the locust count if self.state != STATE_INIT and self.state != STATE_STOPPED: self.state = STATE_HATCHING if self.num_clients > locust_count: # Kill some locusts kill_count = self.num_clients - locust_count self.kill_locusts(kill_count) elif self.num_clients < locust_count: # Spawn some locusts if hatch_rate: self.hatch_rate = hatch_rate spawn_count = locust_count - self.num_clients self.spawn_locusts(spawn_count=spawn_count) else: if hatch_rate: self.hatch_rate = hatch_rate if locust_count: self.spawn_locusts(locust_count, wait=wait) else: self.spawn_locusts(wait=wait) def stop(self): # if we are currently hatching locusts we need to kill the hatching greenlet first if self.hatching_greenlet and not self.hatching_greenlet.ready(): self.hatching_greenlet.kill(block=True) self.locusts.kill(block=True) self.state = STATE_STOPPED def log_exception(self, node_id, msg, formatted_tb): key = hash(formatted_tb) row = self.exceptions.setdefault(key, {"count": 0, "msg": msg, "traceback": formatted_tb, "nodes": set()}) row["count"] += 1 row["nodes"].add(node_id) self.exceptions[key] = row
class LocustRunner(object): def __init__(self, locust_classes, hatch_rate, num_clients, num_requests=None, host=None): self.locust_classes = locust_classes self.hatch_rate = hatch_rate self.num_clients = num_clients self.num_requests = num_requests self.host = host self.locusts = Group() self.state = STATE_INIT self.hatching_greenlet = None # register listener that resets stats when hatching is complete def on_hatch_complete(count): self.state = STATE_RUNNING print "Resetting stats\n" RequestStats.reset_all() events.hatch_complete += on_hatch_complete @property def request_stats(self): return RequestStats.requests @property def errors(self): return RequestStats.errors @property def user_count(self): return len(self.locusts) def weight_locusts(self, amount, stop_timeout = None): """ Distributes the amount of locusts for each WebLocust-class according to it's weight returns a list "bucket" with the weighted locusts """ bucket = [] weight_sum = sum((locust.weight for locust in self.locust_classes)) for locust in self.locust_classes: if not locust.tasks: warnings.warn("Notice: Found locust (%s) got no tasks. Skipping..." % locust.__name__) continue if self.host is not None: locust.host = self.host if stop_timeout is not None: locust.stop_timeout = stop_timeout # create locusts depending on weight percent = locust.weight / float(weight_sum) num_locusts = int(round(amount * percent)) bucket.extend([locust for x in xrange(0, num_locusts)]) return bucket def spawn_locusts(self, spawn_count=None, stop_timeout=None, wait=False): if spawn_count is None: spawn_count = self.num_clients if self.num_requests is not None: RequestStats.global_max_requests = self.num_requests bucket = self.weight_locusts(spawn_count, stop_timeout) spawn_count = len(bucket) if self.state == STATE_INIT or self.state == STATE_STOPPED: self.state = STATE_HATCHING self.num_clients = spawn_count else: self.num_clients += spawn_count print "\nHatching and swarming %i clients at the rate %g clients/s...\n" % (spawn_count, self.hatch_rate) occurence_count = dict([(l.__name__, 0) for l in self.locust_classes]) def hatch(): sleep_time = 1.0 / self.hatch_rate while True: if not bucket: print "All locusts hatched: %s" % ", ".join(["%s: %d" % (name, count) for name, count in occurence_count.iteritems()]) events.hatch_complete.fire(self.num_clients) return locust = bucket.pop(random.randint(0, len(bucket)-1)) occurence_count[locust.__name__] += 1 def start_locust(_): try: locust()() except RescheduleTaskImmediately: pass except GreenletExit: pass new_locust = self.locusts.spawn(start_locust, locust) if len(self.locusts) % 10 == 0: print "%i locusts hatched" % len(self.locusts) gevent.sleep(sleep_time) hatch() if wait: self.locusts.join() print "All locusts dead\n" print_stats(self.request_stats) print_percentile_stats(self.request_stats) #TODO use an event listener, or such, for this? def kill_locusts(self, kill_count): """ Kill a kill_count of weighted locusts from the Group() object in self.locusts """ bucket = self.weight_locusts(kill_count) kill_count = len(bucket) self.num_clients -= kill_count print "killing locusts:", kill_count dying = [] for g in self.locusts: for l in bucket: if l == g.args[0]: dying.append(g) bucket.remove(l) break for g in dying: self.locusts.killone(g) events.hatch_complete.fire(self.num_clients) def start_hatching(self, locust_count=None, hatch_rate=None, wait=False): print "start hatching", locust_count, hatch_rate, self.state if self.state != STATE_RUNNING and self.state != STATE_HATCHING: RequestStats.clear_all() RequestStats.global_start_time = time() # Dynamically changing the locust count if self.state != STATE_INIT and self.state != STATE_STOPPED: self.state = STATE_HATCHING if self.num_clients > locust_count: # Kill some locusts kill_count = self.num_clients - locust_count self.kill_locusts(kill_count) elif self.num_clients < locust_count: # Spawn some locusts if hatch_rate: self.hatch_rate = hatch_rate spawn_count = locust_count - self.num_clients self.spawn_locusts(spawn_count=spawn_count) else: if hatch_rate: self.hatch_rate = hatch_rate if locust_count: self.spawn_locusts(locust_count, wait=wait) else: self.spawn_locusts(wait=wait) def stop(self): # if we are currently hatching locusts we need to kill the hatching greenlet first if self.hatching_greenlet and not self.hatching_greenlet.ready(): self.hatching_greenlet.kill(block=True) self.locusts.kill(block=True) self.state = STATE_STOPPED def start_ramping(self, hatch_rate=None, max_locusts=1000, hatch_stride=100, percent=0.95, response_time_limit=2000, acceptable_fail=0.05, precision=200, start_count=0, calibration_time=15): from rampstats import current_percentile if hatch_rate: self.hatch_rate = hatch_rate def ramp_down_help(clients, hatch_stride): print "ramping down..." hatch_stride = max(hatch_stride/2, precision) clients -= hatch_stride self.start_hatching(clients, self.hatch_rate) return clients, hatch_stride def ramp_up(clients, hatch_stride, boundery_found=False): while True: if self.state != STATE_HATCHING: if self.num_clients >= max_locusts: print "ramp up stopped due to max locusts limit reached:", max_locusts client, hatch_stride = ramp_down_help(clients, hatch_stride) return ramp_down(clients, hatch_stride) gevent.sleep(calibration_time) fail_ratio = RequestStats.sum_stats().fail_ratio if fail_ratio > acceptable_fail: print "ramp up stopped due to acceptable fail ratio %d%% exceeded with fail ratio %d%%" % (acceptable_fail*100, fail_ratio*100) client, hatch_stride = ramp_down_help(clients, hatch_stride) return ramp_down(clients, hatch_stride) p = current_percentile(percent) if p >= response_time_limit: print "ramp up stopped due to percentile response times getting high:", p client, hatch_stride = ramp_down_help(clients, hatch_stride) return ramp_down(clients, hatch_stride) if boundery_found and hatch_stride <= precision: print "sweet spot found, ramping stopped!" return print "ramping up..." if boundery_found: hatch_stride = max((hatch_stride/2),precision) clients += hatch_stride self.start_hatching(clients, self.hatch_rate) gevent.sleep(1) def ramp_down(clients, hatch_stride): while True: if self.state != STATE_HATCHING: if self.num_clients < max_locusts: gevent.sleep(calibration_time) fail_ratio = RequestStats.sum_stats().fail_ratio if fail_ratio <= acceptable_fail: p = current_percentile(percent) if p <= response_time_limit: if hatch_stride <= precision: print "sweet spot found, ramping stopped!" return print "ramping up..." hatch_stride = max((hatch_stride/2),precision) clients += hatch_stride self.start_hatching(clients, self.hatch_rate) return ramp_up(clients, hatch_stride, True) print "ramping down..." hatch_stride = max((hatch_stride/2),precision) clients -= hatch_stride if clients > 0: self.start_hatching(clients, self.hatch_rate) else: print "WARNING: no responses met the ramping thresholds, check your ramp configuration, locustfile and \"--host\" address" print "ramping stopped!" return gevent.sleep(1) if start_count > self.num_clients: self.start_hatching(start_count, hatch_rate) ramp_up(start_count, hatch_stride)
class LocustRunner(object): def __init__(self, locust_classes, hatch_rate, num_clients, num_requests=None, host=None): self.locust_classes = locust_classes self.hatch_rate = hatch_rate self.num_clients = num_clients self.num_requests = num_requests self.host = host self.locusts = Group() self.state = STATE_INIT self.hatching_greenlet = None # register listener that resets stats when hatching is complete def on_hatch_complete(count): self.state = STATE_RUNNING logger.info("Resetting stats\n") RequestStats.reset_all() events.hatch_complete += on_hatch_complete @property def request_stats(self): return RequestStats.requests @property def errors(self): return RequestStats.errors @property def user_count(self): return len(self.locusts) def weight_locusts(self, amount, stop_timeout = None): """ Distributes the amount of locusts for each WebLocust-class according to it's weight returns a list "bucket" with the weighted locusts """ bucket = [] weight_sum = sum((locust.weight for locust in self.locust_classes if locust.tasks)) for locust in self.locust_classes: if not locust.tasks: warnings.warn("Notice: Found locust (%s) got no tasks. Skipping..." % locust.__name__) continue if self.host is not None: locust.host = self.host if stop_timeout is not None: locust.stop_timeout = stop_timeout # create locusts depending on weight percent = locust.weight / float(weight_sum) num_locusts = int(round(amount * percent)) bucket.extend([locust for x in xrange(0, num_locusts)]) return bucket def spawn_locusts(self, spawn_count=None, stop_timeout=None, wait=False): if spawn_count is None: spawn_count = self.num_clients if self.num_requests is not None: RequestStats.global_max_requests = self.num_requests bucket = self.weight_locusts(spawn_count, stop_timeout) spawn_count = len(bucket) if self.state == STATE_INIT or self.state == STATE_STOPPED: self.state = STATE_HATCHING self.num_clients = spawn_count else: self.num_clients += spawn_count logger.info("Hatching and swarming %i clients at the rate %g clients/s..." % (spawn_count, self.hatch_rate)) occurence_count = dict([(l.__name__, 0) for l in self.locust_classes]) def hatch(): sleep_time = 1.0 / self.hatch_rate while True: if not bucket: logger.info("All locusts hatched: %s" % ", ".join(["%s: %d" % (name, count) for name, count in occurence_count.iteritems()])) events.hatch_complete.fire(self.num_clients) return locust = bucket.pop(random.randint(0, len(bucket)-1)) occurence_count[locust.__name__] += 1 def start_locust(_): try: locust()() except RescheduleTaskImmediately: pass except GreenletExit: pass new_locust = self.locusts.spawn(start_locust, locust) if len(self.locusts) % 10 == 0: logger.debug("%i locusts hatched" % len(self.locusts)) gevent.sleep(sleep_time) hatch() if wait: self.locusts.join() logger.info("All locusts dead\n") print_stats(self.request_stats) print_percentile_stats(self.request_stats) #TODO use an event listener, or such, for this? def kill_locusts(self, kill_count): """ Kill a kill_count of weighted locusts from the Group() object in self.locusts """ bucket = self.weight_locusts(kill_count) kill_count = len(bucket) self.num_clients -= kill_count logger.debug("killing locusts: %i", kill_count) dying = [] for g in self.locusts: for l in bucket: if l == g.args[0]: dying.append(g) bucket.remove(l) break for g in dying: self.locusts.killone(g) events.hatch_complete.fire(self.num_clients) def start_hatching(self, locust_count=None, hatch_rate=None, wait=False): if self.state != STATE_RUNNING and self.state != STATE_HATCHING: RequestStats.clear_all() RequestStats.global_start_time = time() # Dynamically changing the locust count if self.state != STATE_INIT and self.state != STATE_STOPPED: self.state = STATE_HATCHING if self.num_clients > locust_count: # Kill some locusts kill_count = self.num_clients - locust_count self.kill_locusts(kill_count) elif self.num_clients < locust_count: # Spawn some locusts if hatch_rate: self.hatch_rate = hatch_rate spawn_count = locust_count - self.num_clients self.spawn_locusts(spawn_count=spawn_count) else: if hatch_rate: self.hatch_rate = hatch_rate if locust_count: self.spawn_locusts(locust_count, wait=wait) else: self.spawn_locusts(wait=wait) def stop(self): # if we are currently hatching locusts we need to kill the hatching greenlet first if self.hatching_greenlet and not self.hatching_greenlet.ready(): self.hatching_greenlet.kill(block=True) self.locusts.kill(block=True) self.state = STATE_STOPPED def start_ramping(self, hatch_rate=None, max_locusts=1000, hatch_stride=100, percent=0.95, response_time_limit=2000, acceptable_fail=0.05, precision=200, start_count=0, calibration_time=15): from rampstats import current_percentile if hatch_rate: self.hatch_rate = hatch_rate def ramp_down_help(clients, hatch_stride): print "ramping down..." hatch_stride = max(hatch_stride/2, precision) clients -= hatch_stride self.start_hatching(clients, self.hatch_rate) return clients, hatch_stride def ramp_up(clients, hatch_stride, boundery_found=False): while True: if self.state != STATE_HATCHING: if self.num_clients >= max_locusts: print "ramp up stopped due to max locusts limit reached:", max_locusts client, hatch_stride = ramp_down_help(clients, hatch_stride) return ramp_down(clients, hatch_stride) gevent.sleep(calibration_time) fail_ratio = RequestStats.sum_stats().fail_ratio if fail_ratio > acceptable_fail: print "ramp up stopped due to acceptable fail ratio %d%% exceeded with fail ratio %d%%" % (acceptable_fail*100, fail_ratio*100) client, hatch_stride = ramp_down_help(clients, hatch_stride) return ramp_down(clients, hatch_stride) p = current_percentile(percent) if p >= response_time_limit: print "ramp up stopped due to percentile response times getting high:", p client, hatch_stride = ramp_down_help(clients, hatch_stride) return ramp_down(clients, hatch_stride) if boundery_found and hatch_stride <= precision: print "sweet spot found, ramping stopped!" return print "ramping up..." if boundery_found: hatch_stride = max((hatch_stride/2),precision) clients += hatch_stride self.start_hatching(clients, self.hatch_rate) gevent.sleep(1) def ramp_down(clients, hatch_stride): while True: if self.state != STATE_HATCHING: if self.num_clients < max_locusts: gevent.sleep(calibration_time) fail_ratio = RequestStats.sum_stats().fail_ratio if fail_ratio <= acceptable_fail: p = current_percentile(percent) if p <= response_time_limit: if hatch_stride <= precision: print "sweet spot found, ramping stopped!" return print "ramping up..." hatch_stride = max((hatch_stride/2),precision) clients += hatch_stride self.start_hatching(clients, self.hatch_rate) return ramp_up(clients, hatch_stride, True) print "ramping down..." hatch_stride = max((hatch_stride/2),precision) clients -= hatch_stride if clients > 0: self.start_hatching(clients, self.hatch_rate) else: print "WARNING: no responses met the ramping thresholds, check your ramp configuration, locustfile and \"--host\" address" print "ramping stopped!" return gevent.sleep(1) if start_count > self.num_clients: self.start_hatching(start_count, hatch_rate) ramp_up(start_count, hatch_stride)
class LocustRunner(object): def __init__(self, locust_classes, options): self.options = options self.locust_classes = locust_classes self.hatch_rate = options.hatch_rate self.num_clients = options.num_clients self.host = options.host self.locusts = Group() self.greenlet = self.locusts self.state = STATE_INIT self.hatching_greenlet = None self.exceptions = {} self.stats = global_stats # register listener that resets stats when hatching is complete def on_hatch_complete(user_count): self.state = STATE_RUNNING if self.options.reset_stats: logger.info("Resetting stats\n") self.stats.reset_all() events.hatch_complete += on_hatch_complete @property def request_stats(self): return self.stats.entries @property def errors(self): return self.stats.errors @property def user_count(self): return len(self.locusts) def weight_locusts(self, amount): """ Distributes the amount of locusts for each WebLocust-class according to it's weight returns a list "bucket" with the weighted locusts """ bucket = [] weight_sum = sum((locust.weight for locust in self.locust_classes if locust.task_set)) residuals = {} for locust in self.locust_classes: if not locust.task_set: warnings.warn( "Notice: Found Locust class (%s) got no task_set. Skipping..." % locust.__name__) continue if self.host is not None: locust.host = self.host # create locusts depending on weight percent = locust.weight / float(weight_sum) num_locusts = int(round(amount * percent)) bucket.extend([locust for x in xrange(0, num_locusts)]) # used to keep track of the amount of rounding was done if we need # to add/remove some instances from bucket residuals[locust] = amount * percent - round(amount * percent) if len(bucket) < amount: # We got too few locust classes in the bucket, so we need to create a few extra locusts, # and we do this by iterating over each of the Locust classes - starting with the one # where the residual from the rounding was the largest - and creating one of each until # we get the correct amount for locust in [ l for l, r in sorted( residuals.items(), key=lambda x: x[1], reverse=True) ][:amount - len(bucket)]: bucket.append(locust) elif len(bucket) > amount: # We've got too many locusts due to rounding errors so we need to remove some for locust in [ l for l, r in sorted(residuals.items(), key=lambda x: x[1]) ][:len(bucket) - amount]: bucket.remove(locust) return bucket def spawn_locusts(self, spawn_count=None, wait=False): if spawn_count is None: spawn_count = self.num_clients bucket = self.weight_locusts(spawn_count) spawn_count = len(bucket) if self.state == STATE_INIT or self.state == STATE_STOPPED: self.state = STATE_HATCHING self.num_clients = spawn_count else: self.num_clients += spawn_count logger.info( "Hatching and swarming %i clients at the rate %g clients/s..." % (spawn_count, self.hatch_rate)) occurrence_count = dict([(l.__name__, 0) for l in self.locust_classes]) def hatch(): sleep_time = 1.0 / self.hatch_rate while True: if not bucket: logger.info("All locusts hatched: %s" % ", ".join([ "%s: %d" % (name, count) for name, count in six.iteritems(occurrence_count) ])) events.hatch_complete.fire(user_count=self.num_clients) return locust = bucket.pop(random.randint(0, len(bucket) - 1)) occurrence_count[locust.__name__] += 1 new_locust = locust() def start_locust(_): try: new_locust.run(runner=self) except GreenletExit: pass self.locusts.spawn(start_locust, new_locust) if len(self.locusts) % 10 == 0: logger.debug("%i locusts hatched" % len(self.locusts)) gevent.sleep(sleep_time) hatch() if wait: self.locusts.join() logger.info("All locusts dead\n") def kill_locusts(self, kill_count): """ Kill a kill_count of weighted locusts from the Group() object in self.locusts """ bucket = self.weight_locusts(kill_count) kill_count = len(bucket) self.num_clients -= kill_count logger.info("Killing %i locusts" % kill_count) dying = [] for g in self.locusts: for l in bucket: if l == type(g.args[0]): dying.append(g) bucket.remove(l) break self.kill_locust_greenlets(dying) events.hatch_complete.fire(user_count=self.num_clients) def kill_locust_greenlets(self, greenlets): """ Kill running locust greenlets. If options.stop_timeout is set, we try to stop the Locust users gracefully """ if self.options.stop_timeout: dying = Group() for g in greenlets: locust = g.args[0] if locust._state == LOCUST_STATE_WAITING: self.locusts.killone(g) else: locust._state = LOCUST_STATE_STOPPING dying.add(g) if not dying.join(timeout=self.options.stop_timeout): logger.info( "Not all locusts finished their tasks & terminated in %s seconds. Killing them..." % self.options.stop_timeout) dying.kill(block=True) else: for g in greenlets: self.locusts.killone(g) def start_hatching(self, locust_count=None, hatch_rate=None, wait=False): if self.state != STATE_RUNNING and self.state != STATE_HATCHING: self.stats.clear_all() self.exceptions = {} events.locust_start_hatching.fire() # Dynamically changing the locust count if self.state != STATE_INIT and self.state != STATE_STOPPED: self.state = STATE_HATCHING if self.num_clients > locust_count: # Kill some locusts kill_count = self.num_clients - locust_count self.kill_locusts(kill_count) elif self.num_clients < locust_count: # Spawn some locusts if hatch_rate: self.hatch_rate = hatch_rate spawn_count = locust_count - self.num_clients self.spawn_locusts(spawn_count=spawn_count) else: events.hatch_complete.fire(user_count=self.num_clients) else: if hatch_rate: self.hatch_rate = hatch_rate if locust_count is not None: self.spawn_locusts(locust_count, wait=wait) else: self.spawn_locusts(wait=wait) def stop(self): # if we are currently hatching locusts we need to kill the hatching greenlet first if self.hatching_greenlet and not self.hatching_greenlet.ready(): self.hatching_greenlet.kill(block=True) self.kill_locust_greenlets([g for g in self.locusts]) self.state = STATE_STOPPED events.locust_stop_hatching.fire() def quit(self): self.stop() self.greenlet.kill(block=True) def log_exception(self, node_id, msg, formatted_tb): key = hash(formatted_tb) row = self.exceptions.setdefault(key, { "count": 0, "msg": msg, "traceback": formatted_tb, "nodes": set() }) row["count"] += 1 row["nodes"].add(node_id) self.exceptions[key] = row