class CrawlBase(object):
spider_count = 0
def __init__(self):
self.group = Group()
self.queue = Queue()
def read_seed(self, file='seeds.txt'):
with open(file) as f:
for line in f:
if len(line) > 0 and line != "\n":
yield line.strip()
else:
return
def dispatch(self):
for url in self.read_seed():
g = gevent.spawn(Spider, self, url)
self.group.add(g)
self.group.join()
def harvest(self):
try:
while True:
content = self.queue.get(timeout=2)
print(content)
except Empty:
pass
def _search(responder, id, search, max_results):
todo = dict()
group = Group()
for ctx in search.contexts:
if len(ctx.results) > max_results/len(search.contexts):
continue
todo[ctx.name] = 0
group.spawn(_run, responder, search, ctx, todo)
group.join()
for ctx in search.contexts[:]:
if not ctx.results:
search.remove_ctx(ctx)
results = list()
while search.contexts and (max_results is None or len(results) < max_results):
ctx = search.next_result_ctx()
if not ctx.results:
break
results.append(ctx.results.pop(0))
display = search.display
for ctx in search.contexts[:]:
if ctx.next is None:
search.remove_ctx(ctx)
if search.more:
cache[search.id] = search
elif search.id in cache:
del cache[search.id]
return dict(id=id, search_id=search.id, display=display, more=search.more, results=results)
def test_quiet_group():
from gevent.pool import Group
group = Group()
group.spawn(divide_by_zero)
group.spawn(divide_by_zero)
with raises_from(ZeroDivisionError, 'divide_by_zero'):
group.join(raise_error=True)
def test_greenlet(self):
queue = JoinableQueue()
requests_done = Event()
g = Greenlet(self._producer, queue, FirstService(), 'Terminator')
h = Greenlet(self._producer, queue, SecondService(), 'Terminator')
i = Greenlet(self._producer, queue, ThirdService(), 'Terminator')
requests = Group()
for request in g, h, i:
requests.add(request)
log.debug('before spawn')
c = spawn(
self._consumer,
done=requests_done,
queue=queue,
)
[r.start() for r in requests]
log.debug('after spawn')
requests.join()
requests_done.set()
log.debug('requests are done')
c.join()
log.debug('consumer is done')
def _push_to_target(self, targets):
"""Get a batch of elements from the queue, and push it to the targets.
This function returns True if it proceeded all the elements in
the queue, and there isn't anything more to read.
"""
if self.queue.empty():
return 0 # nothing
batch = []
pushed = 0
# collecting a batch
while len(batch) < self.batch_size:
item = self.queue.get()
if item == 'END':
pushed += 1 # the 'END' item
break
batch.append(item)
if len(batch) != 0:
greenlets = Group()
for plugin in targets:
green = greenlets.spawn(self._put_data, plugin, batch)
green.link_exception(partial(self._error,
exception.InjectError, plugin))
greenlets.join()
pushed += len(batch)
return pushed
def poll(request):
out = {}
out['res'] = {}
ids = request.GET.get('ids')
done = True
if ids:
group = Group()
greenlets = []
split = ids.split(',')
for id in split:
greenlet = CheckGreenlet(id, group)
greenlets.append(greenlet)
group.join()
for greenlet in greenlets:
out['res'][greenlet.id] = greenlet.result
if not greenlet.done:
done = False
out['done'] = done
return HttpResponse(json.dumps(out), mimetype='application/json')
def test_parallel_folder_syncs(db, folder_name_mapping, monkeypatch):
# test that when we run save_folder_names in parallel, we only create one
# tag for that folder. this happens when the CondstoreFolderSyncEngine
# checks for UID changes.
# patching the heartbeat clear means that we force the first greenlet to
# wait around (there is a deleted folder in folder_name_mapping), thereby
# assuring that the second greenlet will overtake it and force any
# potential race condition around tag creation.
def clear_heartbeat_patch(w, x, y, z):
gevent.sleep(1)
monkeypatch.setattr('inbox.heartbeat.store.HeartbeatStore.remove_folders',
clear_heartbeat_patch)
log = get_logger()
group = Group()
with mailsync_session_scope() as db_session:
group.spawn(save_folder_names, log, ACCOUNT_ID,
folder_name_mapping, db_session)
with mailsync_session_scope() as db_session:
group.spawn(save_folder_names, log, ACCOUNT_ID,
folder_name_mapping, db_session)
group.join()
with mailsync_session_scope() as db_session:
account = db_session.query(Account).get(ACCOUNT_ID)
random_tags = db_session.query(Tag).filter_by(
namespace_id=account.namespace.id,
name='random')
assert random_tags.count() == 1
def _enqueue_children(resources, seed, parent):
threads = Group()
for resource in [r for r in resources if r.parent == parent.name]:
thread = threads.spawn(_process_resource, resources, seed, resource)
thread.link_exception(_create_error_handler(resource.collection))
threads.join()
def patch_all(timeout=180, external_loaded=True, source_complete_callback=None):
with patch_all_lock:
# check config urls
log.debug('checking config urls')
todo = list()
for source in sources.values():
config_url = source.get_config_url()
if config_url is not None and config_url not in todo:
todo.append(config_url)
group = Group()
for config_url in todo:
g = group.spawn(config_url.update)
patch_group.add(g)
group.join()
log.debug('updating repos')
# check for updates
patches = list()
for source in sources.values():
if source.enabled:
def _patch(patches, source, timeout):
try:
patch_one(patches, source, timeout)
finally:
if source_complete_callback is not None:
source_complete_callback(source)
g = group.spawn(_patch, patches, source, timeout)
patch_group.add(g)
group.join()
finalize_patches(patches, external_loaded=external_loaded)
def execute(resources, seed=None):
threads = Group()
for resource in [r for r in resources if not r.parent]:
thread = threads.spawn(_process_resource, resources, seed, resource)
thread.link_exception(_create_error_handler(resource.collection))
threads.join()
if len(_errors):
sys.exit(1)
def terminate():
g = Group()
for torrent in torrents.values():
g.spawn(torrent.save_resume_data)
try:
g.join(timeout=5)
except:
pass
for torrent in torrents.values():
torrent.remove()
def main(): g1 = gevent.spawn(talk, 'foo') g2 = gevent.spawn(talk, 'bar') g3 = gevent.spawn(talk, 'baz') group = Group() group.add(g1) group.add(g1) group.join() group.add(g1) group.join()
def ready(role):
queue = Group()
while True:
info = ScaleInfo(role)
_scale_ctx_stack.push(info)
if config.check_func is not None:
queue.spawn(action, role, config.check_func())
else:
queue.spawn(action, role, check_cpu_utilization(role))
queue.spawn(gevent.sleep, role.cooltime*60)
queue.join()
_scale_ctx_stack.pop()
def Demo1():
g1 = gevent.spawn(talk, 'bar')
g2 = gevent.spawn(talk, 'foo')
g3 = gevent.spawn(talk, 'fizz')
group = Group()
group.add(g1)
group.add(g2)
group.join()
group.add(g3)
group.join()
def sample_manager():
g1 = gevent.spawn(talk, 'bar')
g2 = gevent.spawn(talk, 'foo')
g3 = gevent.spawn(talk, 'fizz')
group = Group()
group.add(g1)
group.add(g2)
group.join()
group.add(g3)
group.join()
class NodeManager(core.Agent):
"""
The class responsible to create nodes.
"""
name = 'manager'
"""
the registered name in the :class:`addressing.AddressBook`
"""
def __init__(self, graph):
"""
Creates a new node_manager
:param graph: the graph to pass to the agents
:type graph: storage.GraphWrapper
"""
self.graph = graph
self.failures = []
self.group = Group()
def setup_node(self, node, greenlet):
greenlet.link_value(node.deactivate_node)
node.graph = self.graph
self.group.add(greenlet)
def unset_node(self, node, greenlet):
del node.graph
self.group.discard(greenlet)
if isinstance(greenlet.value, core.GreenletExit):
self.graph.remove_node(node.id)
print 'Removing', node.id
def create_node(self, cls, parameters):
"""
Creates a new node.
:param cls: the factory creating the new node.
:type cls: callable
:param parameters: the parameters that are forwarded to the node for creation
:type parameters: dict
:return: the actual identifier
:rtype: int | str
"""
node = cls(**parameters)
identifier = self.graph.add_node()
node.start(self._address_book, self._node_db, identifier)
return identifier
def simulation_ended(self):
self.group.join()
def _main():
user_crawler_group = Group()
for _ in xrange(GREENLET_COUNT):
user_crawler_group.spawn(analyze)
with open('ids.txt') as FILE:
for line in FILE:
id = line.strip()
cursor.execute('SELECT COUNT(1) as total_count FROM tb_xweibo_user_info WHERE uid = %s' % id)
result = cursor.fetchone()
if not result['total_count']:
users_fetch_queue.put(id)
user_crawler_group.join()
def test_kill_processlet_group(proc):
group = Group()
group.greenlet_class = lets.Processlet
group.spawn(raise_when_killed)
group.spawn(raise_when_killed)
group.spawn(raise_when_killed)
group.join(0)
assert len(proc.children()) == 3
group.kill()
assert len(proc.children()) == 0
for job in group:
with pytest.raises(Killed):
job.get()
assert job.exit_code == 1
def get_multihoster_account(task, multi_match, file):
if not account.config.use_useraccounts:
print "multihoster off, use_useraccounts false"
return
group = Group()
for pool in account.manager.values():
for acc in pool:
if acc.multi_account:
from . import manager
acc.hoster = manager.find_by_name(acc.name)
group.spawn(acc.boot)
group.join()
accounts = []
best_weight = 0
hostname = file.split_url.host
for pool in account.manager.values():
for acc in pool:
if acc._private_account:
continue
if not acc.multi_account:
continue
if hasattr(acc, 'premium') and not acc.premium:
continue
print acc.premium
if not multi_match(acc, hostname):
continue
try:
weight = acc.weight
except gevent.GreenletExit:
print "greenlet exit"
continue
if weight > best_weight:
accounts = []
best_weight = weight
bisect.insort(accounts, (acc.get_task_pool(task).full() and 1 or 0, len(accounts), acc))
if accounts:
return accounts[0][2]
"""try:
file.log.info('trying multihoster {}'.format(acc.name))
acc.hoster.get_download_context(file)
except gevent.GreenletExit:
raise
except BaseException as e:
log.exception(e)"""
else:
print "multi: no accounts found"
def test_group(self):
def talk(msg):
for i in xrange(3):
print(msg)
g1 = gevent.spawn(talk, 'bar')
g2 = gevent.spawn(talk, 'foo')
g3 = gevent.spawn(talk, 'fizz')
group = Group()
group.add(g1)
group.add(g2)
group.join()
group.add(g3)
group.join()
class Processor(object):
def __init__(self):
self.group = Group()
self.greenlets = []
def spawn(self, func, *args, **kwargs):
g = self.group.spawn(func, *args, **kwargs)
self.greenlets.append(g)
def join(self):
self.group.join()
def values(self):
gs, self.greenlets = self.greenlets, []
return [g.value for g in gs]
def empty(self):
return not bool(self.greenlets)
class MultiZerologEmitter(gevent.Greenlet):
"""Emitter using multiple loggers which are configured by the zerolog server.
"""
def __init__(self, interval):
super(MultiZerologEmitter, self).__init__()
self.interval = interval
self.greenlets = Group()
#self.loggers = 'foo foo.lib foo.web foo.web.request foo.web.db'.split()
self.loggers = 'foo foo.lib foo.lib.bar'.split()
self.levels = 'critical error warning info debug'.split()
self._keep_going = True
def _run(self):
self.greenlets.add(gevent.spawn(self.__random_logger))
#for logger_name in self.loggers:
# self.greenlets.add(gevent.spawn(self.__logger, logger_name))
self.greenlets.join()
def __logger(self, logger_name):
#loggers = 'app app.sub app.sub.lib'.split()
logger = zerolog.getLogger(logger_name)
index = 0
while self._keep_going:
level = random.choice(self.levels)
message = "{0} {1} {2}".format(index, logger_name, level)
getattr(logger, level)(message)
index += 1
gevent.sleep(self.interval)
def __random_logger(self):
index = 0
while self._keep_going:
logger = zerolog.getLogger(random.choice(self.loggers))
level = random.choice(self.levels)
message = "{0} {1} {2}".format(index, logger.name, level)
getattr(logger, level)(message)
index += 1
gevent.sleep(self.interval)
def kill(self, exception=gevent.GreenletExit, **kwargs):
self._keep_going = False
self.greenlets.kill()
super(MultiZerologEmitter, self).kill(exception=exception, **kwargs)
class Dispatcher(gevent.Greenlet):
def __init__(self, collector, publisher, quiet=False):
super(Dispatcher, self).__init__()
self.collector = collector
self.publisher = publisher
self.quiet = quiet
self.greenlets = Group()
self.channel = gevent.queue.Queue(0)
self._keep_going = True
def _run(self):
self.greenlets.spawn(self.__collect)
self.greenlets.spawn(self.__publish)
self.greenlets.join()
def kill(self, exception=gevent.GreenletExit, **kwargs):
self._keep_going = False
self.greenlets.kill()
super(Dispatcher, self).kill(exception=exception, **kwargs)
def __collect(self):
while self._keep_going:
message = self.collector.recv_multipart()
self.channel.put(message)
gevent.sleep()
def __publish(self):
while self._keep_going:
message = self.channel.get()
if not self.quiet:
# message is assumed to be a tuple of: (topic, record_json)
topic,record_json = message
topic = topic.decode()
name_and_level = topic[len(zerolog.stream_prefix):]
logger_name,level_name = name_and_level.split(':')
logger = zerolog.getLocalLogger(logger_name)
if logger.isEnabledFor(logging.getLevelName(level_name)):
# inject log record into local logger
record_dict = json.loads(record_json.decode())
record = logging.makeLogRecord(record_dict)
logger.handle(record)
self.publisher.send_multipart(message)
gevent.sleep()
def runner(test_run_id, datasets, local_storage, credentials):
failures = 0
group = Group()
for dataset_index, dataset in enumerate(datasets):
try:
g = run_mb_test(dataset_index, dataset)
if g:
group.add(g)
except Exception:
failures += 1
logging.error('MBTest FAILED for [{}] {}'.format(dataset_index, dataset),
exc_info = True)
if len(group):
group.join()
mark_as_complete(test_run_id)
return failures
def get(self, ip, timeout=None):
"""Queries all DNSBLs in the group for matches.
:param ip: The IP address to check for.
:param timeout: Timeout in seconds before canceling remaining queries.
:returns: A :class:`set()` containing the DNSBL domain names that
matched a record for the IP address.
"""
matches = set()
group = Group()
with gevent.Timeout(timeout, None):
for dnsbl in self.dnsbls:
thread = self.pool.spawn(self._run_dnsbl_get,
matches, dnsbl, ip)
group.add(thread)
group.join()
group.kill()
return matches
def _search(responder, id, search, max_results):
responder._sent = False
todo = dict()
group = Group()
groups[id] = [group]
for ctx in search.contexts:
groups[id].append(ctx.thumb_pool)
if len(ctx.results) > max_results/len(search.contexts):
continue
todo[ctx.name] = 0
group.spawn(_run, responder, search, ctx, todo)
group.join()
for ctx in search.contexts[:]:
if not ctx.results:
search.remove_ctx(ctx)
results = list()
while search.contexts and (max_results is None or len(results) < max_results):
ctx = search.next_result_ctx()
if not ctx.results:
break
results.append(ctx.results.pop(0))
display = search.display
for ctx in search.contexts[:]:
if ctx.next is None:
search.remove_ctx(ctx)
if search.more:
cache[search.id] = search
elif search.id in cache:
del cache[search.id]
payload = dict(id=id, search_id=search.id, display=display, more=search.more, results=results)
responder.send(payload=payload)
responder._sent = True
try:
del groups[id]
except KeyError:
pass
def _update(self):
found_sources = list()
resp = requests.get(self.url, stream=True)
try:
resp.raise_for_status()
data = yaml.load(resp.raw)
finally:
resp.close()
assert len(data.keys()) > 0
group = Group()
def _add_source(url):
try:
source = add_source(url, self.url)
except:
self.log.warning('error adding new repo {}'.format(url))
else:
found_sources.append(source)
for name, url in data.iteritems():
try:
Url(url)
except:
self.log.warning('invalid patch source entry: {}'.format(url))
try:
source = sources[name]
except KeyError:
self.log.info('adding new repo {}'.format(url))
group.spawn(_add_source, url)
else:
found_sources.append(source)
if source.url != url:
source.log.info('changing url to {}'.format(url))
with transaction:
source.url = url
source.unlink()
group.join()
for source in sources.values():
if source.config_url == self.url and source not in found_sources:
source.log.info('erasing repo')
source.delete(True)
def gevent_click_page():
global TRY_COUNT
TRY_COUNT = int(sys.argv[1])
_log.info('自动点击页面开始...')
# 先获取文章总篇数
driver = webdriver.PhantomJS()
driver.get('https://www.xncoding.com/archives/')
# driver.maximize_window()
posts_count = len(driver.find_elements_by_xpath(
'//article/header/h1[@class="post-title"]/a[@class="post-title-link"]'))
driver.close()
# gevent的pool容量
psize = posts_count / THREAD_COUNT
_log.info('总的文章数量为:{}, 每组需要爬取的文章数:{}'.format(posts_count, psize))
group = Group()
for i in range(0, THREAD_COUNT + 1):
group.add(gevent.spawn(_click_page, posts_count, psize, i))
group.join()
_log.info('成功结束...')
def run(fqn, concurrency=1, numruns=1):
""" Runs a test.
* fnq: fully qualified name
* concurrency: number of concurrent runs
* numruns: number of run per concurrent
"""
set_global_stream('stdout', total=concurrency * numruns)
test = resolve_name(fqn)
klass = test.im_class
ob = klass(test.__name__)
test_result = unittest.TestResult()
group = Group()
for i in range(concurrency):
group.spawn(_run, i, ob, test_result, numruns)
group.join()
return test_result
def get_best(self, task, file):
with self.lock:
if config.use_useraccounts:
all_accounts = self
else:
all_accounts = [a for a in self if a._private_account]
group = Group()
for account in all_accounts:
group.spawn(account.boot)
group.join()
all_accounts = [a for a in all_accounts if a._private_account or (a.enabled and a.last_error is None)]
accounts = []
best_weight = 0
for account in all_accounts:
if file is not None and not account.match(file):
continue
try:
weight = account.weight
except gevent.GreenletExit:
continue
if weight is None or weight < best_weight:
continue
if weight > best_weight:
accounts = []
best_weight = weight
bisect.insort(accounts, (account.get_task_pool(task).full() and 1 or 0, len(accounts), account))
if accounts:
return accounts[0][2]
if len(all_accounts) > 0:
#self.log.warning('found no account. returning first one...')
return all_accounts[0]
else:
self.log.info('found no account. creating a "free" account')
account = self.add(_private_account=True)
account.boot()
return account
def generate_report(opts):
writer = sys.stdout.write
group = Group() if not FAILED_IMPORT else ''
# ======================== START OF CUSTOMIZE REPORT ================================== #
# URL or filepath of your company logo
logo = opts.logo
# Ignores following library keywords in metrics report
ignore_library = IGNORE_LIBRARIES
if opts.ignore:
ignore_library.extend(opts.ignore)
# Ignores following type keywords in metrics report
ignore_type = IGNORE_TYPES
if opts.ignoretype:
ignore_type.extend(opts.ignoretype)
# ======================== END OF CUSTOMIZE REPORT ================================== #
# Report to support file location as arguments
# Source Code Contributed By : Ruud Prijs
# input directory
path = os.path.abspath(os.path.expanduser(opts.path))
# output.xml file
output_name = os.path.join(path, opts.output)
# report.html file
report_name = opts.report_name
# log.html file
log_name = opts.log_name
required_files = (
output_name,
# report_name,
# log_name,
)
missing_files = [
filename for filename in required_files if not os.path.exists(filename)
]
if missing_files:
# We have files missing.
# writer("The following files are missing: {}\n".format(", ".join(missing_files)))
writer("output.xml file is missing: {}".format(
", ".join(missing_files)))
exit(1)
# email status
send_email = opts.email
mtTime = datetime.now().strftime('%Y%m%d-%H%M%S')
# Output result file location
result_file_name = 'metrics-' + mtTime + '.html'
result_file = os.path.join(path, result_file_name)
# Read output.xml file
result = ExecutionResult(output_name)
result.configure(stat_config={
'suite_stat_level': 2,
'tag_stat_combine': 'tagANDanother'
})
writer("Converting .xml to .html file. This may take few minutes...")
# ======= START OF EMAIL SETUP CONTENT ====== #
if send_email:
server = smtplib.SMTP('smtp.gmail.com:587')
msg = MIMEMultipart()
msg['Subject'] = 'Robotframework Automation Status'
sender = opts.sender
recipients = opts.to.split(',') if opts.to else ''
ccrecipients = opts.cc.split(',') if opts.cc else ''
msg['From'] = sender
msg['To'] = ", ".join(recipients)
msg['Cc'] = ", ".join(ccrecipients)
password = opts.pwd
msg.add_header('Content-Type', 'text/html')
# ======= END OF EMAIL SETUP CONTENT ====== #
head_content = """
<!doctype html>
<html lang="en">
<head>
<link rel="shortcut icon" href="https://png.icons8.com/windows/50/000000/bot.png" type="image/x-icon" />
<title>RF Metrics Report</title>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
<meta name="viewport" content="width=device-width, initial-scale=1">
<link href="https://cdn.datatables.net/1.10.19/css/jquery.dataTables.min.css" rel="stylesheet"/>
<link href="https://cdn.datatables.net/buttons/1.5.2/css/buttons.dataTables.min.css" rel="stylesheet"/>
<link href="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/4.1.3/css/bootstrap.min.css" rel="stylesheet"/>
<link href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css" rel="stylesheet">
<script src="https://code.jquery.com/jquery-3.3.1.js" type="text/javascript"></script>
<!-- Bootstrap core Googleccharts -->
<script src="https://www.gstatic.com/charts/loader.js" type="text/javascript"></script>
<script type="text/javascript">google.charts.load('current', {packages: ['corechart']});</script>
<!-- Bootstrap core Datatable-->
<script src="https://code.jquery.com/jquery-3.3.1.js" type="text/javascript"></script>
<script src="https://cdn.datatables.net/1.10.19/js/jquery.dataTables.min.js" type="text/javascript"></script>
<script src="https://cdn.datatables.net/buttons/1.5.2/js/dataTables.buttons.min.js" type="text/javascript"></script>
<script src="https://cdn.datatables.net/buttons/1.5.2/js/buttons.flash.min.js" type="text/javascript"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/jszip/3.1.3/jszip.min.js" type="text/javascript"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/pdfmake/0.1.36/pdfmake.min.js" type="text/javascript"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/pdfmake/0.1.36/vfs_fonts.js" type="text/javascript"></script>
<script src="https://cdn.datatables.net/buttons/1.5.2/js/buttons.html5.min.js" type="text/javascript"></script>
<script src="https://cdn.datatables.net/buttons/1.5.2/js/buttons.print.min.js" type="text/javascript"></script>
<style>
.sidebar {
position: fixed;
top: 0;
bottom: 0;
left: 0;
z-index: 100; /* Behind the navbar */
box-shadow: inset -1px 0 0 rgba(0, 0, 0, .1);
}
.sidebar-sticky {
position: relative;
top: 0;
height: calc(100vh - 48px);
padding-top: .5rem;
overflow-x: hidden;
overflow-y: auto; /* Scrollable contents if viewport is shorter than content. */
}
@supports ((position: -webkit-sticky) or (position: sticky)) {
.sidebar-sticky {
position: -webkit-sticky;
position: sticky;
}
}
.sidebar .nav-link {
color: black;
}
.sidebar .nav-link.active {
color: #007bff;
}
.sidebar .nav-link:hover .feather,
.sidebar .nav-link.active .feather {
color: inherit;
}
[role="main"] {
padding-top: 8px;
}
/* Set height of body and the document to 100% */
body {
height: 100%;
margin: 0;
//font-family: Comic Sans MS;
background-color: white;
}
/* Style tab links */
.tablinkLog {
cursor: pointer;
}
@import url(https://fonts.googleapis.com/css?family=Droid+Sans);
.loader {
position: fixed;
left: 0px;
top: 0px;
width: 100%;
height: 100%;
z-index: 9999;
background: url('http://www.downgraf.com/wp-content/uploads/2014/09/01-progress.gif?e44397') 50% 50% no-repeat rgb(249,249,249);
}
/* TILES */
.tile {
width: 100%;
float: left;
margin: 0px;
list-style: none;
font-size: 30px;
color: #FFF;
-moz-border-radius: 5px;
-webkit-border-radius: 5px;
margin-bottom: 5px;
position: relative;
text-align: center;
color: white!important;
}
.tile.tile-fail {
background: #f44336!important;
}
.tile.tile-pass {
background: #4CAF50!important;
}
.tile.tile-info {
background: #009688!important;
}
.tile.tile-head {
background: #616161!important;
}
.dt-buttons {
margin-left: 5px;
}
</style>
</head>
"""
soup = BeautifulSoup(head_content, "html.parser")
body = soup.new_tag('body')
soup.insert(20, body)
icons_txt = """
<div class="loader"></div>
<div class="container-fluid">
<div class="row">
<nav class="col-md-2 d-none d-md-block bg-light sidebar" style="font-size:16px;">
<div class="sidebar-sticky">
<ul class="nav flex-column">
<img src="%s" style="height:18vh!important;width:95%%;"/>
<br>
<h6 class="sidebar-heading d-flex justify-content-between align-items-center text-muted">
<span>Metrics</span>
<a class="d-flex align-items-center text-muted" href="#"></a>
</h6>
<li class="nav-item">
<a class="tablink nav-link" href="#" id="defaultOpen" onclick="openPage('dashboard', this, 'orange')">
<i class="fa fa-dashboard"></i> Dashboard
</a>
</li>
<li class="nav-item">
<a class="tablink nav-link" href="#" onclick="openPage('suiteMetrics', this, 'orange');executeDataTable('#sm',5)" >
<i class="fa fa-th-large"></i> Suite Metrics
</a>
</li>
<li class="nav-item">
<a class="tablink nav-link" href="#" onclick="openPage('testMetrics', this, 'orange');executeDataTable('#tm',3)">
<i class="fa fa-list-alt"></i> Test Metrics
</a>
</li>
<li class="nav-item">
<a class="tablink nav-link" href="#" onclick="openPage('keywordMetrics', this, 'orange');executeDataTable('#km',3)">
<i class="fa fa-table"></i> Keyword Metrics
</a>
</li>
<li class="nav-item">
<a class="tablink nav-link" href="#" onclick="openPage('log', this, 'orange');">
<i class="fa fa-wpforms"></i> Robot Logs
</a>
</li>
<li class="nav-item">
<a class="tablink nav-link" href="#" onclick="openPage('statistics', this, 'orange');">
<i class="fa fa-envelope-o"></i> Email Metrics
</a>
</li>
</ul>
<h6 class="sidebar-heading d-flex justify-content-between align-items-center text-muted">
<span>Project</span>
<a class="d-flex align-items-center text-muted" href="#"></a>
</h6>
<ul class="nav flex-column mb-2">
<li class="nav-item">
<a style="color:blue;" class="tablink nav-link" target="_blank" href="https://www.github.com">
<i class="fa fa-external-link"></i> Git Hub
</a>
</li>
<li class="nav-item">
<a style="color:blue;" class="tablink nav-link" target="_blank" href="https://www.jira.com">
<i class="fa fa-external-link"></i> JIRA
</a>
</li>
</ul>
</div>
</nav>
</div>
""" % (logo)
body.append(BeautifulSoup(icons_txt, 'html.parser'))
page_content_div = soup.new_tag('div')
page_content_div["role"] = "main"
page_content_div["class"] = "col-md-9 ml-sm-auto col-lg-10 px-4"
body.insert(50, page_content_div)
writer("\n1 of 6: Capturing dashboard content...")
### ============================ START OF DASHBOARD ======================================= ####
test_stats = TestStats()
result.visit(test_stats)
total_suite = test_stats.total_suite
passed_suite = test_stats.passed_suite
failed_suite = test_stats.failed_suite
suitepp = math.ceil(passed_suite * 100.0 / total_suite)
elapsedtime = datetime(
1970, 1, 1) + timedelta(milliseconds=result.suite.elapsedtime)
elapsedtime = elapsedtime.strftime("%X")
myResult = result.generated_by_robot
if myResult:
generator = "Robot"
else:
generator = "Rebot"
stats = result.statistics
total = stats.total.all.total
passed = stats.total.all.passed
failed = stats.total.all.failed
testpp = round(passed * 100.0 / total, 2)
kw_stats = KeywordStats(ignore_library, ignore_type)
result.visit(kw_stats)
total_keywords = kw_stats.total_keywords
passed_keywords = kw_stats.passed_keywords
failed_keywords = kw_stats.failed_keywords
# Handling ZeroDivisionError exception when no keywords are found
if total_keywords > 0:
kwpp = round(passed_keywords * 100.0 / total_keywords, 2)
else:
kwpp = 0
dashboard_content = """
<div class="tabcontent" id="dashboard">
<div class="d-flex flex-column flex-md-row align-items-center p-1 mb-3 bg-light border-bottom shadow-sm">
<h5 class="my-0 mr-md-auto font-weight-normal"><i class="fa fa-dashboard"></i> Dashboard</h5>
<nav class="my-2 my-md-0 mr-md-3" style="color:red">
<a class="p-2"><b style="color:black;">Execution Time: </b>%s h</a>
<a class="p-2"><b style="color:black;cursor: pointer;" data-toggle="tooltip" title=".xml file is created by">Generated By: </b>%s</a>
</nav>
</div>
<div class="row">
<div class="col-md-3" onclick="openPage('suiteMetrics', this, '')" data-toggle="tooltip" title="Click to view Suite metrics" style="cursor: pointer;">
<a class="tile tile-head">
Suite
<p style="font-size:12px">Statistics</p>
</a>
</div>
<div class="col-md-3">
<a class="tile tile-info">
%s
<p style="font-size:12px">Total</p>
</a>
</div>
<div class="col-md-3">
<a class="tile tile-pass">
%s
<p style="font-size:12px">Pass</p>
</a>
</div>
<div class="col-md-3">
<a class="tile tile-fail">
%s
<p style="font-size:12px">Fail</p>
</a>
</div>
</div>
<div class="row">
<div class="col-md-3" onclick="openPage('testMetrics', this, '')" data-toggle="tooltip" title="Click to view Test metrics" style="cursor: pointer;">
<a class="tile tile-head">
Test
<p style="font-size:12px">Statistics</p>
</a>
</div>
<div class="col-md-3">
<a class="tile tile-info">
%s
<p style="font-size:12px">Total</p>
</a>
</div>
<div class="col-md-3">
<a class="tile tile-pass">
%s
<p style="font-size:12px">Pass</p>
</a>
</div>
<div class="col-md-3">
<a class="tile tile-fail">
%s
<p style="font-size:12px">Fail</p>
</a>
</div>
</div>
<div class="row">
<div class="col-md-3" onclick="openPage('keywordMetrics', this, '')" data-toggle="tooltip" title="Click to view Keyword metrics" style="cursor: pointer;">
<a class="tile tile-head">
Keyword
<p style="font-size:12px">Statistics</p>
</a>
</div>
<div class="col-md-3">
<a class="tile tile-info">
%s
<p style="font-size:12px">Total</p>
</a>
</div>
<div class="col-md-3">
<a class="tile tile-pass">
%s
<p style="font-size:12px">Pass</p>
</a>
</div>
<div class="col-md-3">
<a class="tile tile-fail">
%s
<p style="font-size:12px">Fail</p>
</a>
</div>
</div>
<hr></hr>
<div class="row">
<div class="col-md-4" style="background-color:white;height:280px;width:auto;border:groove;">
<span style="font-weight:bold">Suite Status:</span>
<div id="suiteChartID" style="height:250px;width:auto;"></div>
</div>
<div class="col-md-4" style="background-color:white;height:280px;width:auto;border:groove;">
<span style="font-weight:bold">Test Status:</span>
<div id="testChartID" style="height:250px;width:auto;"></div>
</div>
<div class="col-md-4" style="background-color:white;height:280px;width:auto;border:groove;">
<span style="font-weight:bold">Keyword Status:</span>
<div id="keywordChartID" style="height:250px;width:auto;"></div>
</div>
</div>
<hr></hr>
<div class="row">
<div class="col-md-12" style="background-color:white;height:450px;width:auto;border:groove;">
<span style="font-weight:bold">Top 10 Suite Performance(sec):</span>
<div id="suiteBarID" style="height:400px;width:auto;"></div>
</div>
<div class="col-md-12" style="background-color:white;height:450px;width:auto;border:groove;">
<span style="font-weight:bold">Top 10 Test Performance(sec):</span>
<div id="testsBarID" style="height:400px;width:auto;"></div>
</div>
<div class="col-md-12" style="background-color:white;height:450px;width:auto;border:groove;">
<span style="font-weight:bold">Top 10 Keywords Performance(sec):</span>
<div id="keywordsBarID" style="height:400px;width:auto;"></div>
</div>
</div>
<div class="row">
<div class="col-md-12" style="height:25px;width:auto;">
<p class="text-muted" style="text-align:center;font-size:9px">robotframework-metrics</p>
</div>
</div>
<script>
window.onload = function(){
executeDataTable('#sm',5);
executeDataTable('#tm',3);
executeDataTable('#km',3);
createPieChart(%s,%s,'suiteChartID','Suite Status:');
createBarGraph('#sm',0,5,10,'suiteBarID','Elapsed Time(s): ','Suite');
createPieChart(%s,%s,'testChartID','Tests Status:');
createBarGraph('#tm',1,3,10,'testsBarID','Elapsed Time(s): ','Test');
createPieChart(%s,%s,'keywordChartID','Keywords Status:');
createBarGraph('#km',1,3,10,'keywordsBarID','Elapsed Time(s): ','Keyword');
};
</script>
<script>
function openInNewTab(url,element_id) {
var element_id= element_id;
var win = window.open(url, '_blank');
win.focus();
$('body').scrollTo(element_id);
}
</script>
</div>
""" % (elapsedtime, generator, total_suite, passed_suite, failed_suite,
total, passed, failed, total_keywords, passed_keywords,
failed_keywords, passed_suite, failed_suite, passed, failed,
passed_keywords, failed_keywords)
page_content_div.append(BeautifulSoup(dashboard_content, 'html.parser'))
### ============================ END OF DASHBOARD ============================================ ####
writer("\n2 of 6: Capturing suite metrics...")
### ============================ START OF SUITE METRICS ======================================= ####
# Tests div
suite_div = soup.new_tag('div')
suite_div["id"] = "suiteMetrics"
suite_div["class"] = "tabcontent"
page_content_div.insert(50, suite_div)
test_icon_txt = """
<h4><b><i class="fa fa-table"></i> Suite Metrics</b></h4>
<hr></hr>
"""
suite_div.append(BeautifulSoup(test_icon_txt, 'html.parser'))
# Create table tag
table = soup.new_tag('table')
table["id"] = "sm"
table["class"] = "table table-striped table-bordered"
suite_div.insert(10, table)
thead = soup.new_tag('thead')
table.insert(0, thead)
tr = soup.new_tag('tr')
thead.insert(0, tr)
th = soup.new_tag('th')
th.string = "Suite Name"
tr.insert(0, th)
th = soup.new_tag('th')
th.string = "Status"
tr.insert(1, th)
th = soup.new_tag('th')
th.string = "Total"
tr.insert(2, th)
th = soup.new_tag('th')
th.string = "Pass"
tr.insert(3, th)
th = soup.new_tag('th')
th.string = "Fail"
tr.insert(4, th)
th = soup.new_tag('th')
th.string = "Time (s)"
tr.insert(5, th)
suite_tbody = soup.new_tag('tbody')
table.insert(11, suite_tbody)
### =============== GET SUITE METRICS =============== ###
if group:
group.spawn(result.visit, SuiteResults(soup, suite_tbody, log_name))
else:
result.visit(SuiteResults(soup, suite_tbody, log_name))
test_icon_txt = """
<div class="row">
<div class="col-md-12" style="height:25px;width:auto;">
</div>
</div>
"""
suite_div.append(BeautifulSoup(test_icon_txt, 'html.parser'))
### ============================ END OF SUITE METRICS ============================================ ####
writer("\n3 of 6: Capturing test metrics...")
### ============================ START OF TEST METRICS ======================================= ####
# Tests div
tm_div = soup.new_tag('div')
tm_div["id"] = "testMetrics"
tm_div["class"] = "tabcontent"
page_content_div.insert(100, tm_div)
test_icon_txt = """
<h4><b><i class="fa fa-table"></i> Test Metrics</b></h4>
<hr></hr>
"""
tm_div.append(BeautifulSoup(test_icon_txt, 'html.parser'))
# Create table tag
table = soup.new_tag('table')
table["id"] = "tm"
table["class"] = "table table-striped table-bordered"
tm_div.insert(10, table)
thead = soup.new_tag('thead')
table.insert(0, thead)
tr = soup.new_tag('tr')
thead.insert(0, tr)
th = soup.new_tag('th')
th.string = "Suite Name"
tr.insert(0, th)
th = soup.new_tag('th')
th.string = "Test Case"
tr.insert(1, th)
th = soup.new_tag('th')
th.string = "Status"
tr.insert(2, th)
th = soup.new_tag('th')
th.string = "Time (s)"
tr.insert(3, th)
test_tbody = soup.new_tag('tbody')
table.insert(11, test_tbody)
### =============== GET TEST METRICS =============== ###
if group:
group.spawn(result.visit, TestResults(soup, test_tbody, log_name))
else:
result.visit(TestResults(soup, test_tbody, log_name))
test_icon_txt = """
<div class="row">
<div class="col-md-12" style="height:25px;width:auto;">
</div>
</div>
"""
tm_div.append(BeautifulSoup(test_icon_txt, 'html.parser'))
### ============================ END OF TEST METRICS ============================================ ####
writer("\n4 of 6: Capturing keyword metrics...")
### ============================ START OF KEYWORD METRICS ======================================= ####
# Keywords div
km_div = soup.new_tag('div')
km_div["id"] = "keywordMetrics"
km_div["class"] = "tabcontent"
page_content_div.insert(150, km_div)
keyword_icon_txt = """
<h4><b><i class="fa fa-table"></i> Keyword Metrics</b></h4>
<hr></hr>
"""
km_div.append(BeautifulSoup(keyword_icon_txt, 'html.parser'))
# Create table tag
# <table id="myTable">
table = soup.new_tag('table')
table["id"] = "km"
table["class"] = "table table-striped table-bordered"
km_div.insert(10, table)
thead = soup.new_tag('thead')
table.insert(0, thead)
tr = soup.new_tag('tr')
thead.insert(0, tr)
th = soup.new_tag('th')
th.string = "Test Case"
tr.insert(1, th)
th = soup.new_tag('th')
th.string = "Keyword"
tr.insert(1, th)
th = soup.new_tag('th')
th.string = "Status"
tr.insert(2, th)
th = soup.new_tag('th')
th.string = "Time (s)"
tr.insert(3, th)
kw_tbody = soup.new_tag('tbody')
table.insert(1, kw_tbody)
if group:
group.spawn(
result.visit,
KeywordResults(soup, kw_tbody, ignore_library, ignore_type))
group.join()
else:
result.visit(
KeywordResults(soup, kw_tbody, ignore_library, ignore_type))
test_icon_txt = """
<div class="row">
<div class="col-md-12" style="height:25px;width:auto;">
</div>
</div>
"""
km_div.append(BeautifulSoup(test_icon_txt, 'html.parser'))
### ============================ END OF KEYWORD METRICS ======================================= ####
### ============================ START OF LOGS ====================================== ###
# Logs div
log_div = soup.new_tag('div')
log_div["id"] = "log"
log_div["class"] = "tabcontent"
page_content_div.insert(200, log_div)
test_icon_txt = """
<p style="text-align:right">** <b>Report.html</b> and <b>Log.html</b> need to be in current folder in order to display here</p>
<div class="embed-responsive embed-responsive-4by3">
<iframe class="embed-responsive-item" src=%s></iframe>
</div>
""" % (log_name)
log_div.append(BeautifulSoup(test_icon_txt, 'html.parser'))
### ============================ END OF LOGS ======================================= ####
### ============================ EMAIL STATISTICS ================================== ###
# Statistics div
statisitcs_div = soup.new_tag('div')
statisitcs_div["id"] = "statistics"
statisitcs_div["class"] = "tabcontent"
page_content_div.insert(300, statisitcs_div)
emailStatistics = """
<h4><b><i class="fa fa-envelope-o"></i> Email Statistics</b></h4>
<hr></hr>
<button id="create" class="btn btn-primary active inner" role="button" onclick="updateTextArea();this.style.visibility= 'hidden';"><i class="fa fa-cogs"></i> Generate Statistics Email</button>
<a download="message.eml" class="btn btn-primary active inner" role="button" id="downloadlink" style="display: none; width: 300px;"><i class="fa fa-download"></i> Click Here To Download Email</a>
<script>
function updateTextArea() {
var suite = "<b>Top 10 Suite Performance:</b><br><br>" + $("#suiteBarID table")[0].outerHTML;
var test = "<b>Top 10 Test Performance:</b><br><br>" + $("#testsBarID table")[0].outerHTML;
var keyword ="<b>Top 10 Keyword Performance:</b><br><br>" + $("#keywordsBarID table")[0].outerHTML;
var saluation="<pre><br>Please refer RF Metrics Report for detailed statistics.<br><br>Regards,<br>QA Team</pre></body></html>";
document.getElementById("textbox").value += "<br>" + suite + "<br>" + test + "<br>" + keyword + saluation;
$("#create").click(function(){
$(this).remove();
});
}
</script>
<textarea id="textbox" class="col-md-12" style="height: 400px; padding:1em;">
To: [email protected]
Subject: Automation Execution Status
X-Unsent: 1
Content-Type: text/html
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<title>Test Email Sample</title>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<meta name="viewport" content="width=device-width, initial-scale=1.0 " />
<style>
body {
background-color:#F2F2F2;
}
body, html, table,pre,b {
font-family: Calibri, Arial, sans-serif;
font-size: 1em;
}
.pastdue { color: crimson; }
table {
border: 1px solid silver;
padding: 6px;
margin-left: 30px;
width: 600px;
}
thead {
text-align: center;
font-size: 1.1em;
background-color: #B0C4DE;
font-weight: bold;
color: #2D2C2C;
}
tbody {
text-align: center;
}
th {
width: 25%%;
word-wrap:break-word;
}
</style>
</head>
<body><pre>Hi Team,
Following are the last build execution statistics.
<b>Metrics:<b>
</pre>
<table>
<thead>
<th style="width: 25%%;">Statistics</th>
<th style="width: 25%%;">Total</th>
<th style="width: 25%%;">Pass</th>
<th style="width: 25%%;">Fail</th>
</thead>
<tbody>
<tr>
<td style="text-align: left;font-weight: bold;"> SUITE </td>
<td style="background-color: #F5DEB3;text-align: center;">%s</td>
<td style="background-color: #90EE90;text-align: center;">%s</td>
<td style="background-color: #F08080;text-align: center;">%s</td>
</tr>
<tr>
<td style="text-align: left;font-weight: bold;"> TESTS </td>
<td style="background-color: #F5DEB3;text-align: center;">%s</td>
<td style="background-color: #90EE90;text-align: center;">%s</td>
<td style="background-color: #F08080;text-align: center;">%s</td>
</tr>
<tr>
<td style="text-align: left;font-weight: bold;"> KEYWORDS </td>
<td style="background-color: #F5DEB3;text-align: center;">%s</td>
<td style="background-color: #90EE90;text-align: center;">%s</td>
<td style="background-color: #F08080;text-align: center;">%s</td>
</tr>
</tbody>
</table>
</textarea>
""" % (total_suite, passed_suite, failed_suite, total, passed, failed,
total_keywords, passed_keywords, failed_keywords)
statisitcs_div.append(BeautifulSoup(emailStatistics, 'html.parser'))
### ============================ END OF EMAIL STATISTICS ================================== ###
script_text = """
<script>
(function () {
var textFile = null,
makeTextFile = function (text) {
var data = new Blob([text], {type: 'text/plain'});
if (textFile !== null) {
window.URL.revokeObjectURL(textFile);
}
textFile = window.URL.createObjectURL(data);
return textFile;
};
var create = document.getElementById('create'),
textbox = document.getElementById('textbox');
create.addEventListener('click', function () {
var link = document.getElementById('downloadlink');
link.href = makeTextFile(textbox.value);
link.style.display = 'block';
}, false);
})();
</script>
<script>
function createPieChart(passed_count,failed_count,ChartID,ChartName){
var status = [];
status.push(['Status', 'Percentage']);
status.push(['PASS',parseInt(passed_count)],['FAIL',parseInt(failed_count)]);
var data = google.visualization.arrayToDataTable(status);
var options = {
pieHole: 0.6,
legend: 'none',
chartArea: {width: "95%",height: "90%"},
colors: ['green', 'red'],
};
var chart = new google.visualization.PieChart(document.getElementById(ChartID));
chart.draw(data, options);
}
</script>
<script>
function createBarGraph(tableID,keyword_column,time_column,limit,ChartID,Label,type){
var status = [];
css_selector_locator = tableID + ' tbody >tr'
var rows = $(css_selector_locator);
var columns;
var myColors = [
'#4F81BC',
'#C0504E',
'#9BBB58',
'#24BEAA',
'#8064A1',
'#4AACC5',
'#F79647',
'#815E86',
'#76A032',
'#34558B'
];
status.push([type, Label,{ role: 'annotation'}, {role: 'style'}]);
for (var i = 0; i < rows.length; i++) {
if (i == Number(limit)){
break;
}
//status = [];
name_value = $(rows[i]).find('td');
time=($(name_value[Number(time_column)]).html()).trim();
keyword=($(name_value[Number(keyword_column)]).html()).trim();
status.push([keyword,parseFloat(time),parseFloat(time),myColors[i]]);
}
var data = google.visualization.arrayToDataTable(status);
var options = {
legend: 'none',
chartArea: {width: "92%",height: "75%"},
bar: {
groupWidth: '90%'
},
annotations: {
alwaysOutside: true,
textStyle: {
fontName: 'Comic Sans MS',
fontSize: 13,
bold: true,
italic: true,
color: "black", // The color of the text.
},
},
hAxis: {
textStyle: {
fontName: 'Arial',
fontSize: 10,
}
},
vAxis: {
gridlines: { count: 10 },
textStyle: {
fontName: 'Comic Sans MS',
fontSize: 10,
}
},
};
// Instantiate and draw the chart.
var chart = new google.visualization.ColumnChart(document.getElementById(ChartID));
chart.draw(data, options);
}
</script>
<script>
function executeDataTable(tabname,sortCol) {
var fileTitle;
switch(tabname) {
case "#sm":
fileTitle = "SuiteMetrics";
break;
case "#tm":
fileTitle = "TestMetrics";
break;
case "#km":
fileTitle = "KeywordMetrics";
break;
default:
fileTitle = "metrics";
}
$(tabname).DataTable(
{
retrieve: true,
"order": [[ Number(sortCol), "desc" ]],
dom: 'l<".margin" B>frtip',
buttons: [
'copy',
{
extend: 'csv',
filename: function() {
return fileTitle + '-' + new Date().toLocaleString();
},
title : '',
},
{
extend: 'excel',
filename: function() {
return fileTitle + '-' + new Date().toLocaleString();
},
title : '',
},
{
extend: 'pdf',
filename: function() {
return fileTitle + '-' + new Date().toLocaleString();
},
title : '',
},
{
extend: 'print',
title : '',
},
],
}
);
}
</script>
<script>
function openPage(pageName,elmnt,color) {
var i, tabcontent, tablinks;
tabcontent = document.getElementsByClassName("tabcontent");
for (i = 0; i < tabcontent.length; i++) {
tabcontent[i].style.display = "none";
}
tablinks = document.getElementsByClassName("tablink");
for (i = 0; i < tablinks.length; i++) {
tablinks[i].style.backgroundColor = "";
}
document.getElementById(pageName).style.display = "block";
elmnt.style.backgroundColor = color;
}
// Get the element with id="defaultOpen" and click on it
document.getElementById("defaultOpen").click();
</script>
<script>
// Get the element with id="defaultOpen" and click on it
document.getElementById("defaultOpen").click();
</script>
<script>
$(window).on('load',function(){$('.loader').fadeOut();});
</script>
"""
body.append(BeautifulSoup(script_text, 'html.parser'))
### ====== WRITE TO RF_METRICS_REPORT.HTML ===== ###
# Write output as html file
with open(result_file, 'w') as outfile:
outfile.write(soup.prettify())
# Wait for 2 seconds - File is generated
time.sleep(2)
# ====== EMAIL CONTENT ========== #
email_content = """
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<title>Robotframework Metrics</title>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<meta name="viewport" content="width=device-width, initial-scale=1.0 " />
<style>
body {
background-color:#F2F2F2;
}
body, html, table,span,b {
font-family: Calibri, Arial, sans-serif;
font-size: 1em;
}
.pastdue { color: crimson; }
table {
border: 1px solid silver;
padding: 6px;
margin-left: 30px;
width: 600px;
}
thead {
text-align: center;
font-size: 1.1em;
background-color: #B0C4DE;
font-weight: bold;
color: #2D2C2C;
}
tbody {
text-align: center;
}
th {
word-wrap:break-word;
}
td {
height: 25px;
}
.dt-buttons {
margin-left: 30px;
}
</style>
</head>
<body>
<span>Hi Team,<br>Following are the last build execution status.<br><br><b>Metrics:<b><br><br></span>
<table>
<thead>
<th style="width: 25vh;"> Stats </th>
<th style="width: 20vh;"> Total </th>
<th style="width: 20vh;"> Pass </th>
<th style="width: 20vh;"> Fail </th>
<th style="width: 15vh;"> Perc (%%)</th>
</thead>
<tbody>
<tr>
<td style="text-align: left;font-weight: bold;"> SUITE </td>
<td style="text-align: center;">%s</td>
<td style="text-align: center;">%s</td>
<td style="text-align: center;">%s</td>
<td style="text-align: center;">%s</td>
</tr>
<tr>
<td style="text-align: left;font-weight: bold;"> TESTS </td>
<td style="text-align: center;">%s</td>
<td style="text-align: center;">%s</td>
<td style="text-align: center;">%s</td>
<td style="text-align: center;">%s</td>
</tr>
<tr>
<td style="text-align: left;font-weight: bold;"> KEYWORDS </td>
<td style="text-align: center;">%s</td>
<td style="text-align: center;">%s</td>
<td style="text-align: center;">%s</td>
<td style="text-align: center;">%s</td>
</tr>
</tbody>
</table>
<span><br><b>Info:<b><br><br></span>
<table>
<tbody>
<tr>
<td style="text-align: left;font-weight: normal;width: 30vh;"> Execution Time </td>
<td style="text-align: center;font-weight: normal;">%s h</td>
</tr>
<tr>
<td style="text-align: left;font-weight: normal;width: 50vh;"> Generated By </td>
<td style="text-align: center;font-weight: normal;">%s</td>
</tr>
</tbody>
</table>
<span style="text-align: left;font-weight: normal;"><br>Please refer robotframework-metrics report for detailed info.<br><br>Regards,<br>QA Team</span>
</body></html>
""" % (total_suite, passed_suite, failed_suite, suitepp, total, passed,
failed, testpp, total_keywords, passed_keywords, failed_keywords,
kwpp, elapsedtime, generator)
#msg.set_payload(email_content)
msg.attach(MIMEText(email_content, 'html'))
# Attach robotframework file
rfmetrics = MIMEBase('application', "octet-stream")
rfmetrics.set_payload(open(result_file, "rb").read())
encoders.encode_base64(rfmetrics)
attachmentName = 'attachment; filename=%s' % (result_file_name)
rfmetrics.add_header('Content-Disposition', attachmentName)
msg.attach(rfmetrics)
if send_email:
# Start server
server.starttls()
writer("\n5 of 6: Sending email with robotmetrics.html...")
# Login Credentials for sending the mail
server.login(msg['From'], password)
server.sendmail(sender, recipients, msg.as_string())
writer("\n6 of 6: Email sent successfully!")
else:
writer("\n6 of 6: Skipping step 5 (send email)!")
writer(
"\nResults file created successfully and can be found at {}\n".format(
result_file))
class Producer(object):
"""High level NSQ producer.
A Producer will connect to the nsqd tcp addresses and support async
publishing (``PUB`` & ``MPUB`` & ``DPUB``) of messages to `nsqd` over the
TCP protocol.
Example publishing a message::
from gnsq import Producer
producer = Producer('localhost:4150')
producer.start()
producer.publish('topic', b'hello world')
:param nsqd_tcp_addresses: a sequence of string addresses of the nsqd
instances this consumer should connect to
:param max_backoff_duration: the maximum time we will allow a backoff state
to last in seconds. If zero, backoff wil not occur
:param **kwargs: passed to :class:`~gnsq.NsqdTCPClient` initialization
"""
def __init__(self,
nsqd_tcp_addresses=[],
max_backoff_duration=128,
**kwargs):
if not nsqd_tcp_addresses:
raise ValueError('must specify at least one nsqd or lookupd')
self.nsqd_tcp_addresses = parse_nsqds(nsqd_tcp_addresses)
self.max_backoff_duration = max_backoff_duration
self.conn_kwargs = kwargs
self.logger = logging.getLogger(__name__)
self._state = INIT
self._connections = Queue()
self._connection_backoffs = defaultdict(self._create_backoff)
self._response_queues = {}
self._workers = Group()
@cached_property
def on_response(self):
"""Emitted when a response is received.
The signal sender is the consumer and the ` ` is sent as an
argument.
"""
return blinker.Signal(doc='Emitted when a response is received.')
@cached_property
def on_error(self):
"""Emitted when an error is received.
The signal sender is the consumer and the ``error`` is sent as an
argument.
"""
return blinker.Signal(doc='Emitted when a error is received.')
@cached_property
def on_auth(self):
"""Emitted after a connection is successfully authenticated.
The signal sender is the consumer and the ``conn`` and parsed
``response`` are sent as arguments.
"""
return blinker.Signal(doc='Emitted when a response is received.')
@cached_property
def on_close(self):
"""Emitted after :meth:`close`.
The signal sender is the consumer.
"""
return blinker.Signal(doc='Emitted after the consumer is closed.')
def start(self):
"""Start discovering and listing to connections."""
if self._state == CLOSED:
raise NSQException('producer already closed')
if self.is_running:
self.logger.warn('producer already started')
return
self.logger.debug('starting producer...')
self._state = RUNNING
for address in self.nsqd_tcp_addresses:
address, port = address.split(':')
self.connect_to_nsqd(address, int(port))
def close(self):
"""Immediately close all connections and stop workers."""
if not self.is_running:
return
self._state = CLOSED
self.logger.debug('closing connection(s)')
while True:
try:
conn = self._connections.get(block=False)
except Empty:
break
conn.close_stream()
self.on_close.send(self)
def join(self, timeout=None, raise_error=False):
"""Block until all connections have closed and workers stopped."""
self._workers.join(timeout, raise_error)
@property
def is_running(self):
"""Check if the producer is currently running."""
return self._state == RUNNING
def connect_to_nsqd(self, address, port):
if not self.is_running:
return
conn = NsqdTCPClient(address, port, **self.conn_kwargs)
self.logger.debug('[%s] connecting...', conn)
conn.on_response.connect(self.handle_response)
conn.on_error.connect(self.handle_error)
conn.on_auth.connect(self.handle_auth)
try:
conn.connect()
conn.identify()
except NSQException as error:
self.logger.warn('[%s] connection failed (%r)', conn, error)
self.handle_connection_failure(conn)
return
# Check if we've closed since we started
if not self.is_running:
self.handle_connection_failure(conn)
return
self.logger.info('[%s] connection successful', conn)
self.handle_connection_success(conn)
def _listen(self, conn):
try:
conn.listen()
except NSQException as error:
self.logger.warning('[%s] connection lost (%r)', conn, error)
self.handle_connection_failure(conn)
def handle_connection_success(self, conn):
self._response_queues[conn] = deque()
self._put_connection(conn)
self._workers.spawn(self._listen, conn)
self._connection_backoffs[conn].success()
def handle_connection_failure(self, conn):
conn.close_stream()
self._clear_responses(conn, NSQException('connection closed'))
if not self.is_running:
return
seconds = self._connection_backoffs[conn].failure().get_interval()
self.logger.debug('[%s] retrying in %ss', conn, seconds)
gevent.spawn_later(seconds, self.connect_to_nsqd, conn.address,
conn.port)
def handle_auth(self, conn, response):
metadata = []
if response.get('identity'):
metadata.append("Identity: %r" % response['identity'])
if response.get('permission_count'):
metadata.append("Permissions: %d" % response['permission_count'])
if response.get('identity_url'):
metadata.append(response['identity_url'])
self.logger.info('[%s] AUTH accepted %s', conn, ' '.join(metadata))
self.on_auth.send(self, conn=conn, response=response)
def handle_response(self, conn, response):
self.logger.debug('[%s] response: %s', conn, response)
if response == nsq.OK:
if conn in self._response_queues:
result = self._response_queues[conn].popleft()
result.set(response)
self.on_response.send(self, response=response)
def handle_error(self, conn, error):
self.logger.debug('[%s] error: %s', conn, error)
self._clear_responses(conn, error)
self.on_error.send(self, error=error)
def _create_backoff(self):
return BackoffTimer(max_interval=self.max_backoff_duration)
def _clear_responses(self, conn, error):
# All relevent errors are fatal
for result in self._response_queues.pop(conn, []):
result.set_exception(error)
def _get_connection(self, block=True, timeout=None):
if not self.is_running:
raise NSQException('producer not running')
while True:
try:
conn = self._connections.get(block=block, timeout=timeout)
except Empty:
raise NSQNoConnections
if conn.is_connected:
return conn
# Discard closed connections
def _put_connection(self, conn):
if not self.is_running:
return
self._connections.put(conn)
def publish(self,
topic,
data,
defer=None,
block=True,
timeout=None,
raise_error=True):
"""Publish a message to the given topic.
:param topic: the topic to publish to
:param data: bytestring data to publish
:param defer: duration in milliseconds to defer before publishing
(requires nsq 0.3.6)
:param block: wait for a connection to become available before
publishing the message. If block is `False` and no connections
are available, :class:`~gnsq.errors.NSQNoConnections` is raised
:param timeout: if timeout is a positive number, it blocks at most
``timeout`` seconds before raising
:class:`~gnsq.errors.NSQNoConnections`
:param raise_error: if ``True``, it blocks until a response is received
from the nsqd server, and any error response is raised. Otherwise
an :class:`~gevent.event.AsyncResult` is returned
"""
result = AsyncResult()
conn = self._get_connection(block=block, timeout=timeout)
try:
self._response_queues[conn].append(result)
conn.publish(topic, data, defer=defer)
finally:
self._put_connection(conn)
if raise_error:
return result.get()
return result
def multipublish(self,
topic,
messages,
block=True,
timeout=None,
raise_error=True):
"""Publish an iterable of messages to the given topic.
:param topic: the topic to publish to
:param messages: iterable of bytestrings to publish
:param block: wait for a connection to become available before
publishing the message. If block is `False` and no connections
are available, :class:`~gnsq.errors.NSQNoConnections` is raised
:param timeout: if timeout is a positive number, it blocks at most
``timeout`` seconds before raising
:class:`~gnsq.errors.NSQNoConnections`
:param raise_error: if ``True``, it blocks until a response is received
from the nsqd server, and any error response is raised. Otherwise
an :class:`~gevent.event.AsyncResult` is returned
"""
result = AsyncResult()
conn = self._get_connection(block=block, timeout=timeout)
try:
self._response_queues[conn].append(result)
conn.multipublish(topic, messages)
finally:
self._put_connection(conn)
if raise_error:
return result.get()
return result
def get_all_company():
company_group = Group()
for page in range(1, 500):
company_group.add(gevent.spawn(get_company_page, page))
company_group.join()
def my_task(self):
from gevent.pool import Group
group = Group()
group.spawn(lambda: self.query1())
group.spawn(lambda: self.query2())
group.join() # wait for greenlets to finish
class StressTest(object):
def __init__(self,logger=None,redis_manage_obj=None):
self._logger = logger
self._redis_manange_obj = redis_manage_obj
self._group = None
self._pid = None
self._schedule = None
self._stress_count = None
def init(self):
try:
if self._redis_manange_obj:
self._redis_manange_obj.init_redis_pool()
self._group = Group()
self._pid = os.getpid()
self._stress_count = StressCount()
self._schedule = sched.scheduler(time.time, time.sleep)
except:
self._logger.error(traceback.format_exc())
def schedule(self,msg):
try:
while True:
self._schedule.enter(1,0,self.sectond_print,("test1",))
self._schedule.run()
except:
self._logger.error(traceback.format_exc())
def sectond_print(self,msg):
try:
if self._stress_count:
total_send = self._stress_count._success + self._stress_count._fail
if total_send > 0:
avg_rsp_time = self._stress_count._avg_rsp_time/total_send
info = "process pid:%d,total:%d,success:%d,fail:%d,avg_rsp_time(ms):%d"%(self._pid,total_send,self._stress_count._success,self._stress_count._fail,avg_rsp_time)
self._logger.info(info)
self._stress_count.reset()
except:
self._logger.error(traceback.format_exc())
def process_data(self,data_list):
try:
redis_conn = self._redis_manange_obj.get_redis_obj()
while True:
try:
if len(data_list):
if redis_conn:
for item in data_list:
if not redis_conn.exists(item):
#self._logger.error(item)
continue
begin_time = time.time()
field = GLOBAL_HASH[item].split('=')[0]
result = redis_conn.hget(item,field)
end_time = time.time()
off_time = (end_time - begin_time) * 1000
if result:
self._stress_count.set_stress_count(True,off_time)
# info = "offset_time(ms):%ld"%off_time
# self._logger.info(info)
else:
self._stress_count.set_stress_count(False,off_time)
else:
self._logger.error("error")
except:
self._logger.error(traceback.format_exc())
except SocketError as e:
if e.errno != errno.ECONNRESET:
self._logger.error("socket connect reset!!!")
self.process_data(data_list)
except:
self._logger.error(traceback.format_exc())
def slice_data_list(self):
try:
n_group = len(GLOBAL_LIST)/GLOBAL_RECOERS_NUMS
n_offset = len(GLOBAL_LIST)%GLOBAL_RECOERS_NUMS
if n_offset > 0:
n_groups = n_group + 1
else:
n_groups = n_group
for i in range(n_groups):
if i == n_groups - 1:
begin = i * GLOBAL_RECOERS_NUMS
end = i * GLOBAL_RECOERS_NUMS + n_offset
else:
begin = i * GLOBAL_RECOERS_NUMS
end = i * GLOBAL_RECOERS_NUMS + GLOBAL_RECOERS_NUMS
index_list = GLOBAL_LIST[begin:end]
g = gevent.spawn(self.process_data,index_list)
self._group.add(g)
g = gevent.spawn(self.schedule,"test1")
self._group.add(g)
self._group.join()
except:
self._logger.error(traceback.format_exc())
def hash_stress_test(self):
try:
if self._redis_manange_obj:
redis_conn = self._redis_manange_obj.get_redis_obj()
if redis_conn:
for item in GLOBAL_HASH:
if not redis_conn.exists(item):
#self._logger.error(item)
continue
result = redis_conn.hget(item,GLOBAL_HASH[item])
# if result:
# self._logger.info(result)
except:
self._logger.error(traceback.format_exc())
def string_stress_test(self):
try:
if self._redis_manange_obj:
redis_conn = self._redis_manange_obj.get_redis_obj()
if redis_conn:
for item in GLOBAL_SET:
redis_conn.get(item)
except:
self._logger.error(traceback.format_exc())
def list_stress_test(self):
try:
if self._redis_manange_obj:
redis_conn = self._redis_manange_obj.get_redis_obj()
except:
self._logger.error(traceback.format_exc())
def generate_report(opts):
logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.INFO)
group = Group() if not FAILED_IMPORT else ''
# START OF CUSTOMIZE REPORT
# URL or filepath of your company logo
logo = opts.logo
# Ignores following library keywords in metrics report
ignore_library = IGNORE_LIBRARIES
if opts.ignore:
ignore_library.extend(opts.ignore)
# Ignores following type keywords in metrics report
ignore_type = IGNORE_TYPES
if opts.ignoretype:
ignore_type.extend(opts.ignoretype)
# END OF CUSTOMIZE REPORT
# Report to support file location as arguments
# Source Code Contributed By : Ruud Prijs
# input directory
path = os.path.abspath(os.path.expanduser(opts.path))
# output.xml files
output_names = []
# support "*.xml" of output files
if (opts.output == "*.xml"):
for item in os.listdir(path):
if os.path.isfile(item) and item.endswith('.xml'):
output_names.append(item)
else:
for curr_name in opts.output.split(","):
curr_path = os.path.join(path, curr_name)
output_names.append(curr_path)
# log.html file
log_name = opts.log_name
# copy the list of output_names onto the one of required_files; the latter may (in the future)
# contain files that should not be processed as output_names
required_files = list(output_names)
missing_files = [
filename for filename in required_files if not os.path.exists(filename)
]
if missing_files:
# We have files missing.
exit("output.xml file is missing: {}".format(", ".join(missing_files)))
mt_time = datetime.now().strftime('%Y%m%d-%H%M%S')
# Output result file location
if opts.metrics_report_name:
result_file_name = opts.metrics_report_name
else:
result_file_name = 'metrics-' + mt_time + '.html'
result_file = os.path.join(path, result_file_name)
# Read output.xml file
result = ExecutionResult(*output_names)
result.configure(stat_config={
'suite_stat_level': 2,
'tag_stat_combine': 'tagANDanother'
})
logging.info("Converting .xml to .html file. This may take few minutes...")
head_content = """
<!doctype html><html lang="en">
<head>
<link rel="shortcut icon" href="https://png.icons8.com/windows/50/000000/bot.png" type="image/x-icon" />
<title>RF Metrics</title>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<link href="https://cdn.datatables.net/1.10.19/css/jquery.dataTables.min.css" rel="stylesheet"/>
<link href="https://cdn.datatables.net/buttons/1.5.2/css/buttons.dataTables.min.css" rel="stylesheet"/>
<link href="https://cdnjs.cloudflare.com/ajax/libs/twitter-bootstrap/4.1.3/css/bootstrap.min.css" rel="stylesheet"/>
<link href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css" rel="stylesheet">
<script src="https://code.jquery.com/jquery-3.3.1.js" type="text/javascript"></script>
<!-- Bootstrap core Googleccharts -->
<script src="https://www.gstatic.com/charts/loader.js" type="text/javascript"></script>
<script type="text/javascript">google.charts.load('current', {packages: ['corechart']});</script>
<!-- Bootstrap core Datatable-->
<script src="https://cdn.datatables.net/1.10.19/js/jquery.dataTables.min.js" type="text/javascript"></script>
<script src="https://cdn.datatables.net/buttons/1.5.2/js/dataTables.buttons.min.js" type="text/javascript"></script>
<script src="https://cdn.datatables.net/buttons/1.5.2/js/buttons.flash.min.js" type="text/javascript"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/jszip/3.1.3/jszip.min.js" type="text/javascript"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/pdfmake/0.1.36/pdfmake.min.js" type="text/javascript"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/pdfmake/0.1.36/vfs_fonts.js" type="text/javascript"></script>
<script src="https://cdn.datatables.net/buttons/1.5.2/js/buttons.html5.min.js" type="text/javascript"></script>
<script src="https://cdn.datatables.net/buttons/1.5.2/js/buttons.print.min.js" type="text/javascript"></script>
<style>
body {
font-family: -apple-system,sans-serif;
}
.sidenav {
height: 100%;
width: 220px;
position: fixed;
z-index: 1;
top: 0;
left: 0;
background-color: white;
overflow-x: hidden;
border-style: ridge;
}
.sidenav a {
padding: 12px 10px 8px 12px;
text-decoration: none;
font-size: 18px;
color: Black;
display: block;
}
.main {
padding-top: 10px;
}
@media screen and (max-height: 450px) {
.sidenav {padding-top: 15px;}
.sidenav a {font-size: 18px;}
}
.tile {
width: 100%;
float: left;
margin: 0px;
list-style: none;
font-size: 30px;
color: #FFF;
-moz-border-radius: 5px;
-webkit-border-radius: 5px;
margin-bottom: 5px;
position: relative;
text-align: center;
color: white!important;
}
.tile.tile-fail {
background: #f44336!important;
}
.tile.tile-pass {
background: #4CAF50!important;
}
.tile.tile-info {
background: #009688!important;
}
.tile.tile-head {
background: #616161!important;
}
.dt-buttons {
margin-left: 5px;
}
.loader {
position: fixed;
left: 0px;
top: 0px;
width: 100%;
height: 100%;
z-index: 9999;
background: url('https://www.downgraf.com/wp-content/uploads/2014/09/02-loading-blossom-2x.gif')
50% 50% no-repeat rgb(249,249,249);
}
</style>
</head>
"""
soup = BeautifulSoup(head_content, "html.parser")
body = soup.new_tag('body')
soup.insert(20, body)
icons_txt = """
<div class="loader"></div>
<div class="sidenav">
<a> <img src="%s" style="height:20vh;max-width:98%%;"/> </a>
<a class="tablink" href="#" id="defaultOpen" onclick="openPage('dashboard', this, 'orange')"><i class="fa fa-dashboard"></i> Dashboard</a>
<a class="tablink" href="#" onclick="openPage('suiteMetrics', this, 'orange'); executeDataTable('#sm',5)"><i class="fa fa-th-large"></i> Suite Metrics</a>
<a class="tablink" href="#" onclick="openPage('testMetrics', this, 'orange'); executeDataTable('#tm',3)"><i class="fa fa-list-alt"></i> Test Metrics</a>
<a class="tablink" href="#" onclick="openPage('keywordMetrics', this, 'orange'); executeDataTable('#km',3)"><i class="fa fa-table"></i> Keyword Metrics</a>
<a class="tablink" href="#" onclick="openPage('log', this, 'orange');"><i class="fa fa-wpforms"></i> Logs</a>
<a class="tablink" href="#" onclick="openPage('statistics', this, 'orange');"><i class="fa fa-envelope-o"></i> Email</a>
</div>
""" % logo
body.append(BeautifulSoup(icons_txt, 'html.parser'))
page_content_div = soup.new_tag('div')
page_content_div["class"] = "main col-md-9 ml-sm-auto col-lg-10 px-4"
body.insert(50, page_content_div)
logging.info("1 of 4: Capturing dashboard content...")
test_stats = TestStats()
result.visit(test_stats)
total_suite = test_stats.total_suite
passed_suite = test_stats.passed_suite
failed_suite = test_stats.failed_suite
suitepp = round(passed_suite * 100.0 / total_suite, 1)
suitefp = round(failed_suite * 100.0 / total_suite, 1)
elapsedtime = datetime(
1970, 1, 1) + timedelta(milliseconds=result.suite.elapsedtime)
elapsedtime = elapsedtime.strftime("%X")
my_results = result.generated_by_robot
if my_results:
generator = "Robot"
else:
generator = "Rebot"
stats = result.statistics
total = stats.total.all.total
passed = stats.total.all.passed
failed = stats.total.all.failed
testpp = round(passed * 100.0 / total, 1)
testfp = round(failed * 100.0 / total, 1)
kw_stats = KeywordStats(ignore_library, ignore_type)
result.visit(kw_stats)
total_keywords = kw_stats.total_keywords
passed_keywords = kw_stats.passed_keywords
failed_keywords = kw_stats.failed_keywords
# Handling ZeroDivisionError exception when no keywords are found
if total_keywords > 0:
kwpp = round(passed_keywords * 100.0 / total_keywords, 1)
kwfp = round(failed_keywords * 100.0 / total_keywords, 1)
else:
kwpp = 0
kwfp = 0
dashboard_content = """
<div class="tabcontent" id="dashboard">
<div id="stats_screenshot_area">
<div class="d-flex flex-column flex-md-row align-items-center p-1 mb-3 bg-light
border-bottom shadow-sm">
<h5 class="my-0 mr-md-auto font-weight-normal"><i class="fa fa-dashboard"></i> Dashboard</h5>
<nav class="my-2 my-md-0 mr-md-3" style="color:red">
<a class="p-2"><b style="color:black;">Execution Time: </b>%s h</a>
<a class="p-2"><b style="color:black;cursor: pointer;" data-toggle="tooltip" title=".xml file is created by">Generated By: </b>%s</a>
</nav>
</div>
<div class="row">
<div class="col-md-3" onclick="openPage('suiteMetrics', this, '')" data-toggle="tooltip"
title="Click to view Suite metrics" style="cursor: pointer;">
<a class="tile tile-head">
Suite
<p style="font-size:12px">Statistics</p>
</a>
</div>
<div class="col-md-3">
<a class="tile tile-info">
%s
<p style="font-size:12px">Total</p>
</a>
</div>
<div class="col-md-3">
<a class="tile tile-pass">
%s
<p style="font-size:12px">Pass</p>
</a>
</div>
<div class="col-md-3">
<a class="tile tile-fail">
%s
<p style="font-size:12px">Fail</p>
</a>
</div>
</div>
<div class="row">
<div class="col-md-3" onclick="openPage('testMetrics', this, '')" data-toggle="tooltip"
title="Click to view Test metrics" style="cursor: pointer;">
<a class="tile tile-head">
Test
<p style="font-size:12px">Statistics</p>
</a>
</div>
<div class="col-md-3">
<a class="tile tile-info">
%s
<p style="font-size:12px">Total</p>
</a>
</div>
<div class="col-md-3">
<a class="tile tile-pass">
%s
<p style="font-size:12px">Pass</p>
</a>
</div>
<div class="col-md-3">
<a class="tile tile-fail">
%s
<p style="font-size:12px">Fail</p>
</a>
</div>
</div>
<div class="row">
<div class="col-md-3" onclick="openPage('keywordMetrics', this, '')" data-toggle="tooltip"
title="Click to view Keyword metrics" style="cursor: pointer;">
<a class="tile tile-head">
Keyword
<p style="font-size:12px">Statistics</p>
</a>
</div>
<div class="col-md-3">
<a class="tile tile-info">
%s
<p style="font-size:12px">Total</p>
</a>
</div>
<div class="col-md-3">
<a class="tile tile-pass">
%s
<p style="font-size:12px">Pass</p>
</a>
</div>
<div class="col-md-3">
<a class="tile tile-fail">
%s
<p style="font-size:12px">Fail</p>
</a>
</div>
</div>
<hr></hr>
<div class="row">
<div class="col-md-4" style="background-color:white;height:280px;width:auto;border:groove;">
<span style="font-weight:bold">Suite Status:</span>
<div id="suiteChartID" style="height:250px;width:auto;"></div>
</div>
<div class="col-md-4" style="background-color:white;height:280px;width:auto;border:groove;">
<span style="font-weight:bold">Test Status:</span>
<div id="testChartID" style="height:250px;width:auto;"></div>
</div>
<div class="col-md-4" style="background-color:white;height:280px;width:auto;border:groove;">
<span style="font-weight:bold">Keyword Status:</span>
<div id="keywordChartID" style="height:250px;width:auto;"></div>
</div>
</div>
</div>
<hr></hr>
<div class="row">
<div class="col-md-12" style="background-color:white;height:450px;width:auto;border:groove;">
<span style="font-weight:bold">Top 10 Suite Performance(sec):</span>
<div id="suiteBarID" style="height:400px;width:auto;"></div>
</div>
<div class="col-md-12" style="background-color:white;height:450px;width:auto;border:groove;">
<span style="font-weight:bold">Top 10 Test Performance(sec):</span>
<div id="testsBarID" style="height:400px;width:auto;"></div>
</div>
<div class="col-md-12" style="background-color:white;height:450px;width:auto;border:groove;">
<span style="font-weight:bold">Top 10 Keywords Performance(sec):</span>
<div id="keywordsBarID" style="height:400px;width:auto;"></div>
</div>
</div>
<div class="row">
<div class="col-md-12" style="height:25px;width:auto;">
<p class="text-muted" style="text-align:center;font-size:9px">
<a target="_blank" href="https://github.com/adiralashiva8/robotframework-metrics"> robotframework-metrics </a>
</p>
</div>
</div>
<script>
window.onload = function(){
executeDataTable('#sm',5);
executeDataTable('#tm',3);
executeDataTable('#km',3);
createPieChart(%s,%s,'suiteChartID','Suite Status:');
createBarGraph('#sm',0,5,10,'suiteBarID','Elapsed Time (s) ','Suite');
createPieChart(%s,%s,'testChartID','Tests Status:');
createBarGraph('#tm',1,3,10,'testsBarID','Elapsed Time (s) ','Test');
createPieChart(%s,%s,'keywordChartID','Keywords Status:');
createBarGraph('#km',1,3,10,'keywordsBarID','Elapsed Time (s) ','Keyword');
};
</script>
<script>
function openInNewTab(url,element_id) {
var element_id= element_id;
var win = window.open(url, '_blank');
win.focus();
$('body').scrollTo(element_id);
}
</script>
</div>
""" % (elapsedtime, generator, total_suite, passed_suite, failed_suite,
total, passed, failed, total_keywords, passed_keywords,
failed_keywords, passed_suite, failed_suite, passed, failed,
passed_keywords, failed_keywords)
page_content_div.append(BeautifulSoup(dashboard_content, 'html.parser'))
### ============================ END OF DASHBOARD ============================================ ####
logging.info("2 of 4: Capturing suite metrics...")
### ============================ START OF SUITE METRICS ======================================= ####
# Tests div
suite_div = soup.new_tag('div')
suite_div["id"] = "suiteMetrics"
suite_div["class"] = "tabcontent"
page_content_div.insert(50, suite_div)
test_icon_txt = """
<h4><b><i class="fa fa-table"></i> Suite Metrics</b></h4>
<hr></hr>
"""
suite_div.append(BeautifulSoup(test_icon_txt, 'html.parser'))
# Create table tag
table = soup.new_tag('table')
table["id"] = "sm"
table["class"] = "table table-striped table-bordered"
suite_div.insert(10, table)
thead = soup.new_tag('thead')
table.insert(0, thead)
tr = soup.new_tag('tr')
thead.insert(0, tr)
th = soup.new_tag('th')
th.string = "Suite Name"
tr.insert(0, th)
th = soup.new_tag('th')
th.string = "Status"
tr.insert(1, th)
th = soup.new_tag('th')
th.string = "Total"
tr.insert(2, th)
th = soup.new_tag('th')
th.string = "Pass"
tr.insert(3, th)
th = soup.new_tag('th')
th.string = "Fail"
tr.insert(4, th)
th = soup.new_tag('th')
th.string = "Time (s)"
tr.insert(5, th)
suite_tbody = soup.new_tag('tbody')
table.insert(11, suite_tbody)
# GET SUITE METRICS
if group:
group.spawn(result.visit, SuiteResults(soup, suite_tbody, log_name))
else:
result.visit(SuiteResults(soup, suite_tbody, log_name))
test_icon_txt = """
<div class="row">
<div class="col-md-12" style="height:25px;width:auto;">
</div>
</div>
"""
suite_div.append(BeautifulSoup(test_icon_txt, 'html.parser'))
### ============================ END OF SUITE METRICS ============================================ ####
logging.info("3 of 4: Capturing test metrics...")
### ============================ START OF TEST METRICS ======================================= ####
# Tests div
tm_div = soup.new_tag('div')
tm_div["id"] = "testMetrics"
tm_div["class"] = "tabcontent"
page_content_div.insert(100, tm_div)
test_icon_txt = """
<h4><b><i class="fa fa-table"></i> Test Metrics</b></h4>
<hr></hr>
"""
tm_div.append(BeautifulSoup(test_icon_txt, 'html.parser'))
# Create table tag
table = soup.new_tag('table')
table["id"] = "tm"
table["class"] = "table table-striped table-bordered"
tm_div.insert(10, table)
thead = soup.new_tag('thead')
table.insert(0, thead)
tr = soup.new_tag('tr')
thead.insert(0, tr)
th = soup.new_tag('th')
th.string = "Suite Name"
tr.insert(0, th)
th = soup.new_tag('th')
th.string = "Test Case"
tr.insert(1, th)
th = soup.new_tag('th')
th.string = "Status"
tr.insert(2, th)
th = soup.new_tag('th')
th.string = "Time (s)"
tr.insert(3, th)
th = soup.new_tag('th')
th.string = "Error Message"
tr.insert(4, th)
test_tbody = soup.new_tag('tbody')
table.insert(11, test_tbody)
# GET TEST METRICS
if group:
group.spawn(result.visit, TestResults(soup, test_tbody, log_name))
else:
result.visit(TestResults(soup, test_tbody, log_name))
test_icon_txt = """
<div class="row">
<div class="col-md-12" style="height:25px;width:auto;">
</div>
</div>
"""
tm_div.append(BeautifulSoup(test_icon_txt, 'html.parser'))
### ============================ END OF TEST METRICS ============================================ ####
logging.info("4 of 4: Capturing keyword metrics...")
### ============================ START OF KEYWORD METRICS ======================================= ####
# Keywords div
km_div = soup.new_tag('div')
km_div["id"] = "keywordMetrics"
km_div["class"] = "tabcontent"
page_content_div.insert(150, km_div)
keyword_icon_txt = """
<h4><b><i class="fa fa-table"></i> Keyword Metrics</b></h4>
<hr></hr>
"""
km_div.append(BeautifulSoup(keyword_icon_txt, 'html.parser'))
# Create table tag
# <table id="myTable">
table = soup.new_tag('table')
table["id"] = "km"
table["class"] = "table table-striped table-bordered"
km_div.insert(10, table)
thead = soup.new_tag('thead')
table.insert(0, thead)
tr = soup.new_tag('tr')
thead.insert(0, tr)
th = soup.new_tag('th')
th.string = "Test Case"
tr.insert(1, th)
th = soup.new_tag('th')
th.string = "Keyword"
tr.insert(1, th)
th = soup.new_tag('th')
th.string = "Status"
tr.insert(2, th)
th = soup.new_tag('th')
th.string = "Time (s)"
tr.insert(3, th)
kw_tbody = soup.new_tag('tbody')
table.insert(1, kw_tbody)
if group:
group.spawn(
result.visit,
KeywordResults(soup, kw_tbody, ignore_library, ignore_type))
group.join()
else:
result.visit(
KeywordResults(soup, kw_tbody, ignore_library, ignore_type))
test_icon_txt = """
<div class="row">
<div class="col-md-12" style="height:25px;width:auto;">
</div>
</div>
"""
km_div.append(BeautifulSoup(test_icon_txt, 'html.parser'))
# END OF KEYWORD METRICS
# START OF LOGS
# Logs div
log_div = soup.new_tag('div')
log_div["id"] = "log"
log_div["class"] = "tabcontent"
page_content_div.insert(200, log_div)
test_icon_txt = """
<p style="text-align:right">** <b>Report.html</b> and <b>Log.html</b> need to be in current folder in
order to display here</p>
<div class="embed-responsive embed-responsive-4by3">
<iframe class="embed-responsive-item" src=%s></iframe>
</div>
""" % log_name
log_div.append(BeautifulSoup(test_icon_txt, 'html.parser'))
# END OF LOGS
# EMAIL STATISTICS
# Statistics div
statisitcs_div = soup.new_tag('div')
statisitcs_div["id"] = "statistics"
statisitcs_div["class"] = "tabcontent"
page_content_div.insert(300, statisitcs_div)
emailStatistics = """
<h4><b><i class="fa fa-envelope-o"></i> Email Statistics</b></h4>
<hr></hr>
<button id="create" class="btn btn-primary active inner" role="button" onclick="this.style.visibility= 'hidden';"><i class="fa fa-cogs"></i> Generate Statistics Email</button>
<a download="message.eml" class="btn btn-primary active inner" role="button" id="downloadlink" style="display: none; width: 300px;"><i class="fa fa-download"></i> Click Here To Download Email</a>
<textarea id="textbox" class="col-md-12" style="height: 400px; padding:1em;">
To: [email protected]
Subject: Automation Execution Status
X-Unsent: 1
Content-Type: text/html
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<title>Test Email Sample</title>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<meta name="viewport" content="width=device-width, initial-scale=1.0 " />
<style>
body {
background-color:#F2F2F2;
}
body, html, table {
font-family: Courier New, Arial, sans-serif;
font-size: 1em;
}
.pastdue { color: crimson; }
table {
padding: 5px;
margin-left: 30px;
width: 800px;
}
thead {
text-align: center;
font-size: 1.1em;
background-color: #B0C4DE;
font-weight: bold;
color: #2D2C2C;
}
tbody {
text-align: center;
}
th {
width: 25%;
word-wrap:break-word;
}
</style>
</head>
<body>
<p>Hi Team,</p>
<p>Following are the last build execution status</p>
<p></p>
<table>
<tbody>
<tr>
<td style="text-align:left; padding-left:5px;color:#0b6690;">
<h2>Test Automation Report</h2>
</td>
<td style="text-align:right; padding-right:10px;color:#0b6690;">
<h3>Duration: elapsed_time</h3>
</td>
</tr>
</tbody>
</table>
<table>
<tr>
<td></td>
</tr>
</table>
<table>
<tbody>
<tr>
<td style="background-color:#616161; color:white; width:25%">
<table style="width:100%;">
<tbody>
<tr>
<td style="text-align:center; color:white;font-size: 30px;">Suite</td>
</tr>
<tr>
<td style="text-align:center; color:white;font-size: 12px;">Statistics</td>
</tr>
</tbody>
</table>
</td>
<td style="background-color:#009688; color:white; width:25%">
<table style="width:100%;">
<tbody>
<tr>
<td style="text-align:center; color:white;font-size: 30px;">suite_total</td>
</tr>
<tr>
<td style="text-align:center; color:white;font-size: 12px;">Total</td>
</tr>
</tbody>
</table>
</td>
<td style="background-color:#4CAF50; color:white; width:25%">
<table style="width:100%;">
<tbody>
<tr>
<td style="text-align:center; color:white;font-size: 30px;">suite_pass</td>
</tr>
<tr>
<td style="text-align:center; color:white;font-size: 12px;">Pass</td>
</tr>
</tbody>
</table>
</td>
<td style="background-color:#f44336; color:white; width:25%">
<table style="width:100%;">
<tbody>
<tr>
<td style="text-align:center; color:white;font-size: 30px;">suite_fail</td>
</tr>
<tr>
<td style="text-align:center; color:white;font-size: 12px;">Fail</td>
</tr>
</tbody>
</table>
</td>
</tr>
<tr>
<td style="background-color:#616161; color:white; width:25%">
<table style="width:100%;">
<tbody>
<tr>
<td style="text-align:center; color:white;font-size: 30px;">Test</td>
</tr>
<tr>
<td style="text-align:center; color:white;font-size: 12px;">Statistics</td>
</tr>
</tbody>
</table>
</td>
<td style="background-color:#009688; color:white; width:25%">
<table style="width:100%;">
<tbody>
<tr>
<td style="text-align:center; color:white;font-size: 30px;">test_total</td>
</tr>
<tr>
<td style="text-align:center; color:white;font-size: 12px;">Total</td>
</tr>
</tbody>
</table>
</td>
<td style="background-color:#4CAF50; color:white; width:25%">
<table style="width:100%;">
<tbody>
<tr>
<td style="text-align:center; color:white;font-size: 30px;">test_pass</td>
</tr>
<tr>
<td style="text-align:center; color:white;font-size: 12px;">Pass</td>
</tr>
</tbody>
</table>
</td>
<td style="background-color:#f44336; color:white; width:25%">
<table style="width:100%;">
<tbody>
<tr>
<td style="text-align:center; color:white;font-size: 30px;">test_fail</td>
</tr>
<tr>
<td style="text-align:center; color:white;font-size: 12px;">Fail</td>
</tr>
</tbody>
</table>
</td>
</tr>
<tr>
<td style="background-color:#616161; color:white; width:25%">
<table style="width:100%;">
<tbody>
<tr>
<td style="text-align:center; color:white;font-size: 30px;">Keyword</td>
</tr>
<tr>
<td style="text-align:center; color:white;font-size: 12px;">Statistics</td>
</tr>
</tbody>
</table>
</td>
<td style="background-color:#009688; color:white; width:25%">
<table style="width:100%;">
<tbody>
<tr>
<td style="text-align:center; color:white;font-size: 30px;">keyword_total</td>
</tr>
<tr>
<td style="text-align:center; color:white;font-size: 12px;">Total</td>
</tr>
</tbody>
</table>
</td>
<td style="background-color:#4CAF50; color:white; width:25%">
<table style="width:100%;">
<tbody>
<tr>
<td style="text-align:center; color:white;font-size: 30px;">keyword_pass</td>
</tr>
<tr>
<td style="text-align:center; color:white;font-size: 12px;">Pass</td>
</tr>
</tbody>
</table>
</td>
<td style="background-color:#f44336; color:white; width:25%">
<table style="width:100%;">
<tbody>
<tr>
<td style="text-align:center; color:white;font-size: 30px;">keyword_fail</td>
</tr>
<tr>
<td style="text-align:center; color:white;font-size: 12px;">Fail</td>
</tr>
</tbody>
</table>
</td>
</tr>
</tbody>
</table>
<table>
<tr>
<td></td>
</tr>
</table>
<table>
<tbody>
<tr>
<td style="width:33%;color:#0b6690;"><h3>Suite Status</h3></td>
<td style="width:33%;color:#0b6690;"><h3>Test Status</h3></td>
<td style="width:33%;color:#0b6690;"><h3>Keyword Status</h3></td>
</tr>
<tr>
<td>
<img src='https://chart.googleapis.com/chart?cht=p3&chd=t:suite-pass-perc,suite-fail-perc&chs=250x200&chco=3BB032|bc2d29&chdl=suite-pass-perc-pass|suite-fail-perc-fail'/>
</td>
<td>
<img src='https://chart.googleapis.com/chart?cht=p3&chd=t:test-pass-perc,test-fail-perc&chs=250x200&chco=3BB032|bc2d29&chdl=test-pass-perc-pass|test-fail-perc-fail'/>
</td>
<td>
<img src='https://chart.googleapis.com/chart?cht=p3&chd=t:keyword-pass-perc,keyword-fail-perc&chs=250x200&chco=3BB032|bc2d29&chdl=keyword-pass-perc-pass|keyword-fail-perc-fail'/>
</td>
</tr>
</tbody>
</table>
<p>Please refer RF Metrics report for detailed statistics.<p>
<strong>Team QA</strong>
</body></html></textarea>
"""
emailStatistics = emailStatistics.replace("suite_total", str(total_suite))
emailStatistics = emailStatistics.replace("suite_pass", str(passed_suite))
emailStatistics = emailStatistics.replace("suite_fail", str(failed_suite))
emailStatistics = emailStatistics.replace("test_total", str(total))
emailStatistics = emailStatistics.replace("test_pass", str(passed))
emailStatistics = emailStatistics.replace("test_fail", str(failed))
emailStatistics = emailStatistics.replace("keyword_total",
str(total_keywords))
emailStatistics = emailStatistics.replace("keyword_pass",
str(passed_keywords))
emailStatistics = emailStatistics.replace("keyword_fail",
str(failed_keywords))
emailStatistics = emailStatistics.replace("elapsed_time", str(elapsedtime))
emailStatistics = emailStatistics.replace("suite-pass-perc", str(suitepp))
emailStatistics = emailStatistics.replace("suite-fail-perc", str(suitefp))
emailStatistics = emailStatistics.replace("test-pass-perc", str(testpp))
emailStatistics = emailStatistics.replace("test-fail-perc", str(testfp))
emailStatistics = emailStatistics.replace("keyword-pass-perc", str(kwpp))
emailStatistics = emailStatistics.replace("keyword-fail-perc", str(kwfp))
statisitcs_div.append(BeautifulSoup(emailStatistics, 'html.parser'))
# END OF EMAIL STATISTICS
script_text = """
<script>
(function () {
var textFile = null,
makeTextFile = function (text) {
var data = new Blob([text], {type: 'text/plain'});
if (textFile !== null) {
window.URL.revokeObjectURL(textFile);
}
textFile = window.URL.createObjectURL(data);
return textFile;
};
var create = document.getElementById('create'),
textbox = document.getElementById('textbox');
create.addEventListener('click', function () {
var link = document.getElementById('downloadlink');
link.href = makeTextFile(textbox.value);
link.style.display = 'block';
}, false);
})();
</script>
<script>
function createPieChart(passed_count,failed_count,ChartID,ChartName){
var status = [];
status.push(['Status', 'Percentage']);
status.push(['PASS',parseInt(passed_count)],['FAIL',parseInt(failed_count)]);
var data = google.visualization.arrayToDataTable(status);
var options = {
pieHole: 0.6,
legend: 'none',
chartArea: {width: "95%",height: "90%"},
colors: ['green', 'red'],
};
var chart = new google.visualization.PieChart(document.getElementById(ChartID));
chart.draw(data, options);
}
</script>
<script>
function createBarGraph(tableID,keyword_column,time_column,limit,ChartID,Label,type){
var status = [];
css_selector_locator = tableID + ' tbody >tr'
var rows = $(css_selector_locator);
var columns;
var myColors = [
'#4F81BC',
'#C0504E',
'#9BBB58',
'#24BEAA',
'#8064A1',
'#4AACC5',
'#F79647',
'#815E86',
'#76A032',
'#34558B'
];
status.push([type, Label,{ role: 'annotation'}, {role: 'style'}]);
for (var i = 0; i < rows.length; i++) {
if (i == Number(limit)){
break;
}
//status = [];
name_value = $(rows[i]).find('td');
time=($(name_value[Number(time_column)]).html()).trim();
keyword=($(name_value[Number(keyword_column)]).html()).trim();
status.push([keyword,parseFloat(time),parseFloat(time),myColors[i]]);
}
var data = google.visualization.arrayToDataTable(status);
var options = {
legend: 'none',
chartArea: {width: "92%",height: "75%"},
bar: {
groupWidth: '90%'
},
annotations: {
alwaysOutside: true,
textStyle: {
fontName: 'Comic Sans MS',
fontSize: 13,
bold: true,
italic: true,
color: "black", // The color of the text.
},
},
hAxis: {
textStyle: {
fontName: 'Arial',
fontSize: 10,
}
},
vAxis: {
gridlines: { count: 10 },
textStyle: {
fontName: 'Comic Sans MS',
fontSize: 10,
}
},
};
// Instantiate and draw the chart.
var chart = new google.visualization.ColumnChart(document.getElementById(ChartID));
chart.draw(data, options);
}
</script>
<script>
function executeDataTable(tabname,sortCol) {
var fileTitle;
switch(tabname) {
case "#sm":
fileTitle = "SuiteMetrics";
break;
case "#tm":
fileTitle = "TestMetrics";
break;
case "#km":
fileTitle = "KeywordMetrics";
break;
default:
fileTitle = "metrics";
}
$(tabname).DataTable(
{
retrieve: true,
"order": [[ Number(sortCol), "desc" ]],
dom: 'l<".margin" B>frtip',
buttons: [
'copy',
{
extend: 'csv',
filename: function() {
return fileTitle + '-' + new Date().toLocaleString();
},
title : '',
},
{
extend: 'excel',
filename: function() {
return fileTitle + '-' + new Date().toLocaleString();
},
title : '',
},
{
extend: 'pdf',
filename: function() {
return fileTitle + '-' + new Date().toLocaleString();
},
title : '',
},
{
extend: 'print',
title : '',
},
],
}
);
}
</script>
<script>
function openPage(pageName,elmnt,color) {
var i, tabcontent, tablinks;
tabcontent = document.getElementsByClassName("tabcontent");
for (i = 0; i < tabcontent.length; i++) {
tabcontent[i].style.display = "none";
}
tablinks = document.getElementsByClassName("tablink");
for (i = 0; i < tablinks.length; i++) {
tablinks[i].style.backgroundColor = "";
}
document.getElementById(pageName).style.display = "block";
elmnt.style.backgroundColor = color;
}
// Get the element with id="defaultOpen" and click on it
document.getElementById("defaultOpen").click();
</script>
<script>
// Get the element with id="defaultOpen" and click on it
document.getElementById("defaultOpen").click();
</script>
<script>
$(window).on('load',function(){$('.loader').fadeOut();});
</script>
"""
body.append(BeautifulSoup(script_text, 'html.parser'))
# WRITE TO RF_METRICS_REPORT.HTML
# Write output as html file
with open(result_file, 'w') as outfile:
outfile.write(soup.prettify())
logging.info(
"Results file created successfully and can be found at {}".format(
result_file))
def start(self, user_count: int, spawn_rate: float, **kwargs) -> None:
num_workers = len(self.clients.ready) + len(self.clients.running) + len(self.clients.spawning)
if not num_workers:
logger.warning(
"You are running in distributed mode but have no worker servers connected. "
"Please connect workers prior to swarming."
)
return
for user_class in self.user_classes:
if self.environment.host is not None:
user_class.host = self.environment.host
self.target_user_classes_count = weight_users(self.user_classes, user_count)
self.spawn_rate = spawn_rate
logger.info(
"Sending spawn jobs of %d users at %.2f spawn rate to %d ready clients"
% (user_count, spawn_rate, num_workers)
)
worker_spawn_rate = float(spawn_rate) / (num_workers or 1)
if worker_spawn_rate > 100:
logger.warning(
"Your selected spawn rate is very high (>100/worker), and this is known to sometimes cause issues. Do you really need to ramp up that fast?"
)
# Since https://github.com/locustio/locust/pull/1621, the master is responsible for dispatching and controlling
# the total spawn rate which is more CPU intensive for the master. The number 200 is a little arbitrary as the computational
# load on the master greatly depends on the number of workers and the number of user classes. For instance,
# 5 user classes and 5 workers can easily do 200/s. However, 200/s with 50 workers and 20 user classes will likely make the
# dispatch very slow because of the required computations. I (@mboutet) doubt that many Locust's users are
# spawning that rapidly. If so, then they'll likely open issues on GitHub in which case I'll (@mboutet) take a look.
if spawn_rate > 200:
logger.warning(
"Your selected total spawn rate is quite high (>200), and this is known to sometimes cause performance issues on the master. "
"Do you really need to ramp up that fast? If so and if encountering performance issues on the master, free to open an issue."
)
if self.state != STATE_RUNNING and self.state != STATE_SPAWNING:
self.stats.clear_all()
self.exceptions = {}
self.environment.events.test_start.fire(environment=self.environment)
if self.environment.shape_class:
self.environment.shape_class.reset_time()
self.update_state(STATE_SPAWNING)
try:
for dispatched_users in UsersDispatcher(
worker_nodes=self.clients.ready + self.clients.running + self.clients.spawning,
user_classes_count=self.target_user_classes_count,
spawn_rate=spawn_rate,
):
dispatch_greenlets = Group()
for worker_node_id, worker_user_classes_count in dispatched_users.items():
data = {
"timestamp": time.time(),
"user_classes_count": worker_user_classes_count,
"host": self.environment.host,
"stop_timeout": self.environment.stop_timeout,
}
dispatch_greenlets.add(
gevent.spawn_later(
0,
self.server.send_to_client,
Message("spawn", data, worker_node_id),
)
)
dispatched_user_count = sum(map(sum, map(methodcaller("values"), dispatched_users.values())))
logger.debug(
"Sending spawn messages for %g total users to %i client(s)",
dispatched_user_count,
len(dispatch_greenlets),
)
dispatch_greenlets.join()
logger.debug(
"Currently spawned users: %s" % _format_user_classes_count_for_log(self.reported_user_classes_count)
)
except KeyboardInterrupt:
# We need to catch keyboard interrupt. Otherwise, if KeyboardInterrupt is received while in
# a gevent.sleep inside the dispatch_users function, locust won't gracefully shutdown.
self.quit()
# Wait a little for workers to report their users to the master
# so that we can give an accurate log message below and fire the `spawning_complete` event
# when the user count is really at the desired value.
timeout = gevent.Timeout(self._wait_for_workers_report_after_ramp_up())
timeout.start()
try:
while self.user_count != self.target_user_count:
gevent.sleep()
except gevent.Timeout:
pass
finally:
timeout.cancel()
self.environment.events.spawning_complete.fire(user_count=sum(self.target_user_classes_count.values()))
logger.info("All users spawned: %s" % _format_user_classes_count_for_log(self.reported_user_classes_count))
def ping_api(requests_impl, url: str, concurrent_requests: int):
group = Group()
for _ in range(concurrent_requests):
group.spawn(requests_impl.get, url)
group.join()
class ImapSyncMonitor(BaseMailSyncMonitor):
""" Top-level controller for an account's mail sync. Spawns individual
FolderSync greenlets for each folder.
Parameters
----------
poll_frequency: Integer
Seconds to wait between polling for the greenlets spawned
heartbeat: Integer
Seconds to wait between checking on folder sync threads.
refresh_flags_max: Integer
the maximum number of UIDs for which we'll check flags
periodically.
"""
def __init__(self,
account,
heartbeat=1,
poll_frequency=30,
retry_fail_classes=[],
refresh_flags_max=2000):
self.poll_frequency = poll_frequency
self.syncmanager_lock = db_write_lock(account.namespace.id)
self.refresh_flags_max = refresh_flags_max
provider_supports_condstore = provider_info(account.provider,
account.email_address).get(
'condstore', False)
account_supports_condstore = getattr(account, 'supports_condstore',
False)
if provider_supports_condstore or account_supports_condstore:
self.sync_engine_class = CondstoreFolderSyncEngine
else:
self.sync_engine_class = FolderSyncEngine
self.folder_monitors = Group()
BaseMailSyncMonitor.__init__(self, account, heartbeat,
retry_fail_classes)
def prepare_sync(self):
"""Ensures that canonical tags are created for the account, and gets
and save Folder objects for folders on the IMAP backend. Returns a list
of tuples (folder_name, folder_id) for each folder we want to sync (in
order)."""
with mailsync_session_scope() as db_session:
account = db_session.query(ImapAccount).get(self.account_id)
Tag.create_canonical_tags(account.namespace, db_session)
with _pool(self.account_id).get() as crispin_client:
sync_folders = crispin_client.sync_folders()
save_folder_names(log, self.account_id,
crispin_client.folder_names(), db_session)
sync_folder_names_ids = []
for folder_name in sync_folders:
try:
id_, = db_session.query(Folder.id). \
filter(Folder.name == folder_name,
Folder.account_id == self.account_id).one()
sync_folder_names_ids.append((folder_name, id_))
except NoResultFound:
log.error("Missing Folder object when starting sync",
folder_name=folder_name)
raise MailsyncError(
"Missing Folder '{}' on account {}".format(
folder_name, self.account_id))
return sync_folder_names_ids
def sync(self):
""" Start per-folder syncs. Only have one per-folder sync in the
'initial' state at a time.
"""
sync_folder_names_ids = self.prepare_sync()
for folder_name, folder_id in sync_folder_names_ids:
log.info('initializing folder sync')
thread = self.sync_engine_class(
self.account_id, folder_name, folder_id, self.email_address,
self.provider_name, self.poll_frequency, self.syncmanager_lock,
self.refresh_flags_max, self.retry_fail_classes)
thread.start()
self.folder_monitors.add(thread)
while not self._thread_polling(thread) and \
not self._thread_finished(thread) and \
not thread.ready():
sleep(self.heartbeat)
# Allow individual folder sync monitors to shut themselves down
# after completing the initial sync.
if self._thread_finished(thread) or thread.ready():
log.info('folder sync finished/killed',
folder_name=thread.folder_name)
# NOTE: Greenlet is automatically removed from the group.
self.folder_monitors.join()
def all(self):
group = Group()
group.spawn(self.api_calls)
group.spawn(self.visit)
group.join()
# Last modified : 2014-09-11 08:39:47
# Filename : t.py
# Description :
import gevent
from gevent.queue import Queue
from gevent.pool import Group
q = Queue(maxsize = 3)
group = Group()
def worker(n):
print gevent.getcurrent()
while not q.empty():
task = q.get()
print "Worker %s task %d" % (n, task)
gevent.sleep(0)
def boss():
for i in xrange(1,25):
q.put(i)
g4 = gevent.spawn(boss)
g1 = gevent.spawn(worker,"lujinda")
g2 = gevent.spawn(worker,"lilanlan")
g3 = gevent.spawn(worker,"lll")
map(group.add,[g4,g2,g3,g1])
group.join()
class LocustRunner(object):
def __init__(self, locust_classes, options):
self.options = options
self.locust_classes = locust_classes
self.hatch_rate = options.hatch_rate
self.host = options.host
self.locusts = Group()
self.greenlet = Group()
self.state = STATE_INIT
self.hatching_greenlet = None
self.stepload_greenlet = None
self.current_cpu_usage = 0
self.cpu_warning_emitted = False
self.greenlet.spawn(self.monitor_cpu)
self.exceptions = {}
self.stats = global_stats
self.step_load = options.step_load
# register listener that resets stats when hatching is complete
def on_hatch_complete(user_count):
self.state = STATE_RUNNING
if self.options.reset_stats:
logger.info("Resetting stats\n")
self.stats.reset_all()
events.hatch_complete += on_hatch_complete
def __del__(self):
# don't leave any stray greenlets if runner is removed
if len(self.greenlet) > 0:
self.greenlet.kill(block=False)
@property
def request_stats(self):
return self.stats.entries
@property
def errors(self):
return self.stats.errors
@property
def user_count(self):
return len(self.locusts)
def cpu_log_warning(self):
"""Called at the end of the test to repeat the warning & return the status"""
if self.cpu_warning_emitted:
logger.warning(
"Loadgen CPU usage was too high at some point during the test! See https://docs.locust.io/en/stable/running-locust-distributed.html for how to distribute the load over multiple CPU cores or machines"
)
return True
return False
def weight_locusts(self, amount):
"""
Distributes the amount of locusts for each WebLocust-class according to it's weight
returns a list "bucket" with the weighted locusts
"""
bucket = []
weight_sum = sum((locust.weight for locust in self.locust_classes
if locust.task_set))
residuals = {}
for locust in self.locust_classes:
if not locust.task_set:
warnings.warn(
"Notice: Found Locust class (%s) got no task_set. Skipping..."
% locust.__name__)
continue
if self.host is not None:
locust.host = self.host
# create locusts depending on weight
percent = locust.weight / float(weight_sum)
num_locusts = int(round(amount * percent))
bucket.extend([locust for x in range(num_locusts)])
# used to keep track of the amount of rounding was done if we need
# to add/remove some instances from bucket
residuals[locust] = amount * percent - round(amount * percent)
if len(bucket) < amount:
# We got too few locust classes in the bucket, so we need to create a few extra locusts,
# and we do this by iterating over each of the Locust classes - starting with the one
# where the residual from the rounding was the largest - and creating one of each until
# we get the correct amount
for locust in [
l for l, r in sorted(
residuals.items(), key=lambda x: x[1], reverse=True)
][:amount - len(bucket)]:
bucket.append(locust)
elif len(bucket) > amount:
# We've got too many locusts due to rounding errors so we need to remove some
for locust in [
l for l, r in sorted(residuals.items(), key=lambda x: x[1])
][:len(bucket) - amount]:
bucket.remove(locust)
return bucket
def spawn_locusts(self, spawn_count, wait=False, hatch_rate=None):
if hatch_rate is None:
hatch_rate = self.hatch_rate
bucket = self.weight_locusts(spawn_count)
spawn_count = len(bucket)
if self.state == STATE_INIT or self.state == STATE_STOPPED:
self.state = STATE_HATCHING
existing_count = len(self.locusts)
logger.info(
"Hatching and swarming %i users at the rate %g users/s (%i users already running)..."
% (spawn_count, self.hatch_rate, existing_count))
occurrence_count = dict([(l.__name__, 0) for l in self.locust_classes])
def hatch():
sleep_time = 1.0 / self.hatch_rate
while True:
if not bucket:
logger.info(
"All locusts hatched: %s (%i already running)" % (
", ".join([
"%s: %d" % (name, count)
for name, count in occurrence_count.items()
]),
existing_count,
))
events.hatch_complete.fire(user_count=len(self.locusts))
return
locust = bucket.pop(random.randint(0, len(bucket) - 1))
occurrence_count[locust.__name__] += 1
new_locust = locust()
def start_locust(_):
try:
new_locust.run(runner=self)
except GreenletExit:
pass
self.locusts.spawn(start_locust, new_locust)
if len(self.locusts) % 10 == 0:
logger.debug("%i locusts hatched" % len(self.locusts))
if bucket:
gevent.sleep(sleep_time)
hatch()
if wait:
self.locusts.join()
logger.info("All locusts dead\n")
def kill_locusts(self, kill_count):
"""
Kill a kill_count of weighted locusts from the Group() object in self.locusts
"""
bucket = self.weight_locusts(kill_count)
kill_count = len(bucket)
logger.info("Killing %i locusts" % kill_count)
dying = []
for g in self.locusts:
for l in bucket:
if l == type(g.args[0]):
dying.append(g)
bucket.remove(l)
break
self.kill_locust_greenlets(dying)
events.hatch_complete.fire(user_count=self.user_count)
def kill_locust_greenlets(self, greenlets):
"""
Kill running locust greenlets. If options.stop_timeout is set, we try to stop the
Locust users gracefully
"""
if self.options.stop_timeout:
dying = Group()
for g in greenlets:
locust = g.args[0]
if locust._state == LOCUST_STATE_WAITING:
self.locusts.killone(g)
else:
locust._state = LOCUST_STATE_STOPPING
dying.add(g)
if not dying.join(timeout=self.options.stop_timeout):
logger.info(
"Not all locusts finished their tasks & terminated in %s seconds. Killing them..."
% self.options.stop_timeout)
dying.kill(block=True)
else:
for g in greenlets:
self.locusts.killone(g)
def monitor_cpu(self):
process = psutil.Process()
while True:
self.current_cpu_usage = process.cpu_percent()
if self.current_cpu_usage > 90 and not self.cpu_warning_emitted:
logging.warning(
"Loadgen CPU usage above 90%! This may constrain your throughput and may even give inconsistent response time measurements! See https://docs.locust.io/en/stable/running-locust-distributed.html for how to distribute the load over multiple CPU cores or machines"
)
self.cpu_warning_emitted = True
gevent.sleep(CPU_MONITOR_INTERVAL)
def start_hatching(self,
locust_count=None,
hatch_rate=None,
wait=False,
**kwargs):
series = kwargs.pop('series', None)
if series is not None:
self.replay(series, wait=wait, **kwargs)
return
if locust_count is None or hatch_rate is None:
raise ValueError(
'locust_count and hatch_rate must be set when not in replay mode'
)
if self.state != STATE_RUNNING and self.state != STATE_HATCHING:
self.stats.clear_all()
self.exceptions = {}
self.cpu_warning_emitted = False
self.slave_cpu_warning_emitted = False
events.locust_start_hatching.fire()
# Dynamically changing the locust count
if self.state != STATE_INIT and self.state != STATE_STOPPED:
self.state = STATE_HATCHING
if self.user_count > locust_count:
# Kill some locusts
kill_count = self.user_count - locust_count
self.kill_locusts(kill_count)
elif self.user_count < locust_count:
# Spawn some locusts
self.hatch_rate = hatch_rate
spawn_count = locust_count - self.user_count
self.spawn_locusts(spawn_count=spawn_count)
else:
events.hatch_complete.fire(user_count=self.user_count)
else:
self.hatch_rate = hatch_rate
self.spawn_locusts(locust_count, wait=wait)
def replay(self, series, wait=False, lookahead=10., **kwargs):
import pandas as pd
from datetime import datetime
from scipy import interpolate
if not isinstance(series, pd.Series):
raise TypeError('series not understood')
if self.state != STATE_RUNNING and self.state != STATE_HATCHING:
self.stats.clear_all()
self.exceptions = {}
self.cpu_warning_emitted = False
self.slave_cpu_warning_emitted = False
events.locust_start_hatching.fire()
start_time = datetime.now()
def elapsed_seconds():
return (datetime.now() - start_time).total_seconds()
interp_func = interpolate.interp1d(series.index,
series,
fill_value='extrapolate')
# Dynamically changing the locust count
if self.state != STATE_INIT and self.state != STATE_STOPPED:
self.state = STATE_HATCHING
while True:
diff = elapsed_seconds()
logger.info('user_count: %d, desired: %g', self.user_count,
interp_func(diff))
pos = series.index.searchsorted(diff + lookahead)
if pos == len(series):
if series.index[-1] > diff:
gevent.sleep(series.index[-1] - diff)
events.hatch_complete.fire(user_count=self.user_count)
break
locust_count = round(series.iloc[pos])
if locust_count > self.user_count:
spawn_count = locust_count - self.user_count
hatch_rate = spawn_count / lookahead
self.spawn_locusts(spawn_count=spawn_count,
hatch_rate=hatch_rate)
elif locust_count < self.user_count:
self.kill_locusts(self.user_count - locust_count)
sleep_secs = series.index[pos] - elapsed_seconds()
logger.info('Shall sleep %g seconds', sleep_secs)
gevent.sleep(sleep_secs)
if wait:
self.locusts.join()
logger.info("All locusts dead\n")
def start_stepload(self, locust_count, hatch_rate, step_locust_count,
step_duration):
if locust_count < step_locust_count:
logger.error(
"Invalid parameters: total locust count of %d is smaller than step locust count of %d"
% (locust_count, step_locust_count))
return
self.total_clients = locust_count
self.hatch_rate = hatch_rate
self.step_clients_growth = step_locust_count
self.step_duration = step_duration
if self.stepload_greenlet:
logger.info(
"There is an ongoing swarming in Step Load mode, will stop it now."
)
self.stepload_greenlet.kill()
logger.info(
"Start a new swarming in Step Load mode: total locust count of %d, hatch rate of %d, step locust count of %d, step duration of %d "
% (locust_count, hatch_rate, step_locust_count, step_duration))
self.state = STATE_INIT
self.stepload_greenlet = self.greenlet.spawn(self.stepload_worker)
self.stepload_greenlet.link_exception(callback=self.noop)
def stepload_worker(self):
current_num_clients = 0
while self.state == STATE_INIT or self.state == STATE_HATCHING or self.state == STATE_RUNNING:
current_num_clients += self.step_clients_growth
if current_num_clients > int(self.total_clients):
logger.info('Step Load is finished.')
break
self.start_hatching(current_num_clients, self.hatch_rate)
logger.info('Step loading: start hatch job of %d locust.' %
(current_num_clients))
gevent.sleep(self.step_duration)
def stop(self):
# if we are currently hatching locusts we need to kill the hatching greenlet first
if self.hatching_greenlet and not self.hatching_greenlet.ready():
self.hatching_greenlet.kill(block=True)
self.kill_locust_greenlets([g for g in self.locusts])
self.state = STATE_STOPPED
events.locust_stop_hatching.fire()
def quit(self):
self.stop()
self.greenlet.kill(block=True)
def log_exception(self, node_id, msg, formatted_tb):
key = hash(formatted_tb)
row = self.exceptions.setdefault(key, {
"count": 0,
"msg": msg,
"traceback": formatted_tb,
"nodes": set()
})
row["count"] += 1
row["nodes"].add(node_id)
self.exceptions[key] = row
def noop(self, *args, **kwargs):
""" Used to link() greenlets to in order to be compatible with gevent 1.0 """
pass
class ImapSyncMonitor(BaseMailSyncMonitor):
""" Top-level controller for an account's mail sync. Spawns individual
FolderSync greenlets for each folder.
Parameters
----------
poll_frequency: Integer
Seconds to wait between polling for the greenlets spawned
heartbeat: Integer
Seconds to wait between checking on folder sync threads.
refresh_flags_max: Integer
the maximum number of UIDs for which we'll check flags
periodically.
"""
def __init__(self,
account,
heartbeat=1,
poll_frequency=300,
retry_fail_classes=[],
refresh_flags_max=2000):
self.poll_frequency = poll_frequency
self.syncmanager_lock = db_write_lock(account.namespace.id)
self.refresh_flags_max = refresh_flags_max
provider_supports_condstore = provider_info(account.provider).get(
'condstore', False)
account_supports_condstore = getattr(account, 'supports_condstore',
False)
if provider_supports_condstore or account_supports_condstore:
self.sync_engine_class = CondstoreFolderSyncEngine
else:
self.sync_engine_class = FolderSyncEngine
self.folder_monitors = Group()
BaseMailSyncMonitor.__init__(self, account, heartbeat,
retry_fail_classes)
def sync(self):
""" Start per-folder syncs. Only have one per-folder sync in the
'initial' state at a time.
"""
with mailsync_session_scope() as db_session:
with _pool(self.account_id).get() as crispin_client:
sync_folders = crispin_client.sync_folders()
account = db_session.query(ImapAccount)\
.get(self.account_id)
save_folder_names(log, account, crispin_client.folder_names(),
db_session)
Tag.create_canonical_tags(account.namespace, db_session)
folder_id_for = {
name: id_
for id_, name in db_session.query(Folder.id, Folder.name).
filter_by(account_id=self.account_id)
}
saved_states = {
name: state
for name, state in db_session.query(
Folder.name, ImapFolderSyncStatus.state).join(
ImapFolderSyncStatus.folder).filter(
ImapFolderSyncStatus.account_id == self.account_id)
}
for folder_name in sync_folders:
if folder_name not in folder_id_for:
log.error("Missing Folder object when starting sync",
folder_name=folder_name,
folder_id_for=folder_id_for)
raise MailsyncError("Missing Folder '{}' on account {}".format(
folder_name, self.account_id))
if saved_states.get(folder_name) != 'finish':
log.info('initializing folder sync')
# STOPSHIP(emfree): replace by appropriate base class.
thread = self.sync_engine_class(
self.account_id, folder_name, folder_id_for[folder_name],
self.email_address, self.provider_name,
self.poll_frequency, self.syncmanager_lock,
self.refresh_flags_max, self.retry_fail_classes)
thread.start()
self.folder_monitors.add(thread)
while not self._thread_polling(thread) and \
not self._thread_finished(thread) and \
not thread.ready():
sleep(self.heartbeat)
# Allow individual folder sync monitors to shut themselves down
# after completing the initial sync.
if self._thread_finished(thread) or thread.ready():
log.info('folder sync finished/killed',
folder_name=thread.folder_name)
# NOTE: Greenlet is automatically removed from the group.
self.folder_monitors.join()
class Crawler(object):
def __init__(self, seed, port='30000', **kwargs):
self.result = {}
self.probed = set()
self.task_lst = Queue()
self.pool = Group()
self.conf = kwargs
addr = seed.split(':') ### Invalid the seed if conf NOT set
if len(
addr
) == 1: ### if seed str not contain port, use conf['port'] or default
addr += [port]
elif len(addr) > 2:
### TODO: Usage
sys.stderr.write('Invalid seed %s\n' % seed)
exit(22)
#self.task_lst.put_nowait(dict(Host=':'.join(addr)))
sys.stderr.write('Crawler start from %s\n' % (':'.join(addr)))
def req(self,
ip,
port=30003,
apiId='1',
apiVer='3.0',
method='getchordringinfo',
params={},
timeout=20,
**kwargs):
r = ''
ret = {}
d = dict(id=apiId, jsonrpc=apiVer, method=method, params=params)
try:
r = requests.post('http://%s:%s' % (ip, port),
json=d,
headers={'Content-Type': 'application/json'},
timeout=timeout)
ret = json.loads(r.text)
except Exception as e:
sys.stderr.write(
'%s: met Error [%s] when request [%s] from %s:%s resp [%s]\n' %
(time.strftime('%F %T'), e.message, method, ip, port,
r.text if r else ''))
raise e
return ret
def parse(self, resp):
succ_lst = []
for vn in resp.get('result', {}).get('Vnodes', []):
### new discovery
succ_lst += [n for n in vn.pop('Successors', []) if n]
succ_lst += [n for n in vn.pop('Finger', []) if n]
succ_lst += [vn.pop('Predecessor') or {}]
return succ_lst
def info_from_task(self, task):
ip, port = task.get('Host', '').split(':')
return task.get('Id', ''), ip, int(port) + 3
def task_to_node(self, task):
return task
def worker(self, timeout=20):
while True:
try:
t = self.task_lst.get(timeout=timeout)
Id, ip, port = self.info_from_task(t)
if ip and port and Id not in self.probed: ### Is valid task and Not crawl yet
self.probed.add(
Id) ### mark it as crawled whatever success or fail
new_nodes = self.parse(self.req(ip, port, **self.conf))
self.result[Id] = self.task_to_node(
t) ### Add to crawl result
[self.task_lst.put_nowait(n)
for n in new_nodes] ### add new_nodes into task_lst
except Empty as e:
sys.stderr.write('%s: worker exit due to err %s\n' %
(time.strftime('%F %T'), type(e)))
break
except Exception as e:
sys.stderr.write('%s: worker req %s met err %s\n' %
(time.strftime('%F %T'), str(t), type(e)))
# print traceback.format_exc(e) ### stay for debug
continue
def debug(self, interval=5):
while True:
sys.stderr.write('Craw results %d\n' % len(self.result))
gevent.sleep(interval)
def run(self, timeout=20, thread=1, **kwargs):
gevent.spawn(self.debug, 5)
self.pool.map(self.worker, [timeout] * int(thread))
self.pool.join()
sys.stderr.write('Total: %d Nodes\n' % len(self.result))
class Runner:
"""
Orchestrates the load test by starting and stopping the users.
Use one of the :meth:`create_local_runner <locust.env.Environment.create_local_runner>`,
:meth:`create_master_runner <locust.env.Environment.create_master_runner>` or
:meth:`create_worker_runner <locust.env.Environment.create_worker_runner>` methods on
the :class:`Environment <locust.env.Environment>` instance to create a runner of the
desired type.
"""
def __init__(self, environment):
self.environment = environment
self.user_greenlets = Group()
self.greenlet = Group()
self.state = STATE_INIT
self.spawning_greenlet = None
self.shape_greenlet = None
self.shape_last_state = None
self.current_cpu_usage = 0
self.cpu_warning_emitted = False
self.worker_cpu_warning_emitted = False
self.greenlet.spawn(self.monitor_cpu).link_exception(greenlet_exception_handler)
self.exceptions = {}
self.target_user_classes_count: Dict[str, int] = {}
self.custom_messages = {}
# set up event listeners for recording requests
def on_request_success(request_type, name, response_time, response_length, **_kwargs):
self.stats.log_request(request_type, name, response_time, response_length)
def on_request_failure(request_type, name, response_time, response_length, exception, **_kwargs):
self.stats.log_request(request_type, name, response_time, response_length)
self.stats.log_error(request_type, name, exception)
# temporarily set log level to ignore warnings to suppress deprication message
loglevel = logging.getLogger().level
logging.getLogger().setLevel(logging.ERROR)
self.environment.events.request_success.add_listener(on_request_success)
self.environment.events.request_failure.add_listener(on_request_failure)
logging.getLogger().setLevel(loglevel)
self.connection_broken = False
# register listener that resets stats when spawning is complete
def on_spawning_complete(user_count):
self.update_state(STATE_RUNNING)
if environment.reset_stats:
logger.info("Resetting stats\n")
self.stats.reset_all()
self.environment.events.spawning_complete.add_listener(on_spawning_complete)
def __del__(self):
# don't leave any stray greenlets if runner is removed
if self.greenlet and len(self.greenlet) > 0:
self.greenlet.kill(block=False)
@property
def user_classes(self):
return self.environment.user_classes
@property
def user_classes_by_name(self):
return self.environment.user_classes_by_name
@property
def stats(self) -> RequestStats:
return self.environment.stats
@property
def errors(self):
return self.stats.errors
@property
def user_count(self):
"""
:returns: Number of currently running users
"""
return len(self.user_greenlets)
@property
def user_classes_count(self) -> Dict[str, int]:
"""
:returns: Number of currently running users for each user class
"""
user_classes_count = {user_class.__name__: 0 for user_class in self.user_classes}
for user_greenlet in self.user_greenlets:
try:
user = user_greenlet.args[0]
except IndexError:
# TODO: Find out why args is sometimes empty. In gevent code,
# the supplied args are cleared in the gevent.greenlet.Greenlet.__free,
# so it seems a good place to start investigating. My suspicion is that
# the supplied args are emptied whenever the greenlet is dead, so we can
# simply ignore the greenlets with empty args.
logger.debug(
"ERROR: While calculating number of running users, we encountered a user that didnt have proper args %s (user_greenlet.dead=%s)",
user_greenlet,
user_greenlet.dead,
)
continue
user_classes_count[user.__class__.__name__] += 1
return user_classes_count
def update_state(self, new_state):
"""
Updates the current state
"""
# I (cyberwiz) commented out this logging, because it is too noisy even for debug level
# Uncomment it if you are specifically debugging state transitions
# logger.debug("Updating state to '%s', old state was '%s'" % (new_state, self.state))
self.state = new_state
def cpu_log_warning(self):
"""Called at the end of the test to repeat the warning & return the status"""
if self.cpu_warning_emitted:
logger.warning(
"CPU usage was too high at some point during the test! See https://docs.locust.io/en/stable/running-locust-distributed.html for how to distribute the load over multiple CPU cores or machines"
)
return True
return False
def spawn_users(self, user_classes_spawn_count: Dict[str, int], wait: bool = False):
if self.state == STATE_INIT or self.state == STATE_STOPPED:
self.update_state(STATE_SPAWNING)
logger.debug(
"Spawning additional %s (%s already running)..."
% (json.dumps(user_classes_spawn_count), json.dumps(self.user_classes_count))
)
def spawn(user_class: str, spawn_count: int):
n = 0
while n < spawn_count:
new_user = self.user_classes_by_name[user_class](self.environment)
new_user.start(self.user_greenlets)
n += 1
if n % 10 == 0 or n == spawn_count:
logger.debug("%i users spawned" % self.user_count)
logger.debug("All users of class %s spawned" % user_class)
for user_class, spawn_count in user_classes_spawn_count.items():
spawn(user_class, spawn_count)
if wait:
self.user_greenlets.join()
logger.info("All users stopped\n")
def stop_users(self, user_classes_stop_count: Dict[str, int]):
async_calls_to_stop = Group()
stop_group = Group()
for user_class, stop_count in user_classes_stop_count.items():
if self.user_classes_count[user_class] == 0:
continue
to_stop = []
for user_greenlet in self.user_greenlets:
if len(to_stop) == stop_count:
break
try:
user = user_greenlet.args[0]
except IndexError:
logger.error(
"While stopping users, we encountered a user that didnt have proper args %s", user_greenlet
)
continue
if isinstance(user, self.user_classes_by_name[user_class]):
to_stop.append(user)
if not to_stop:
continue
while True:
user_to_stop: User = to_stop.pop()
logger.debug("Stopping %s" % user_to_stop.greenlet.name)
if user_to_stop.greenlet is greenlet.getcurrent():
# User called runner.quit(), so don't block waiting for killing to finish
user_to_stop.group.killone(user_to_stop.greenlet, block=False)
elif self.environment.stop_timeout:
async_calls_to_stop.add(gevent.spawn_later(0, user_to_stop.stop, force=False))
stop_group.add(user_to_stop.greenlet)
else:
async_calls_to_stop.add(gevent.spawn_later(0, user_to_stop.stop, force=True))
if not to_stop:
break
async_calls_to_stop.join()
if not stop_group.join(timeout=self.environment.stop_timeout):
logger.info(
"Not all users finished their tasks & terminated in %s seconds. Stopping them..."
% self.environment.stop_timeout
)
stop_group.kill(block=True)
logger.debug(
"%g users have been stopped, %g still running", sum(user_classes_stop_count.values()), self.user_count
)
def monitor_cpu(self):
process = psutil.Process()
while True:
self.current_cpu_usage = process.cpu_percent()
if self.current_cpu_usage > 90 and not self.cpu_warning_emitted:
logging.warning(
"CPU usage above 90%! This may constrain your throughput and may even give inconsistent response time measurements! See https://docs.locust.io/en/stable/running-locust-distributed.html for how to distribute the load over multiple CPU cores or machines"
)
self.cpu_warning_emitted = True
gevent.sleep(CPU_MONITOR_INTERVAL)
def start(self, user_count: int, spawn_rate: float, wait: bool = False):
"""
Start running a load test
:param user_count: Total number of users to start
:param spawn_rate: Number of users to spawn per second
:param wait: If True calls to this method will block until all users are spawned.
If False (the default), a greenlet that spawns the users will be
started and the call to this method will return immediately.
"""
if self.state != STATE_RUNNING and self.state != STATE_SPAWNING:
self.stats.clear_all()
self.exceptions = {}
self.cpu_warning_emitted = False
self.worker_cpu_warning_emitted = False
if wait and user_count - self.user_count > spawn_rate:
raise ValueError("wait is True but the amount of users to add is greater than the spawn rate")
for user_class in self.user_classes:
if self.environment.host is not None:
user_class.host = self.environment.host
self.target_user_classes_count = weight_users(self.user_classes, user_count)
local_worker_node = WorkerNode(id="local")
local_worker_node.user_classes_count = self.user_classes_count
if self.state != STATE_INIT and self.state != STATE_STOPPED:
self.update_state(STATE_SPAWNING)
logger.info("Ramping to %d users using a %.2f spawn rate" % (user_count, spawn_rate))
try:
for dispatched_users in UsersDispatcher(
worker_nodes=[local_worker_node],
user_classes_count=self.target_user_classes_count,
spawn_rate=spawn_rate,
):
user_classes_spawn_count = {}
user_classes_stop_count = {}
user_classes_count = dispatched_users[local_worker_node.id]
logger.debug("Ramping to %s" % _format_user_classes_count_for_log(user_classes_count))
for user_class, user_class_count in user_classes_count.items():
if self.user_classes_count[user_class] > user_class_count:
user_classes_stop_count[user_class] = self.user_classes_count[user_class] - user_class_count
elif self.user_classes_count[user_class] < user_class_count:
user_classes_spawn_count[user_class] = user_class_count - self.user_classes_count[user_class]
if wait:
# spawn_users will block, so we need to call stop_users first
self.stop_users(user_classes_stop_count)
self.spawn_users(user_classes_spawn_count, wait)
else:
# call spawn_users before stopping the users since stop_users
# can be blocking because of the stop_timeout
self.spawn_users(user_classes_spawn_count, wait)
self.stop_users(user_classes_stop_count)
except KeyboardInterrupt:
# We need to catch keyboard interrupt. Otherwise, if KeyboardInterrupt is received while in
# a gevent.sleep inside the dispatch_users function, locust won't gracefully shutdown.
self.quit()
logger.info("All users spawned: %s" % _format_user_classes_count_for_log(self.user_classes_count))
self.environment.events.spawning_complete.fire(user_count=sum(self.target_user_classes_count.values()))
def start_shape(self):
if self.shape_greenlet:
logger.info("There is an ongoing shape test running. Editing is disabled")
return
logger.info("Shape test starting. User count and spawn rate are ignored for this type of load test")
self.update_state(STATE_INIT)
self.shape_greenlet = self.greenlet.spawn(self.shape_worker)
self.shape_greenlet.link_exception(greenlet_exception_handler)
self.environment.shape_class.reset_time()
def shape_worker(self):
logger.info("Shape worker starting")
while self.state == STATE_INIT or self.state == STATE_SPAWNING or self.state == STATE_RUNNING:
new_state = self.environment.shape_class.tick()
if new_state is None:
logger.info("Shape test stopping")
if self.environment.parsed_options and self.environment.parsed_options.headless:
self.quit()
else:
self.stop()
self.shape_greenlet = None
self.shape_last_state = None
return
elif self.shape_last_state == new_state:
gevent.sleep(1)
else:
user_count, spawn_rate = new_state
logger.info("Shape test updating to %d users at %.2f spawn rate" % (user_count, spawn_rate))
# TODO: This `self.start()` call is blocking until the ramp-up is completed. This can leads
# to unexpected behaviours such as the one in the following example:
# A load test shape has the following stages:
# stage 1: (user_count=100, spawn_rate=1) for t < 50s
# stage 2: (user_count=120, spawn_rate=1) for t < 100s
# stage 3: (user_count=130, spawn_rate=1) for t < 120s
# Because the first stage will take 100s to complete, the second stage
# will be skipped completely because the shape worker will be blocked
# at the `self.start()` of the first stage.
# Of couse, this isn't a problem if the load test shape is well-defined.
# We should probably use a `gevent.timeout` with a duration a little over
# `(user_count - prev_user_count) / spawn_rate` in order to limit the runtime
# of each load test shape stage.
self.start(user_count=user_count, spawn_rate=spawn_rate)
self.shape_last_state = new_state
def stop(self):
"""
Stop a running load test by stopping all running users
"""
logger.debug("Stopping all users")
self.update_state(STATE_CLEANUP)
# if we are currently spawning users we need to kill the spawning greenlet first
if self.spawning_greenlet and not self.spawning_greenlet.ready():
self.spawning_greenlet.kill(block=True)
if self.environment.shape_class is not None and self.shape_greenlet is not greenlet.getcurrent():
# If the test was not started yet and locust is
# stopped/quit, shape_greenlet will be None.
if self.shape_greenlet is not None:
self.shape_greenlet.kill(block=True)
self.shape_greenlet = None
self.shape_last_state = None
self.stop_users(self.user_classes_count)
self.update_state(STATE_STOPPED)
self.cpu_log_warning()
def quit(self):
"""
Stop any running load test and kill all greenlets for the runner
"""
self.stop()
self.greenlet.kill(block=True)
def log_exception(self, node_id, msg, formatted_tb):
key = hash(formatted_tb)
row = self.exceptions.setdefault(key, {"count": 0, "msg": msg, "traceback": formatted_tb, "nodes": set()})
row["count"] += 1
row["nodes"].add(node_id)
self.exceptions[key] = row
@property
def target_user_count(self) -> int:
return sum(self.target_user_classes_count.values())
def register_message(self, msg_type, listener):
"""
Register a listener for a custom message from another node
:param msg_type: The type of the message to listen for
:param listener: The function to execute when the message is received
"""
self.custom_messages[msg_type] = listener
class DelugeClient(object):
def __init__(self, disconnect=None, restore_events=False):
"""A deluge client session.
:param disconnect: func, a disconnect callback, called when connection is dropped
:param restore_events: bool, keep event handlers on re-connect
"""
self.transfer = DelugeTransfer()
self.modules = []
self.restore_events = restore_events
self._event_handlers = defaultdict(list)
self._disconnect_handler = disconnect
self._greenlets = Group()
self._responses = {}
self._request_counter = 0
def _get_local_auth(self):
xdg_config = os.path.expanduser(
os.environ.get("XDG_CONFIG_HOME", "~/.config"))
config_home = os.path.join(xdg_config, "deluge")
auth_file = os.path.join(config_home, "auth")
username = password = ""
with open(auth_file) as fd:
for line in fd:
if line.startswith("#"):
continue
auth = line.split(":")
if len(auth) >= 2 and auth[0] == "localclient":
username, password = auth[0], auth[1]
break
return username, password
def _transfer_message(self, message):
self.transfer.transfer_message((message.format(), ))
def _create_module_method(self, module, method):
fullname = "{0}.{1}".format(module, method)
def func(obj, *args, **kwargs):
return self._call(fullname, *args, **kwargs)
func.__name__ = method
return func
def _introspect(self):
self.modules = []
methods = self._call("daemon.get_method_list").get()
methodmap = defaultdict(dict)
splitter = lambda v: v.split(".")
for module, method in imap(splitter, methods):
methodmap[module][method] = self._create_module_method(
module, method)
for module, methods in methodmap.items():
clsname = "DelugeModule{0}".format(module.capitalize())
cls = type(clsname, (), methods)
setattr(self, module, cls())
self.modules.append(module)
def _restore_events(self):
self._call("daemon.set_event_interest",
self._event_handlers.keys()).get()
def _handle_message(self, message):
if not isinstance(message, tuple):
return
if len(message) < 3:
return
message_type = message[0]
if message_type == RPC_EVENT:
event = message[1]
values = message[2]
if event in self._event_handlers:
for handler in self._event_handlers[event]:
gevent.spawn(handler, *values)
elif message_type in (RPC_RESPONSE, RPC_ERROR):
request_id = message[1]
value = message[2]
if request_id in self._responses:
response = self._responses[request_id]
if message_type == RPC_RESPONSE:
response.set(value)
elif message_type == RPC_ERROR:
err = DelugeRPCError(*value)
response.set_exception(err)
del self._responses[request_id]
def _wait_for_messages(self):
for message in self.transfer.read_messages():
self._handle_message(message)
if self._disconnect_handler:
self._disconnect_handler(self)
def _call(self, method, *args, **kwargs):
req = DelugeRPCRequest(self._request_counter, method, *args, **kwargs)
self.transfer.send_request(req)
response = DelugeRPCResponse()
self._responses[self._request_counter] = response
self._request_counter += 1
return response
def connect(self, host="127.0.0.1", port=58846, username="", password=""):
"""Connects to a daemon process.
:param host: str, the hostname of the daemon
:param port: int, the port of the daemon
:param username: str, the username to login with
:param password: str, the password to login with
"""
if not self.restore_events:
self._event_handlers.clear()
# Connect transport
self.transfer.connect((host, port))
# Start a message listener
self._greenlets.spawn(self._wait_for_messages)
# Attempt to fetch local auth info if needed
if not username and host in ("127.0.0.1", "localhost"):
username, password = self._get_local_auth()
# Authenticate
self._call("daemon.login", username, password).get()
# Introspect available methods
self._introspect()
# Keep event handlers if specified
if self.restore_events:
self._restore_events()
@property
def connected(self):
return self.transfer.connected
def disconnect(self):
"""Disconnects from the daemon."""
self.transfer.disconnect()
def register_event(self, event, handler):
"""Registers a handler that will be called when an event is received from the daemon.
:params event: str, the event to handle
:params handler: func, the handler function, f(args)
"""
self._event_handlers[event].append(handler)
return self._call("daemon.set_event_interest", [event])
def run(self):
"""Blocks until all event loops are done."""
self._greenlets.join()
class LocustRunner(object):
def __init__(self, locust_classes, options):
self.options = options
self.locust_classes = locust_classes
self.hatch_rate = options.hatch_rate
self.num_clients = options.num_clients
self.num_requests = options.num_requests
self.host = options.host
self.locusts = Group()
self.state = STATE_INIT
self.hatching_greenlet = None
self.exceptions = {}
self.stats = global_stats
# register listener that resets stats when hatching is complete
def on_hatch_complete(user_count):
self.state = STATE_RUNNING
if not self.options.no_reset_stats:
logger.info("Resetting stats\n")
self.stats.reset_all()
events.hatch_complete += on_hatch_complete
@property
def request_stats(self):
return self.stats.entries
@property
def errors(self):
return self.stats.errors
@property
def user_count(self):
return len(self.locusts)
def weight_locusts(self, amount, stop_timeout = None):
"""
Distributes the amount of locusts for each WebLocust-class according to it's weight
returns a list "bucket" with the weighted locusts
"""
bucket = []
weight_sum = sum((locust.weight for locust in self.locust_classes if locust.task_set))
for locust in self.locust_classes:
if not locust.task_set:
warnings.warn("Notice: Found Locust class (%s) got no task_set. Skipping..." % locust.__name__)
continue
if self.host is not None:
locust.host = self.host
if stop_timeout is not None:
locust.stop_timeout = stop_timeout
# create locusts depending on weight
percent = locust.weight / float(weight_sum)
num_locusts = int(round(amount * percent))
bucket.extend([locust for x in xrange(0, num_locusts)])
return bucket
def spawn_locusts(self, spawn_count=None, stop_timeout=None, wait=False):
if spawn_count is None:
spawn_count = self.num_clients
if self.num_requests is not None:
self.stats.max_requests = self.num_requests
bucket = self.weight_locusts(spawn_count, stop_timeout)
spawn_count = len(bucket)
if self.state == STATE_INIT or self.state == STATE_STOPPED:
self.state = STATE_HATCHING
self.num_clients = spawn_count
else:
self.num_clients += spawn_count
logger.info("Hatching and swarming %i clients at the rate %g clients/s..." % (spawn_count, self.hatch_rate))
occurence_count = dict([(l.__name__, 0) for l in self.locust_classes])
def hatch():
sleep_time = 1.0 / self.hatch_rate
while True:
if not bucket:
logger.info("All locusts hatched: %s" % ", ".join(["%s: %d" % (name, count) for name, count in six.iteritems(occurence_count)]))
events.hatch_complete.fire(user_count=self.num_clients)
return
locust = bucket.pop(random.randint(0, len(bucket)-1))
occurence_count[locust.__name__] += 1
def start_locust(_):
try:
locust().run()
except GreenletExit:
pass
new_locust = self.locusts.spawn(start_locust, locust)
if len(self.locusts) % 10 == 0:
logger.debug("%i locusts hatched" % len(self.locusts))
gevent.sleep(sleep_time)
hatch()
if wait:
self.locusts.join()
logger.info("All locusts dead\n")
def kill_locusts(self, kill_count):
"""
Kill a kill_count of weighted locusts from the Group() object in self.locusts
"""
bucket = self.weight_locusts(kill_count)
kill_count = len(bucket)
self.num_clients -= kill_count
logger.info("Killing %i locusts" % kill_count)
dying = []
for g in self.locusts:
for l in bucket:
if l == g.args[0]:
dying.append(g)
bucket.remove(l)
break
for g in dying:
self.locusts.killone(g)
events.hatch_complete.fire(user_count=self.num_clients)
def start_hatching(self, locust_count=None, hatch_rate=None, wait=False):
if self.state != STATE_RUNNING and self.state != STATE_HATCHING:
self.stats.clear_all()
self.stats.start_time = time()
self.exceptions = {}
events.locust_start_hatching.fire()
# Dynamically changing the locust count
if self.state != STATE_INIT and self.state != STATE_STOPPED:
self.state = STATE_HATCHING
if self.num_clients > locust_count:
# Kill some locusts
kill_count = self.num_clients - locust_count
self.kill_locusts(kill_count)
elif self.num_clients < locust_count:
# Spawn some locusts
if hatch_rate:
self.hatch_rate = hatch_rate
spawn_count = locust_count - self.num_clients
self.spawn_locusts(spawn_count=spawn_count)
else:
events.hatch_complete.fire(user_count=self.num_clients)
else:
if hatch_rate:
self.hatch_rate = hatch_rate
if locust_count is not None:
self.spawn_locusts(locust_count, wait=wait)
else:
self.spawn_locusts(wait=wait)
def stop(self):
# if we are currently hatching locusts we need to kill the hatching greenlet first
if self.hatching_greenlet and not self.hatching_greenlet.ready():
self.hatching_greenlet.kill(block=True)
self.locusts.kill(block=True)
self.state = STATE_STOPPED
events.locust_stop_hatching.fire()
def switch(self, key):
global locust_tests
try:
self.locust_classes = locust_tests[key].values()
except KeyError:
logger.error("No task named %s, restart application to reload tasks", key)
def log_exception(self, node_id, msg, formatted_tb):
key = hash(formatted_tb)
row = self.exceptions.setdefault(key, {"count": 0, "msg": msg, "traceback": formatted_tb, "nodes": set()})
row["count"] += 1
row["nodes"].add(node_id)
self.exceptions[key] = row
@classmethod
def set_fan_level_natural(cls, fan, level):
return {'code': fan.set_natural_speed(int(level))}
class InMsg(list):
def __init__(self, data, to, **kwargs):
super(InMsg, self).__init__(**kwargs)
self.extend(data)
self.to = to
class OutMsg(dict):
def __init__(self, data, to, **kwargs):
super(OutMsg, self).__init__(**kwargs)
self.update(data)
self.to = to
if __name__ == '__main__':
server = StreamServer((args.host, args.port), socket_incoming_connection)
logger.debug('Starting server on %s %s' % (args.host, args.port))
services = Group()
services.spawn(server.serve_forever)
services.spawn(Fan_commands_handler, args.ip, args.token, receive)
services.spawn(socket_msg_sender, sockets, send)
services.join()
import geventcurl as pycurl
from cStringIO import StringIO
from gevent.pool import Group
urls = ['www.gevent.org', 'www.google.com', 'www.python.org']
def get(url):
c = pycurl.Curl()
c.setopt(c.URL, url)
output = StringIO()
c.setopt(c.WRITEFUNCTION, output.write)
c.perform()
result = output.getvalue()
print url, ` result[:50] `
c.close()
g = Group()
for url in urls:
g.spawn(get, url)
g.join()
def stop_users(self, user_count, stop_rate=None):
"""
Stop `user_count` weighted users at a rate of `stop_rate`
"""
if user_count == 0 or stop_rate == 0:
return
bucket = self.weight_users(user_count)
user_count = len(bucket)
to_stop = []
for user_greenlet in self.user_greenlets:
try:
user = user_greenlet.args[0]
except IndexError:
logger.error(
"While stopping users, we encountered a user that didnt have proper args %s",
user_greenlet)
continue
for user_class in bucket:
if isinstance(user, user_class):
to_stop.append(user)
bucket.remove(user_class)
break
if not to_stop:
return
if stop_rate is None or stop_rate >= user_count:
sleep_time = 0
logger.info("Stopping %i users" % (user_count))
else:
sleep_time = 1.0 / stop_rate
logger.info("Stopping %i users at rate of %g users/s" %
(user_count, stop_rate))
async_calls_to_stop = Group()
stop_group = Group()
while True:
user_to_stop: User = to_stop.pop(
random.randint(0,
len(to_stop) - 1))
logger.debug("Stopping %s" % user_to_stop._greenlet.name)
if user_to_stop._greenlet is greenlet.getcurrent():
# User called runner.quit(), so dont block waiting for killing to finish"
user_to_stop._group.killone(user_to_stop._greenlet,
block=False)
elif self.environment.stop_timeout:
async_calls_to_stop.add(
gevent.spawn_later(0, user_to_stop.stop, force=False))
stop_group.add(user_to_stop._greenlet)
else:
async_calls_to_stop.add(
gevent.spawn_later(0, user_to_stop.stop, force=True))
if to_stop:
gevent.sleep(sleep_time)
else:
break
async_calls_to_stop.join()
if not stop_group.join(timeout=self.environment.stop_timeout):
logger.info(
"Not all users finished their tasks & terminated in %s seconds. Stopping them..."
% self.environment.stop_timeout)
stop_group.kill(block=True)
logger.info("%i Users have been stopped, %g still running" %
(user_count, len(self.user_greenlets)))
class Runner:
"""
Orchestrates the load test by starting and stopping the users.
Use one of the :meth:`create_local_runner <locust.env.Environment.create_local_runner>`,
:meth:`create_master_runner <locust.env.Environment.create_master_runner>` or
:meth:`create_worker_runner <locust.env.Environment.create_worker_runner>` methods on
the :class:`Environment <locust.env.Environment>` instance to create a runner of the
desired type.
"""
def __init__(self, environment):
self.environment = environment
self.user_greenlets = Group()
self.greenlet = Group()
self.state = STATE_INIT
self.spawning_greenlet = None
self.shape_greenlet = None
self.shape_last_state = None
self.current_cpu_usage = 0
self.cpu_warning_emitted = False
self.greenlet.spawn(
self.monitor_cpu).link_exception(greenlet_exception_handler)
self.exceptions = {}
self.target_user_count = None
self.custom_messages = {}
# set up event listeners for recording requests
def on_request_success(request_type, name, response_time,
response_length, **_kwargs):
self.stats.log_request(request_type, name, response_time,
response_length)
def on_request_failure(request_type, name, response_time,
response_length, exception, **_kwargs):
self.stats.log_request(request_type, name, response_time,
response_length)
self.stats.log_error(request_type, name, exception)
# temporarily set log level to ignore warnings to suppress deprication message
loglevel = logging.getLogger().level
logging.getLogger().setLevel(logging.ERROR)
self.environment.events.request_success.add_listener(
on_request_success)
self.environment.events.request_failure.add_listener(
on_request_failure)
logging.getLogger().setLevel(loglevel)
self.connection_broken = False
# register listener that resets stats when spawning is complete
def on_spawning_complete(user_count):
self.update_state(STATE_RUNNING)
if environment.reset_stats:
logger.info("Resetting stats\n")
self.stats.reset_all()
self.environment.events.spawning_complete.add_listener(
on_spawning_complete)
def __del__(self):
# don't leave any stray greenlets if runner is removed
if self.greenlet and len(self.greenlet) > 0:
self.greenlet.kill(block=False)
@property
def user_classes(self):
return self.environment.user_classes
@property
def stats(self) -> RequestStats:
return self.environment.stats
@property
def errors(self):
return self.stats.errors
@property
def user_count(self):
"""
:returns: Number of currently running users
"""
return len(self.user_greenlets)
def update_state(self, new_state):
"""
Updates the current state
"""
# I (cyberwiz) commented out this logging, because it is too noisy even for debug level
# Uncomment it if you are specifically debugging state transitions
# logger.debug("Updating state to '%s', old state was '%s'" % (new_state, self.state))
self.state = new_state
def cpu_log_warning(self):
"""Called at the end of the test to repeat the warning & return the status"""
if self.cpu_warning_emitted:
logger.warning(
"CPU usage was too high at some point during the test! See https://docs.locust.io/en/stable/running-locust-distributed.html for how to distribute the load over multiple CPU cores or machines"
)
return True
return False
def weight_users(self, amount) -> List[Type[User]]:
"""
Distributes the amount of users for each WebLocust-class according to it's weight
returns a list "bucket" with the weighted users
"""
bucket = []
weight_sum = sum([user.weight for user in self.user_classes])
residuals = {}
for user in self.user_classes:
if self.environment.host is not None:
user.host = self.environment.host
# create users depending on weight
percent = user.weight / float(weight_sum)
num_users = int(round(amount * percent))
bucket.extend([user for x in range(num_users)])
# used to keep track of the amount of rounding was done if we need
# to add/remove some instances from bucket
residuals[user] = amount * percent - round(amount * percent)
if len(bucket) < amount:
# We got too few User classes in the bucket, so we need to create a few extra users,
# and we do this by iterating over each of the User classes - starting with the one
# where the residual from the rounding was the largest - and creating one of each until
# we get the correct amount
for user in [
l for l, r in sorted(
residuals.items(), key=lambda x: x[1], reverse=True)
][:amount - len(bucket)]:
bucket.append(user)
elif len(bucket) > amount:
# We've got too many users due to rounding errors so we need to remove some
for user in [
l for l, r in sorted(residuals.items(), key=lambda x: x[1])
][:len(bucket) - amount]:
bucket.remove(user)
return bucket
def spawn_users(self, spawn_count, spawn_rate, wait=False):
bucket = self.weight_users(spawn_count)
spawn_count = len(bucket)
if self.state == STATE_INIT or self.state == STATE_STOPPED:
self.update_state(STATE_SPAWNING)
existing_count = len(self.user_greenlets)
logger.info(
"Spawning %i users at the rate %g users/s (%i users already running)..."
% (spawn_count, spawn_rate, existing_count))
occurrence_count = dict([(l.__name__, 0) for l in self.user_classes])
def spawn():
sleep_time = 1.0 / spawn_rate
while True:
if not bucket:
logger.info("All users spawned: %s (%i total running)" % (
", ".join([
"%s: %d" % (name, count)
for name, count in occurrence_count.items()
]),
len(self.user_greenlets),
))
self.environment.events.spawning_complete.fire(
user_count=len(self.user_greenlets))
return
user_class = bucket.pop(random.randint(0, len(bucket) - 1))
occurrence_count[user_class.__name__] += 1
new_user = user_class(self.environment)
new_user.start(self.user_greenlets)
if len(self.user_greenlets) % 10 == 0:
logger.debug("%i users spawned" % len(self.user_greenlets))
if bucket:
gevent.sleep(sleep_time)
spawn()
if wait:
self.user_greenlets.join()
logger.info("All users stopped\n")
def stop_users(self, user_count, stop_rate=None):
"""
Stop `user_count` weighted users at a rate of `stop_rate`
"""
if user_count == 0 or stop_rate == 0:
return
bucket = self.weight_users(user_count)
user_count = len(bucket)
to_stop = []
for user_greenlet in self.user_greenlets:
try:
user = user_greenlet.args[0]
except IndexError:
logger.error(
"While stopping users, we encountered a user that didnt have proper args %s",
user_greenlet)
continue
for user_class in bucket:
if isinstance(user, user_class):
to_stop.append(user)
bucket.remove(user_class)
break
if not to_stop:
return
if stop_rate is None or stop_rate >= user_count:
sleep_time = 0
logger.info("Stopping %i users" % (user_count))
else:
sleep_time = 1.0 / stop_rate
logger.info("Stopping %i users at rate of %g users/s" %
(user_count, stop_rate))
async_calls_to_stop = Group()
stop_group = Group()
while True:
user_to_stop: User = to_stop.pop(
random.randint(0,
len(to_stop) - 1))
logger.debug("Stopping %s" % user_to_stop._greenlet.name)
if user_to_stop._greenlet is greenlet.getcurrent():
# User called runner.quit(), so dont block waiting for killing to finish"
user_to_stop._group.killone(user_to_stop._greenlet,
block=False)
elif self.environment.stop_timeout:
async_calls_to_stop.add(
gevent.spawn_later(0, user_to_stop.stop, force=False))
stop_group.add(user_to_stop._greenlet)
else:
async_calls_to_stop.add(
gevent.spawn_later(0, user_to_stop.stop, force=True))
if to_stop:
gevent.sleep(sleep_time)
else:
break
async_calls_to_stop.join()
if not stop_group.join(timeout=self.environment.stop_timeout):
logger.info(
"Not all users finished their tasks & terminated in %s seconds. Stopping them..."
% self.environment.stop_timeout)
stop_group.kill(block=True)
logger.info("%i Users have been stopped, %g still running" %
(user_count, len(self.user_greenlets)))
def monitor_cpu(self):
process = psutil.Process()
while True:
self.current_cpu_usage = process.cpu_percent()
if self.current_cpu_usage > 90 and not self.cpu_warning_emitted:
logging.warning(
"CPU usage above 90%! This may constrain your throughput and may even give inconsistent response time measurements! See https://docs.locust.io/en/stable/running-locust-distributed.html for how to distribute the load over multiple CPU cores or machines"
)
self.cpu_warning_emitted = True
gevent.sleep(CPU_MONITOR_INTERVAL)
def start(self, user_count, spawn_rate, wait=False):
"""
Start running a load test
:param user_count: Total number of users to start
:param spawn_rate: Number of users to spawn per second
:param wait: If True calls to this method will block until all users are spawned.
If False (the default), a greenlet that spawns the users will be
started and the call to this method will return immediately.
"""
if self.state != STATE_RUNNING and self.state != STATE_SPAWNING:
self.stats.clear_all()
self.exceptions = {}
self.cpu_warning_emitted = False
self.worker_cpu_warning_emitted = False
self.target_user_count = user_count
if self.state != STATE_INIT and self.state != STATE_STOPPED:
logger.debug(
"Updating running test with %d users, %.2f spawn rate and wait=%r"
% (user_count, spawn_rate, wait))
self.update_state(STATE_SPAWNING)
if self.user_count > user_count:
# Stop some users
stop_count = self.user_count - user_count
self.stop_users(stop_count, spawn_rate)
elif self.user_count < user_count:
# Spawn some users
spawn_count = user_count - self.user_count
self.spawn_users(spawn_count=spawn_count,
spawn_rate=spawn_rate)
else:
self.environment.events.spawning_complete.fire(
user_count=self.user_count)
else:
self.spawn_rate = spawn_rate
self.spawn_users(user_count, spawn_rate=spawn_rate, wait=wait)
def start_shape(self):
if self.shape_greenlet:
logger.info(
"There is an ongoing shape test running. Editing is disabled")
return
logger.info(
"Shape test starting. User count and spawn rate are ignored for this type of load test"
)
self.update_state(STATE_INIT)
self.shape_greenlet = self.greenlet.spawn(self.shape_worker)
self.shape_greenlet.link_exception(greenlet_exception_handler)
self.environment.shape_class.reset_time()
def shape_worker(self):
logger.info("Shape worker starting")
while self.state == STATE_INIT or self.state == STATE_SPAWNING or self.state == STATE_RUNNING:
new_state = self.environment.shape_class.tick()
if new_state is None:
logger.info("Shape test stopping")
if self.environment.parsed_options and self.environment.parsed_options.headless:
self.quit()
else:
self.stop()
elif self.shape_last_state == new_state:
gevent.sleep(1)
else:
user_count, spawn_rate = new_state
logger.info(
"Shape test updating to %d users at %.2f spawn rate" %
(user_count, spawn_rate))
self.start(user_count=user_count, spawn_rate=spawn_rate)
self.shape_last_state = new_state
def stop(self):
"""
Stop a running load test by stopping all running users
"""
logger.debug("Stopping all users")
self.update_state(STATE_CLEANUP)
# if we are currently spawning users we need to kill the spawning greenlet first
if self.spawning_greenlet and not self.spawning_greenlet.ready():
self.spawning_greenlet.kill(block=True)
self.stop_users(self.user_count)
self.update_state(STATE_STOPPED)
self.cpu_log_warning()
def quit(self):
"""
Stop any running load test and kill all greenlets for the runner
"""
self.stop()
self.greenlet.kill(block=True)
def log_exception(self, node_id, msg, formatted_tb):
key = hash(formatted_tb)
row = self.exceptions.setdefault(key, {
"count": 0,
"msg": msg,
"traceback": formatted_tb,
"nodes": set()
})
row["count"] += 1
row["nodes"].add(node_id)
self.exceptions[key] = row
def register_message(self, msg_type, listener):
"""
Register a listener for a custom message from another node
:param msg_type: The type of the message to listen for
:param listener: The function to execute when the message is received
"""
self.custom_messages[msg_type] = listener
class Runner(object):
"""
Orchestrates the load test by starting and stopping the users.
Use one of the :meth:`create_local_runner <locust.env.Environment.create_local_runner>`,
:meth:`create_master_runner <locust.env.Environment.create_master_runner>` or
:meth:`create_worker_runner <locust.env.Environment.create_worker_runner>` methods on
the :class:`Environment <locust.env.Environment>` instance to create a runner of the
desired type.
"""
def __init__(self, environment):
self.environment = environment
self.user_greenlets = Group()
self.greenlet = Group()
self.state = STATE_INIT
self.hatching_greenlet = None
self.stepload_greenlet = None
self.shape_greenlet = None
self.shape_last_state = None
self.current_cpu_usage = 0
self.cpu_warning_emitted = False
self.greenlet.spawn(
self.monitor_cpu).link_exception(greenlet_exception_handler)
self.exceptions = {}
self.target_user_count = None
# set up event listeners for recording requests
def on_request_success(request_type, name, response_time,
response_length, **kwargs):
self.stats.log_request(request_type, name, response_time,
response_length)
def on_request_failure(request_type, name, response_time,
response_length, exception, **kwargs):
self.stats.log_request(request_type, name, response_time,
response_length)
self.stats.log_error(request_type, name, exception)
self.environment.events.request_success.add_listener(
on_request_success)
self.environment.events.request_failure.add_listener(
on_request_failure)
self.connection_broken = False
# register listener that resets stats when hatching is complete
def on_hatch_complete(user_count):
self.state = STATE_RUNNING
if environment.reset_stats:
logger.info("Resetting stats\n")
self.stats.reset_all()
self.environment.events.hatch_complete.add_listener(on_hatch_complete)
def __del__(self):
# don't leave any stray greenlets if runner is removed
if self.greenlet and len(self.greenlet) > 0:
self.greenlet.kill(block=False)
@property
def user_classes(self):
return self.environment.user_classes
@property
def stats(self) -> RequestStats:
return self.environment.stats
@property
def errors(self):
return self.stats.errors
@property
def user_count(self):
"""
:returns: Number of currently running users
"""
return len(self.user_greenlets)
def cpu_log_warning(self):
"""Called at the end of the test to repeat the warning & return the status"""
if self.cpu_warning_emitted:
logger.warning(
"CPU usage was too high at some point during the test! See https://docs.locust.io/en/stable/running-locust-distributed.html for how to distribute the load over multiple CPU cores or machines"
)
return True
return False
def weight_users(self, amount):
"""
Distributes the amount of users for each WebLocust-class according to it's weight
returns a list "bucket" with the weighted users
"""
bucket = []
weight_sum = sum([user.weight for user in self.user_classes])
residuals = {}
for user in self.user_classes:
if self.environment.host is not None:
user.host = self.environment.host
# create users depending on weight
percent = user.weight / float(weight_sum)
num_users = int(round(amount * percent))
bucket.extend([user for x in range(num_users)])
# used to keep track of the amount of rounding was done if we need
# to add/remove some instances from bucket
residuals[user] = amount * percent - round(amount * percent)
if len(bucket) < amount:
# We got too few User classes in the bucket, so we need to create a few extra users,
# and we do this by iterating over each of the User classes - starting with the one
# where the residual from the rounding was the largest - and creating one of each until
# we get the correct amount
for user in [
l for l, r in sorted(
residuals.items(), key=lambda x: x[1], reverse=True)
][:amount - len(bucket)]:
bucket.append(user)
elif len(bucket) > amount:
# We've got too many users due to rounding errors so we need to remove some
for user in [
l for l, r in sorted(residuals.items(), key=lambda x: x[1])
][:len(bucket) - amount]:
bucket.remove(user)
return bucket
def spawn_users(self, spawn_count, hatch_rate, wait=False):
bucket = self.weight_users(spawn_count)
spawn_count = len(bucket)
if self.state == STATE_INIT or self.state == STATE_STOPPED:
self.state = STATE_HATCHING
existing_count = len(self.user_greenlets)
logger.info(
"Hatching and swarming %i users at the rate %g users/s (%i users already running)..."
% (spawn_count, hatch_rate, existing_count))
occurrence_count = dict([(l.__name__, 0) for l in self.user_classes])
def hatch():
sleep_time = 1.0 / hatch_rate
while True:
if not bucket:
logger.info(
"All users hatched: %s (%i already running)" % (
", ".join([
"%s: %d" % (name, count)
for name, count in occurrence_count.items()
]),
existing_count,
))
self.environment.events.hatch_complete.fire(
user_count=len(self.user_greenlets))
return
user_class = bucket.pop(random.randint(0, len(bucket) - 1))
occurrence_count[user_class.__name__] += 1
new_user = user_class(self.environment)
new_user.start(self.user_greenlets)
if len(self.user_greenlets) % 10 == 0:
logger.debug("%i users hatched" % len(self.user_greenlets))
if bucket:
gevent.sleep(sleep_time)
hatch()
if wait:
self.user_greenlets.join()
logger.info("All users stopped\n")
def stop_users(self, user_count, stop_rate=None):
"""
Stop `user_count` weighted users at a rate of `stop_rate`
"""
if user_count == 0 or stop_rate == 0:
return
bucket = self.weight_users(user_count)
user_count = len(bucket)
to_stop = []
for g in self.user_greenlets:
for l in bucket:
user = g.args[0]
if l == type(user):
to_stop.append(user)
bucket.remove(l)
break
if not to_stop:
return
if stop_rate == None or stop_rate >= user_count:
sleep_time = 0
logger.info("Stopping %i users immediately" % (user_count))
else:
sleep_time = 1.0 / stop_rate
logger.info("Stopping %i users at rate of %g users/s" %
(user_count, stop_rate))
if self.environment.stop_timeout:
stop_group = Group()
while True:
user_to_stop = to_stop.pop(random.randint(0, len(to_stop) - 1))
logger.debug('Stopping %s' % user_to_stop._greenlet.name)
if self.environment.stop_timeout:
if not user_to_stop.stop(self.user_greenlets, force=False):
# User.stop() returns False if the greenlet was not stopped, so we'll need
# to add it's greenlet to our stopping Group so we can wait for it to finish it's task
stop_group.add(user_to_stop._greenlet)
else:
user_to_stop.stop(self.user_greenlets, force=True)
if to_stop:
gevent.sleep(sleep_time)
else:
break
if self.environment.stop_timeout and not stop_group.join(
timeout=self.environment.stop_timeout):
logger.info(
"Not all users finished their tasks & terminated in %s seconds. Stopping them..."
% self.environment.stop_timeout)
stop_group.kill(block=True)
logger.info("%i Users have been stopped" % user_count)
def monitor_cpu(self):
process = psutil.Process()
while True:
self.current_cpu_usage = process.cpu_percent()
if self.current_cpu_usage > 90 and not self.cpu_warning_emitted:
logging.warning(
"CPU usage above 90%! This may constrain your throughput and may even give inconsistent response time measurements! See https://docs.locust.io/en/stable/running-locust-distributed.html for how to distribute the load over multiple CPU cores or machines"
)
self.cpu_warning_emitted = True
gevent.sleep(CPU_MONITOR_INTERVAL)
def start(self, user_count, hatch_rate, wait=False):
"""
Start running a load test
:param user_count: Number of users to start
:param hatch_rate: Number of users to spawn per second
:param wait: If True calls to this method will block until all users are spawned.
If False (the default), a greenlet that spawns the users will be
started and the call to this method will return immediately.
"""
if self.state != STATE_RUNNING and self.state != STATE_HATCHING:
self.stats.clear_all()
self.exceptions = {}
self.cpu_warning_emitted = False
self.worker_cpu_warning_emitted = False
self.target_user_count = user_count
if self.state != STATE_INIT and self.state != STATE_STOPPED:
logger.debug(
"Updating running test with %d users, %.2f hatch rate and wait=%r"
% (user_count, hatch_rate, wait))
self.state = STATE_HATCHING
if self.user_count > user_count:
# Stop some users
stop_count = self.user_count - user_count
self.stop_users(stop_count, hatch_rate)
elif self.user_count < user_count:
# Spawn some users
spawn_count = user_count - self.user_count
self.spawn_users(spawn_count=spawn_count,
hatch_rate=hatch_rate)
else:
self.environment.events.hatch_complete.fire(
user_count=self.user_count)
else:
self.hatch_rate = hatch_rate
self.spawn_users(user_count, hatch_rate=hatch_rate, wait=wait)
def start_stepload(self, user_count, hatch_rate, step_user_count,
step_duration):
if user_count < step_user_count:
logger.error(
"Invalid parameters: total user count of %d is smaller than step user count of %d"
% (user_count, step_user_count))
return
self.total_users = user_count
if self.stepload_greenlet:
logger.info(
"There is an ongoing swarming in Step Load mode, will stop it now."
)
self.stepload_greenlet.kill()
logger.info(
"Start a new swarming in Step Load mode: total user count of %d, hatch rate of %d, step user count of %d, step duration of %d "
% (user_count, hatch_rate, step_user_count, step_duration))
self.state = STATE_INIT
self.stepload_greenlet = self.greenlet.spawn(self.stepload_worker,
hatch_rate,
step_user_count,
step_duration)
self.stepload_greenlet.link_exception(greenlet_exception_handler)
def stepload_worker(self, hatch_rate, step_users_growth, step_duration):
current_num_users = 0
while self.state == STATE_INIT or self.state == STATE_HATCHING or self.state == STATE_RUNNING:
current_num_users += step_users_growth
if current_num_users > int(self.total_users):
logger.info("Step Load is finished")
break
self.start(current_num_users, hatch_rate)
logger.info("Step loading: start hatch job of %d user" %
(current_num_users))
gevent.sleep(step_duration)
def start_shape(self):
if self.shape_greenlet:
logger.info(
"There is an ongoing shape test running. Editing is disabled")
return
logger.info(
"Shape test starting. User count and hatch rate are ignored for this type of load test"
)
self.state = STATE_INIT
self.shape_greenlet = self.greenlet.spawn(self.shape_worker)
self.shape_greenlet.link_exception(greenlet_exception_handler)
def shape_worker(self):
logger.info("Shape worker starting")
while self.state == STATE_INIT or self.state == STATE_HATCHING or self.state == STATE_RUNNING:
new_state = self.environment.shape_class.tick()
if new_state is None:
logger.info("Shape test stopping")
self.stop()
elif self.shape_last_state == new_state:
gevent.sleep(1)
else:
user_count, hatch_rate = new_state
logger.info(
"Shape test updating to %d users at %.2f hatch rate" %
(user_count, hatch_rate))
self.start(user_count=user_count, hatch_rate=hatch_rate)
self.shape_last_state = new_state
def stop(self):
"""
Stop a running load test by stopping all running users
"""
self.state = STATE_CLEANUP
# if we are currently hatching users we need to kill the hatching greenlet first
if self.hatching_greenlet and not self.hatching_greenlet.ready():
self.hatching_greenlet.kill(block=True)
self.stop_users(self.user_count)
self.state = STATE_STOPPED
self.cpu_log_warning()
def quit(self):
"""
Stop any running load test and kill all greenlets for the runner
"""
self.stop()
self.greenlet.kill(block=True)
def log_exception(self, node_id, msg, formatted_tb):
key = hash(formatted_tb)
row = self.exceptions.setdefault(key, {
"count": 0,
"msg": msg,
"traceback": formatted_tb,
"nodes": set()
})
row["count"] += 1
row["nodes"].add(node_id)
self.exceptions[key] = row
class Consumer(object):
"""High level NSQ consumer.
A Consumer will connect to the nsqd tcp addresses or poll the provided
nsqlookupd http addresses for the configured topic and send signals to
message handlers connected to the :attr:`on_message` signal or provided by
``message_handler``.
Messages will automatically be finished when the message handle returns
unless :meth:`message.enable_async() <gnsq.Message.enable_async>` is called.
If an exception occurs or :class:`~gnsq.errors.NSQRequeueMessage` is raised,
the message will be requeued.
The Consumer will handle backing off of failed messages up to a configurable
``max_interval`` as well as automatically reconnecting to dropped
connections.
Example usage::
from gnsq import Consumer
consumer = gnsq.Consumer('topic', 'channel', 'localhost:4150')
@consumer.on_message.connect
def handler(consumer, message):
print 'got message:', message.body
consumer.start()
:param topic: specifies the desired NSQ topic
:param channel: specifies the desired NSQ channel
:param nsqd_tcp_addresses: a sequence of string addresses of the nsqd
instances this consumer should connect to
:param lookupd_http_addresses: a sequence of string addresses of the
nsqlookupd instances this consumer should query for producers of the
specified topic
:param name: a string that is used for logging messages (defaults to
``'gnsq.consumer.{topic}.{channel}'``)
:param message_handler: the callable that will be executed for each message
received
:param max_tries: the maximum number of attempts the consumer will make to
process a message after which messages will be automatically discarded
:param max_in_flight: the maximum number of messages this consumer will
pipeline for processing. this value will be divided evenly amongst the
configured/discovered nsqd producers
:param requeue_delay: the default delay to use when requeueing a failed
message
:param lookupd_poll_interval: the amount of time in seconds between querying
all of the supplied nsqlookupd instances. A random amount of time based
on this value will be initially introduced in order to add jitter when
multiple consumers are running
:param lookupd_poll_jitter: the maximum fractional amount of jitter to add
to the lookupd poll loop. This helps evenly distribute requests even if
multiple consumers restart at the same time.
:param low_ready_idle_timeout: the amount of time in seconds to wait for a
message from a producer when in a state where RDY counts are
re-distributed (ie. `max_in_flight` < `num_producers`)
:param max_backoff_duration: the maximum time we will allow a backoff state
to last in seconds. If zero, backoff wil not occur
:param backoff_on_requeue: if ``False``, backoff will only occur on
exception
:param **kwargs: passed to :class:`~gnsq.NsqdTCPClient` initialization
"""
def __init__(self,
topic,
channel,
nsqd_tcp_addresses=[],
lookupd_http_addresses=[],
name=None,
message_handler=None,
max_tries=5,
max_in_flight=1,
requeue_delay=0,
lookupd_poll_interval=60,
lookupd_poll_jitter=0.3,
low_ready_idle_timeout=10,
max_backoff_duration=128,
backoff_on_requeue=True,
**kwargs):
if not nsqd_tcp_addresses and not lookupd_http_addresses:
raise ValueError('must specify at least one nsqd or lookupd')
self.nsqd_tcp_addresses = parse_nsqds(nsqd_tcp_addresses)
self.lookupds = parse_lookupds(lookupd_http_addresses)
self.iterlookupds = cycle(self.lookupds)
self.topic = topic
self.channel = channel
self.max_tries = max_tries
self.max_in_flight = max_in_flight
self.requeue_delay = requeue_delay
self.lookupd_poll_interval = lookupd_poll_interval
self.lookupd_poll_jitter = lookupd_poll_jitter
self.low_ready_idle_timeout = low_ready_idle_timeout
self.backoff_on_requeue = backoff_on_requeue
self.max_backoff_duration = max_backoff_duration
self.conn_kwargs = kwargs
if name:
self.name = name
else:
self.name = '%s.%s.%s' % (__name__, self.topic, self.channel)
if message_handler is not None:
self.on_message.connect(message_handler, weak=False)
self.logger = logging.getLogger(self.name)
self._state = INIT
self._redistributed_ready_event = Event()
self._connection_backoffs = defaultdict(self._create_backoff)
self._message_backoffs = defaultdict(self._create_backoff)
self._connections = {}
self._workers = Group()
self._killables = Group()
@cached_property
def on_message(self):
"""Emitted when a message is received.
The signal sender is the consumer and the ``message`` is sent as an
argument. The ``message_handler`` param is connected to this signal.
"""
return blinker.Signal(doc='Emitted when a message is received.')
@cached_property
def on_response(self):
"""Emitted when a response is received.
The signal sender is the consumer and the ``response`` is sent as an
argument.
"""
return blinker.Signal(doc='Emitted when a response is received.')
@cached_property
def on_error(self):
"""Emitted when an error is received.
The signal sender is the consumer and the ``error`` is sent as an
argument.
"""
return blinker.Signal(doc='Emitted when a error is received.')
@cached_property
def on_finish(self):
"""Emitted after a message is successfully finished.
The signal sender is the consumer and the ``message_id`` is sent as an
argument.
"""
return blinker.Signal(doc='Emitted after the a message is finished.')
@cached_property
def on_requeue(self):
"""Emitted after a message is requeued.
The signal sender is the consumer and the ``message_id`` and ``timeout``
are sent as arguments.
"""
return blinker.Signal(doc='Emitted after the a message is requeued.')
@cached_property
def on_giving_up(self):
"""Emitted after a giving up on a message.
Emitted when a message has exceeded the maximum number of attempts
(``max_tries``) and will no longer be requeued. This is useful to
perform tasks such as writing to disk, collecting statistics etc. The
signal sender is the consumer and the ``message`` is sent as an
argument.
"""
return blinker.Signal(doc='Sent after a giving up on a message.')
@cached_property
def on_auth(self):
"""Emitted after a connection is successfully authenticated.
The signal sender is the consumer and the ``conn`` and parsed
``response`` are sent as arguments.
"""
return blinker.Signal(doc='Emitted when a response is received.')
@cached_property
def on_exception(self):
"""Emitted when an exception is caught while handling a message.
The signal sender is the consumer and the ``message`` and ``error`` are
sent as arguments.
"""
return blinker.Signal(doc='Emitted when an exception is caught.')
@cached_property
def on_close(self):
"""Emitted after :meth:`close`.
The signal sender is the consumer.
"""
return blinker.Signal(doc='Emitted after the consumer is closed.')
def start(self, block=True):
"""Start discovering and listing to connections."""
if self._state == INIT:
if not any(self.on_message.receivers_for(blinker.ANY)):
raise RuntimeError('no receivers connected to on_message')
self.logger.debug('starting %s...', self.name)
self._state = RUNNING
self.query_nsqd()
if self.lookupds:
self.query_lookupd()
self._killables.add(self._workers.spawn(self._poll_lookupd))
self._killables.add(self._workers.spawn(self._poll_ready))
else:
self.logger.warning('%s already started', self.name)
if block:
self.join()
def close(self):
"""Immediately close all connections and stop workers."""
if not self.is_running:
return
self._state = CLOSED
self.logger.debug('killing %d worker(s)', len(self._killables))
self._killables.kill(block=False)
self.logger.debug('closing %d connection(s)', len(self._connections))
for conn in self._connections:
conn.close_stream()
self.on_close.send(self)
def join(self, timeout=None, raise_error=False):
"""Block until all connections have closed and workers stopped."""
self._workers.join(timeout, raise_error)
@property
def is_running(self):
"""Check if consumer is currently running."""
return self._state == RUNNING
@property
def is_starved(self):
"""Evaluate whether any of the connections are starved.
This property should be used by message handlers to reliably identify
when to process a batch of messages.
"""
return any(conn.is_starved for conn in self._connections)
@property
def total_ready_count(self):
return sum(c.ready_count for c in self._connections)
@property
def total_in_flight(self):
return sum(c.in_flight for c in self._connections)
def query_nsqd(self):
self.logger.debug('querying nsqd...')
for address in self.nsqd_tcp_addresses:
address, port = address.split(':')
self.connect_to_nsqd(address, int(port))
def query_lookupd(self):
self.logger.debug('querying lookupd...')
lookupd = next(self.iterlookupds)
try:
producers = lookupd.lookup(self.topic)['producers']
self.logger.debug('found %d producers', len(producers))
except Exception as error:
self.logger.warning('Failed to lookup %s on %s (%s)', self.topic,
lookupd.address, error)
return
for producer in producers:
self.connect_to_nsqd(producer['broadcast_address'],
producer['tcp_port'])
def _poll_lookupd(self):
try:
delay = self.lookupd_poll_interval * self.lookupd_poll_jitter
gevent.sleep(random.random() * delay)
while True:
gevent.sleep(self.lookupd_poll_interval)
self.query_lookupd()
except gevent.GreenletExit:
pass
def _poll_ready(self):
try:
while True:
if self._redistributed_ready_event.wait(5):
self._redistributed_ready_event.clear()
self._redistribute_ready_state()
except gevent.GreenletExit:
pass
def _redistribute_ready_state(self):
if not self.is_running:
return
if len(self._connections) > self.max_in_flight:
ready_state = self._get_unsaturated_ready_state()
else:
ready_state = self._get_saturated_ready_state()
for conn, count in ready_state.items():
if conn.ready_count == count:
self.logger.debug('[%s] RDY count already %d', conn, count)
continue
self.logger.debug('[%s] sending RDY %d', conn, count)
try:
conn.ready(count)
except NSQSocketError as error:
self.logger.warning('[%s] RDY %d failed (%r)', conn, count,
error)
def _get_unsaturated_ready_state(self):
ready_state = {}
active = []
for conn, state in self._connections.items():
if state == BACKOFF:
ready_state[conn] = 0
elif state in (RUNNING, THROTTLED):
active.append(conn)
random.shuffle(active)
for conn in active[self.max_in_flight:]:
ready_state[conn] = 0
for conn in active[:self.max_in_flight]:
ready_state[conn] = 1
return ready_state
def _get_saturated_ready_state(self):
ready_state = {}
active = []
now = time.time()
for conn, state in self._connections.items():
if state == BACKOFF:
ready_state[conn] = 0
elif state == THROTTLED:
ready_state[conn] = 1
elif state == RUNNING:
if (now - conn.last_message) > self.low_ready_idle_timeout:
self.logger.info(
'[%s] idle connection, giving up RDY count', conn)
ready_state[conn] = 1
else:
active.append(conn)
if not active:
return ready_state
ready_available = self.max_in_flight - sum(ready_state.values())
connection_max_in_flight = ready_available // len(active)
for conn in active:
ready_state[conn] = connection_max_in_flight
for conn in random.sample(active, ready_available % len(active)):
ready_state[conn] += 1
return ready_state
def redistribute_ready_state(self):
self._redistributed_ready_event.set()
def connect_to_nsqd(self, address, port):
if not self.is_running:
return
conn = NsqdTCPClient(address, port, **self.conn_kwargs)
if conn in self._connections:
self.logger.debug('[%s] already connected', conn)
return
self._connections[conn] = INIT
self.logger.debug('[%s] connecting...', conn)
conn.on_message.connect(self.handle_message)
conn.on_response.connect(self.handle_response)
conn.on_error.connect(self.handle_error)
conn.on_finish.connect(self.handle_finish)
conn.on_requeue.connect(self.handle_requeue)
conn.on_auth.connect(self.handle_auth)
try:
conn.connect()
conn.identify()
if conn.max_ready_count < self.max_in_flight:
msg = ('[%s] max RDY count %d < consumer max in flight %d, '
'truncation possible')
self.logger.warning(msg, conn, conn.max_ready_count,
self.max_in_flight)
conn.subscribe(self.topic, self.channel)
except NSQException as error:
self.logger.warning('[%s] connection failed (%r)', conn, error)
self.handle_connection_failure(conn)
return
# Check if we've closed since we started
if not self.is_running:
self.handle_connection_failure(conn)
return
self.logger.info('[%s] connection successful', conn)
self.handle_connection_success(conn)
def _listen(self, conn):
try:
conn.listen()
except NSQException as error:
self.logger.warning('[%s] connection lost (%r)', conn, error)
self.handle_connection_failure(conn)
def handle_connection_success(self, conn):
self._connections[conn] = THROTTLED
self._workers.spawn(self._listen, conn)
self.redistribute_ready_state()
if str(conn) not in self.nsqd_tcp_addresses:
return
self._connection_backoffs[conn].success()
def handle_connection_failure(self, conn):
del self._connections[conn]
conn.close_stream()
if not self.is_running:
return
self.redistribute_ready_state()
if str(conn) not in self.nsqd_tcp_addresses:
return
seconds = self._connection_backoffs[conn].failure().get_interval()
self.logger.debug('[%s] retrying in %ss', conn, seconds)
gevent.spawn_later(seconds, self.connect_to_nsqd, conn.address,
conn.port)
def handle_auth(self, conn, response):
metadata = []
if response.get('identity'):
metadata.append("Identity: %r" % response['identity'])
if response.get('permission_count'):
metadata.append("Permissions: %d" % response['permission_count'])
if response.get('identity_url'):
metadata.append(response['identity_url'])
self.logger.info('[%s] AUTH accepted %s', conn, ' '.join(metadata))
self.on_auth.send(self, conn=conn, response=response)
def handle_response(self, conn, response):
self.logger.debug('[%s] response: %s', conn, response)
self.on_response.send(self, response=response)
def handle_error(self, conn, error):
self.logger.debug('[%s] error: %s', conn, error)
self.on_error.send(self, error=error)
def _handle_message(self, message):
if self.max_tries and message.attempts > self.max_tries:
self.logger.warning("giving up on message '%s' after max tries %d",
message.id, self.max_tries)
self.on_giving_up.send(self, message=message)
return message.finish()
self.on_message.send(self, message=message)
if not self.is_running:
return
if message.is_async():
return
if message.has_responded():
return
message.finish()
def handle_message(self, conn, message):
self.logger.debug('[%s] got message: %s', conn, message.id)
try:
return self._handle_message(message)
except NSQRequeueMessage as error:
if error.backoff is None:
backoff = self.backoff_on_requeue
else:
backoff = error.backoff
except Exception as error:
backoff = True
self.logger.exception(
'[%s] caught exception while handling message', conn)
self.on_exception.send(self, message=message, error=error)
if not self.is_running:
return
if message.has_responded():
return
try:
message.requeue(self.requeue_delay, backoff)
except NSQException as error:
self.logger.warning('[%s] error requeueing message (%r)', conn,
error)
def _create_backoff(self):
return BackoffTimer(max_interval=self.max_backoff_duration)
def _start_backoff(self, conn):
self._connections[conn] = BACKOFF
interval = self._message_backoffs[conn].get_interval()
gevent.spawn_later(interval, self._start_throttled, conn)
self.logger.info('[%s] backing off for %s seconds', conn, interval)
self.redistribute_ready_state()
def _start_throttled(self, conn):
if self._connections.get(conn) != BACKOFF:
return
self._connections[conn] = THROTTLED
self.logger.info('[%s] testing backoff state with RDY 1', conn)
self.redistribute_ready_state()
def _complete_backoff(self, conn):
if self._message_backoffs[conn].is_reset():
self._connections[conn] = RUNNING
self.logger.info('throttle complete, resuming normal operation')
self.redistribute_ready_state()
else:
self._start_backoff(conn)
def _finish_message(self, conn, backoff):
if not self.max_backoff_duration:
return
try:
state = self._connections[conn]
except KeyError:
return
if state == BACKOFF:
return
if backoff:
self._message_backoffs[conn].failure()
self._start_backoff(conn)
elif state == THROTTLED:
self._message_backoffs[conn].success()
self._complete_backoff(conn)
def handle_finish(self, conn, message_id):
self.logger.debug('[%s] finished message: %s', conn, message_id)
self._finish_message(conn, backoff=False)
self.on_finish.send(self, message_id=message_id)
def handle_requeue(self, conn, message_id, timeout, backoff):
self.logger.debug('[%s] requeued message: %s (%s)', conn, message_id,
timeout)
self._finish_message(conn, backoff=backoff)
self.on_requeue.send(self, message_id=message_id, timeout=timeout)
class LocustRunner(object):
def __init__(self, locust_classes, options):
self.options = options
self.locust_classes = locust_classes
self.hatch_rate = options.hatch_rate
self.num_clients = options.num_clients
self.host = options.host
self.locusts = Group()
self.greenlet = self.locusts
self.state = STATE_INIT
self.hatching_greenlet = None
self.stepload_greenlet = None
self.exceptions = {}
self.stats = global_stats
self.step_load = options.step_load
# register listener that resets stats when hatching is complete
def on_hatch_complete(user_count):
self.state = STATE_RUNNING
if self.options.reset_stats:
logger.info("Resetting stats\n")
self.stats.reset_all()
events.hatch_complete += on_hatch_complete
@property
def request_stats(self):
return self.stats.entries
@property
def errors(self):
return self.stats.errors
@property
def user_count(self):
return len(self.locusts)
def weight_locusts(self, amount):
"""
Distributes the amount of locusts for each WebLocust-class according to it's weight
returns a list "bucket" with the weighted locusts
"""
bucket = []
weight_sum = sum((locust.weight for locust in self.locust_classes
if locust.task_set))
residuals = {}
for locust in self.locust_classes:
if not locust.task_set:
warnings.warn(
"Notice: Found Locust class (%s) got no task_set. Skipping..."
% locust.__name__)
continue
if self.host is not None:
locust.host = self.host
# create locusts depending on weight
percent = locust.weight / float(weight_sum)
num_locusts = int(round(amount * percent))
bucket.extend([locust for x in xrange(0, num_locusts)])
# used to keep track of the amount of rounding was done if we need
# to add/remove some instances from bucket
residuals[locust] = amount * percent - round(amount * percent)
if len(bucket) < amount:
# We got too few locust classes in the bucket, so we need to create a few extra locusts,
# and we do this by iterating over each of the Locust classes - starting with the one
# where the residual from the rounding was the largest - and creating one of each until
# we get the correct amount
for locust in [
l for l, r in sorted(
residuals.items(), key=lambda x: x[1], reverse=True)
][:amount - len(bucket)]:
bucket.append(locust)
elif len(bucket) > amount:
# We've got too many locusts due to rounding errors so we need to remove some
for locust in [
l for l, r in sorted(residuals.items(), key=lambda x: x[1])
][:len(bucket) - amount]:
bucket.remove(locust)
return bucket
def spawn_locusts(self, spawn_count=None, wait=False):
if spawn_count is None:
spawn_count = self.num_clients
bucket = self.weight_locusts(spawn_count)
spawn_count = len(bucket)
if self.state == STATE_INIT or self.state == STATE_STOPPED:
self.state = STATE_HATCHING
self.num_clients = spawn_count
else:
self.num_clients += spawn_count
logger.info(
"Hatching and swarming %i clients at the rate %g clients/s..." %
(spawn_count, self.hatch_rate))
occurrence_count = dict([(l.__name__, 0) for l in self.locust_classes])
def hatch():
sleep_time = 1.0 / self.hatch_rate
while True:
if not bucket:
logger.info("All locusts hatched: %s" % ", ".join([
"%s: %d" % (name, count)
for name, count in six.iteritems(occurrence_count)
]))
events.hatch_complete.fire(user_count=self.num_clients)
return
locust = bucket.pop(random.randint(0, len(bucket) - 1))
occurrence_count[locust.__name__] += 1
new_locust = locust()
def start_locust(_):
try:
new_locust.run(runner=self)
except GreenletExit:
pass
self.locusts.spawn(start_locust, new_locust)
if len(self.locusts) % 10 == 0:
logger.debug("%i locusts hatched" % len(self.locusts))
gevent.sleep(sleep_time)
hatch()
if wait:
self.locusts.join()
logger.info("All locusts dead\n")
def kill_locusts(self, kill_count):
"""
Kill a kill_count of weighted locusts from the Group() object in self.locusts
"""
bucket = self.weight_locusts(kill_count)
kill_count = len(bucket)
self.num_clients -= kill_count
logger.info("Killing %i locusts" % kill_count)
dying = []
for g in self.locusts:
for l in bucket:
if l == type(g.args[0]):
dying.append(g)
bucket.remove(l)
break
self.kill_locust_greenlets(dying)
events.hatch_complete.fire(user_count=self.num_clients)
def kill_locust_greenlets(self, greenlets):
"""
Kill running locust greenlets. If options.stop_timeout is set, we try to stop the
Locust users gracefully
"""
if self.options.stop_timeout:
dying = Group()
for g in greenlets:
locust = g.args[0]
if locust._state == LOCUST_STATE_WAITING:
self.locusts.killone(g)
else:
locust._state = LOCUST_STATE_STOPPING
dying.add(g)
if not dying.join(timeout=self.options.stop_timeout):
logger.info(
"Not all locusts finished their tasks & terminated in %s seconds. Killing them..."
% self.options.stop_timeout)
dying.kill(block=True)
else:
for g in greenlets:
self.locusts.killone(g)
def start_hatching(self, locust_count=None, hatch_rate=None, wait=False):
if self.state != STATE_RUNNING and self.state != STATE_HATCHING:
self.stats.clear_all()
self.exceptions = {}
events.locust_start_hatching.fire()
# Dynamically changing the locust count
if self.state != STATE_INIT and self.state != STATE_STOPPED:
self.state = STATE_HATCHING
if self.num_clients > locust_count:
# Kill some locusts
kill_count = self.num_clients - locust_count
self.kill_locusts(kill_count)
elif self.num_clients < locust_count:
# Spawn some locusts
if hatch_rate:
self.hatch_rate = hatch_rate
spawn_count = locust_count - self.num_clients
self.spawn_locusts(spawn_count=spawn_count)
else:
events.hatch_complete.fire(user_count=self.num_clients)
else:
if hatch_rate:
self.hatch_rate = hatch_rate
if locust_count is not None:
self.spawn_locusts(locust_count, wait=wait)
else:
self.spawn_locusts(wait=wait)
def start_stepload(self, locust_count, hatch_rate, step_locust_count,
step_duration):
if locust_count < step_locust_count:
logger.error(
"Invalid parameters: total locust count of %d is smaller than step locust count of %d"
% (locust_count, step_locust_count))
return
self.total_clients = locust_count
self.hatch_rate = hatch_rate
self.step_clients_growth = step_locust_count
self.step_duration = step_duration
if self.stepload_greenlet:
logger.info(
"There is an ongoing swarming in Step Load mode, will stop it now."
)
self.greenlet.killone(self.stepload_greenlet)
logger.info(
"Start a new swarming in Step Load mode: total locust count of %d, hatch rate of %d, step locust count of %d, step duration of %d "
% (locust_count, hatch_rate, step_locust_count, step_duration))
self.state = STATE_INIT
self.stepload_greenlet = self.greenlet.spawn(self.stepload_worker)
self.stepload_greenlet.link_exception(callback=self.noop)
def stepload_worker(self):
current_num_clients = 0
while self.state == STATE_INIT or self.state == STATE_HATCHING or self.state == STATE_RUNNING:
current_num_clients += self.step_clients_growth
if current_num_clients > int(self.total_clients):
logger.info('Step Load is finished.')
break
self.start_hatching(current_num_clients, self.hatch_rate)
logger.info('Step loading: start hatch job of %d locust.' %
(current_num_clients))
gevent.sleep(self.step_duration)
def stop(self):
# if we are currently hatching locusts we need to kill the hatching greenlet first
if self.hatching_greenlet and not self.hatching_greenlet.ready():
self.hatching_greenlet.kill(block=True)
self.kill_locust_greenlets([g for g in self.locusts])
self.state = STATE_STOPPED
events.locust_stop_hatching.fire()
def quit(self):
self.stop()
self.greenlet.kill(block=True)
def log_exception(self, node_id, msg, formatted_tb):
key = hash(formatted_tb)
row = self.exceptions.setdefault(key, {
"count": 0,
"msg": msg,
"traceback": formatted_tb,
"nodes": set()
})
row["count"] += 1
row["nodes"].add(node_id)
self.exceptions[key] = row
def noop(self, *args, **kwargs):
""" Used to link() greenlets to in order to be compatible with gevent 1.0 """
pass
#print e.ename
print e.evalue
print e.traceback
try:
print "Testing a timeout...",
echo.call('sleep', args=[2.3], timeout=1.1)
print "FAIL, timeout didn't work!"
except RPCTimeoutError, e:
print "OK, got an expected timeout:"
print repr(e)
print
print 'Ignoring result... ',
echo.call('error', ignore=True)
print 'OK\n'
spawn(printer, "[echo] Sleeping for 2 seconds...", echo.sleep, 2.0)
math = GreenRPCClient(green_env='gevent')
# By connecting to two instances, requests are load balanced.
math.connect('tcp://127.0.0.1:5556')
math.connect('tcp://127.0.0.1:5557')
for i in range(5):
for j in range(5):
spawn(printer, "[math] Adding: %s + %s" % (i, j), math.add, i, j)
tasks.join()
def Groupspawn():
greenlet = Group()
greenlet.spawn(green1, "green1", " so green")
greenlet.spawn(green2, "green2")
greenlet.spawn(green3, "green3")
greenlet.join()
class KafkaProductor(object):
def __init__(self, queues):
self.queues = queues
self.kafka = None
self.topic = None
self.sessions = None
self.group = None
self.rdkafka = None
self._init_parse()
def is_rdkafka(self):
try:
from pykafka import rdkafka
self.rdkafka = True
except:
self.rdkafka = False
def _init_parse(self):
self.sessions = [
self.is_connection(data['kafka'], data['topic'], data['queue'])
for data in self.queues
]
self.is_rdkafka()
def encode_topic(self, topic):
if isinstance(topic, bytes) is not True:
topic = str.encode(str(topic))
return topic
def switch(self):
sleep(0)
def is_connection(self, host, topic, queue):
try:
client = KafkaClient(hosts=host)
session = client.topics[self.encode_topic(topic)]
except:
raise KafkaConnectErr(host)
return (session, queue)
def start_a_producer(self, session):
_session, _queue = session[0], session[1]
with _session.get_producer(delivery_reports=True,
use_rdkafka=self.rdkafka) as producer:
count = 0
while True:
try:
msg = _queue.get_nowait()
except:
self.switch()
continue
count += 1
# producer.produce will only access bytes, translate the str to bytes first
producer.produce(msg.encode())
if count % 1000 == 0:
while True:
try:
msg, exc = producer.get_delivery_report(
block=False)
self.switch()
if exc is not None:
print('Failed to deliver msg {}: {}'.format(
msg.partition_key, repr(exc)))
except:
break
@property
def start(self):
self.group = Group()
for session in self.sessions:
self.group.add(spawn(self.start_a_producer, session))
self.group.join()
def _run(self, *args, **kwargs):
group = Group()
for task in self._tasks:
group.start(Greenlet(task))
group.join(raise_error=True)
class LocustRunner(object):
def __init__(self, environment, locust_classes):
environment.runner = self
self.environment = environment
self.locust_classes = locust_classes
self.locusts = Group()
self.greenlet = Group()
self.state = STATE_INIT
self.hatching_greenlet = None
self.stepload_greenlet = None
self.current_cpu_usage = 0
self.cpu_warning_emitted = False
self.greenlet.spawn(self.monitor_cpu)
self.exceptions = {}
self.stats = RequestStats()
# set up event listeners for recording requests
def on_request_success(request_type, name, response_time,
response_length, **kwargs):
self.stats.log_request(request_type, name, response_time,
response_length)
def on_request_failure(request_type, name, response_time,
response_length, exception, **kwargs):
self.stats.log_request(request_type, name, response_time,
response_length)
self.stats.log_error(request_type, name, exception)
self.environment.events.request_success.add_listener(
on_request_success)
self.environment.events.request_failure.add_listener(
on_request_failure)
self.connection_broken = False
# register listener that resets stats when hatching is complete
def on_hatch_complete(user_count):
self.state = STATE_RUNNING
if environment.reset_stats:
logger.info("Resetting stats\n")
self.stats.reset_all()
self.environment.events.hatch_complete.add_listener(on_hatch_complete)
def __del__(self):
# don't leave any stray greenlets if runner is removed
if self.greenlet and len(self.greenlet) > 0:
self.greenlet.kill(block=False)
@property
def errors(self):
return self.stats.errors
@property
def user_count(self):
return len(self.locusts)
def cpu_log_warning(self):
"""Called at the end of the test to repeat the warning & return the status"""
if self.cpu_warning_emitted:
logger.warning(
"Loadgen CPU usage was too high at some point during the test! See https://docs.locust.io/en/stable/running-locust-distributed.html for how to distribute the load over multiple CPU cores or machines"
)
return True
return False
def weight_locusts(self, amount):
"""
Distributes the amount of locusts for each WebLocust-class according to it's weight
returns a list "bucket" with the weighted locusts
"""
bucket = []
weight_sum = sum([locust.weight for locust in self.locust_classes])
residuals = {}
for locust in self.locust_classes:
if self.environment.host is not None:
locust.host = self.environment.host
# create locusts depending on weight
percent = locust.weight / float(weight_sum)
num_locusts = int(round(amount * percent))
bucket.extend([locust for x in range(num_locusts)])
# used to keep track of the amount of rounding was done if we need
# to add/remove some instances from bucket
residuals[locust] = amount * percent - round(amount * percent)
if len(bucket) < amount:
# We got too few locust classes in the bucket, so we need to create a few extra locusts,
# and we do this by iterating over each of the Locust classes - starting with the one
# where the residual from the rounding was the largest - and creating one of each until
# we get the correct amount
for locust in [
l for l, r in sorted(
residuals.items(), key=lambda x: x[1], reverse=True)
][:amount - len(bucket)]:
bucket.append(locust)
elif len(bucket) > amount:
# We've got too many locusts due to rounding errors so we need to remove some
for locust in [
l for l, r in sorted(residuals.items(), key=lambda x: x[1])
][:len(bucket) - amount]:
bucket.remove(locust)
return bucket
def spawn_locusts(self, spawn_count, hatch_rate, wait=False):
bucket = self.weight_locusts(spawn_count)
spawn_count = len(bucket)
if self.state == STATE_INIT or self.state == STATE_STOPPED:
self.state = STATE_HATCHING
existing_count = len(self.locusts)
logger.info(
"Hatching and swarming %i users at the rate %g users/s (%i users already running)..."
% (spawn_count, hatch_rate, existing_count))
occurrence_count = dict([(l.__name__, 0) for l in self.locust_classes])
def hatch():
sleep_time = 1.0 / hatch_rate
while True:
if not bucket:
logger.info(
"All locusts hatched: %s (%i already running)" % (
", ".join([
"%s: %d" % (name, count)
for name, count in occurrence_count.items()
]),
existing_count,
))
self.environment.events.hatch_complete.fire(
user_count=len(self.locusts))
return
locust_class = bucket.pop(random.randint(0, len(bucket) - 1))
occurrence_count[locust_class.__name__] += 1
new_locust = locust_class(self.environment)
new_locust.start(self.locusts)
if len(self.locusts) % 10 == 0:
logger.debug("%i locusts hatched" % len(self.locusts))
if bucket:
gevent.sleep(sleep_time)
hatch()
if wait:
self.locusts.join()
logger.info("All locusts dead\n")
def kill_locusts(self, kill_count):
"""
Kill a kill_count of weighted locusts from the Group() object in self.locusts
"""
bucket = self.weight_locusts(kill_count)
kill_count = len(bucket)
logger.info("Killing %i locusts" % kill_count)
to_kill = []
for g in self.locusts:
for l in bucket:
user = g.args[0]
if l == type(user):
to_kill.append(user)
bucket.remove(l)
break
self.kill_locust_instances(to_kill)
self.environment.events.hatch_complete.fire(user_count=self.user_count)
def kill_locust_instances(self, users):
if self.environment.stop_timeout:
dying = Group()
for user in users:
if not user.stop(self.locusts, force=False):
# Locust.stop() returns False if the greenlet was not killed, so we'll need
# to add it's greenlet to our dying Group so we can wait for it to finish it's task
dying.add(user._greenlet)
if not dying.join(timeout=self.environment.stop_timeout):
logger.info(
"Not all locusts finished their tasks & terminated in %s seconds. Killing them..."
% self.environment.stop_timeout)
dying.kill(block=True)
else:
for user in users:
user.stop(self.locusts, force=True)
def monitor_cpu(self):
process = psutil.Process()
while True:
self.current_cpu_usage = process.cpu_percent()
if self.current_cpu_usage > 90 and not self.cpu_warning_emitted:
logging.warning(
"Loadgen CPU usage above 90%! This may constrain your throughput and may even give inconsistent response time measurements! See https://docs.locust.io/en/stable/running-locust-distributed.html for how to distribute the load over multiple CPU cores or machines"
)
self.cpu_warning_emitted = True
gevent.sleep(CPU_MONITOR_INTERVAL)
def start(self, locust_count, hatch_rate, wait=False):
if self.state != STATE_RUNNING and self.state != STATE_HATCHING:
self.stats.clear_all()
self.exceptions = {}
self.cpu_warning_emitted = False
self.worker_cpu_warning_emitted = False
# Dynamically changing the locust count
if self.state != STATE_INIT and self.state != STATE_STOPPED:
self.state = STATE_HATCHING
if self.user_count > locust_count:
# Kill some locusts
kill_count = self.user_count - locust_count
self.kill_locusts(kill_count)
elif self.user_count < locust_count:
# Spawn some locusts
spawn_count = locust_count - self.user_count
self.spawn_locusts(spawn_count=spawn_count,
hatch_rate=hatch_rate)
else:
self.environment.events.hatch_complete.fire(
user_count=self.user_count)
else:
self.hatch_rate = hatch_rate
self.spawn_locusts(locust_count, hatch_rate=hatch_rate, wait=wait)
def start_stepload(self, locust_count, hatch_rate, step_locust_count,
step_duration):
if locust_count < step_locust_count:
logger.error(
"Invalid parameters: total locust count of %d is smaller than step locust count of %d"
% (locust_count, step_locust_count))
return
self.total_clients = locust_count
if self.stepload_greenlet:
logger.info(
"There is an ongoing swarming in Step Load mode, will stop it now."
)
self.stepload_greenlet.kill()
logger.info(
"Start a new swarming in Step Load mode: total locust count of %d, hatch rate of %d, step locust count of %d, step duration of %d "
% (locust_count, hatch_rate, step_locust_count, step_duration))
self.state = STATE_INIT
self.stepload_greenlet = self.greenlet.spawn(self.stepload_worker,
hatch_rate,
step_locust_count,
step_duration)
def stepload_worker(self, hatch_rate, step_clients_growth, step_duration):
current_num_clients = 0
while self.state == STATE_INIT or self.state == STATE_HATCHING or self.state == STATE_RUNNING:
current_num_clients += step_clients_growth
if current_num_clients > int(self.total_clients):
logger.info('Step Load is finished.')
break
self.start(current_num_clients, hatch_rate)
logger.info('Step loading: start hatch job of %d locust.' %
(current_num_clients))
gevent.sleep(step_duration)
def stop(self):
self.state = STATE_CLEANUP
# if we are currently hatching locusts we need to kill the hatching greenlet first
if self.hatching_greenlet and not self.hatching_greenlet.ready():
self.hatching_greenlet.kill(block=True)
self.kill_locust_instances([g.args[0] for g in self.locusts])
self.state = STATE_STOPPED
self.cpu_log_warning()
def quit(self):
self.stop()
self.greenlet.kill(block=True)
def log_exception(self, node_id, msg, formatted_tb):
key = hash(formatted_tb)
row = self.exceptions.setdefault(key, {
"count": 0,
"msg": msg,
"traceback": formatted_tb,
"nodes": set()
})
row["count"] += 1
row["nodes"].add(node_id)
self.exceptions[key] = row
def main(workers):
queue = SubmitQueue()
worker_group = Group()
for _ in range(workers):
worker_group.start(SubmitWorker(queue))
worker_group.join()
