def test_fiveSecondFailure(self):
"""
If more than 1% of requests take longer than 5 seconds,
L{ReportStatistics.failures} returns a list containing a string
describing that.
"""
logger = ReportStatistics()
for _ignore in range(98):
logger.observe(
dict(type='response',
method='GET',
success=True,
duration=2.5,
user='******',
client_type="test",
client_id="1234"))
logger.observe(
dict(type='response',
method='GET',
success=True,
duration=5.5,
user='******',
client_type="test",
client_id="1234"))
self.assertEqual(
["Greater than 1% GET exceeded 5 second response time"],
logger.failures())
def test_noFailures(self):
"""
If fewer than 1% of requests fail, fewer than 1% of requests take 5
seconds or more, and fewer than 5% of requests take 3 seconds or more,
L{ReportStatistics.failures} returns an empty list.
"""
logger = ReportStatistics()
logger.observe(dict(
type='response', method='GET', success=True,
duration=2.5, user='******', client_type="test", client_id="1234"))
self.assertEqual([], logger.failures())
def test_simFailures(self):
"""
L{ReportStatistics.countSimFailures} returns the number of clients observed to
have caused an error in the simulation.
"""
logger = ReportStatistics()
clients = ['c01', 'c02', 'c03']
for client in clients:
logger.observe(dict(
type='sim-failure', reason="testing %s" % (client,)))
self.assertEqual(len(clients), logger.countSimFailures())
def test_simFailures(self):
"""
L{ReportStatistics.countSimFailures} returns the number of clients observed to
have caused an error in the simulation.
"""
logger = ReportStatistics()
clients = ['c01', 'c02', 'c03']
for client in clients:
logger.observe(
dict(type='sim-failure', reason="testing %s" % (client, )))
self.assertEqual(len(clients), logger.countSimFailures())
def test_noFailures(self):
"""
If fewer than 1% of requests fail, fewer than 1% of requests take 5
seconds or more, and fewer than 5% of requests take 3 seconds or more,
L{ReportStatistics.failures} returns an empty list.
"""
logger = ReportStatistics()
logger.observe(dict(
type='response', method='GET', success=True,
duration=2.5, user='******', client_type="test", client_id="1234"))
self.assertEqual([], logger.failures())
def test_countClients(self):
"""
L{ReportStatistics.countClients} returns the number of clients observed to
have acted in the simulation.
"""
logger = ReportStatistics()
clients = ['c01', 'c02', 'c03']
for client in clients:
logger.observe(dict(
type='response', method='GET', success=True,
duration=1.23, user="******", client_type="test", client_id=client))
self.assertEqual(len(clients), logger.countClients())
def test_countUsers(self):
"""
L{ReportStatistics.countUsers} returns the number of users observed to
have acted in the simulation.
"""
logger = ReportStatistics()
users = ['user01', 'user02', 'user03']
for user in users:
logger.observe(dict(
type='response', method='GET', success=True,
duration=1.23, user=user, client_type="test", client_id="1234"))
self.assertEqual(len(users), logger.countUsers())
def test_countClients(self):
"""
L{ReportStatistics.countClients} returns the number of clients observed to
have acted in the simulation.
"""
logger = ReportStatistics()
clients = ['c01', 'c02', 'c03']
for client in clients:
logger.observe(dict(
type='response', method='GET', success=True,
duration=1.23, user="******", client_type="test", client_id=client))
self.assertEqual(len(clients), logger.countClients())
def test_countUsers(self):
"""
L{ReportStatistics.countUsers} returns the number of users observed to
have acted in the simulation.
"""
logger = ReportStatistics()
users = ['user01', 'user02', 'user03']
for user in users:
logger.observe(dict(
type='response', method='GET', success=True,
duration=1.23, user=user, client_type="test", client_id="1234"))
self.assertEqual(len(users), logger.countUsers())
def test_methodsCountedSeparately(self):
"""
The counts for one method do not affect the results of another method.
"""
logger = ReportStatistics()
for _ignore in range(99):
logger.observe(dict(
type='response', method='GET', success=True,
duration=2.5, user='******', client_type="test", client_id="1234"
))
logger.observe(dict(
type='response', method='POST', success=True,
duration=2.5, user='******', client_type="test", client_id="1234"
))
logger.observe(dict(
type='response', method='GET', success=False,
duration=2.5, user='******', client_type="test", client_id="1234"
))
logger.observe(dict(
type='response', method='POST', success=False,
duration=2.5, user='******', client_type="test", client_id="1234"
))
self.assertEqual([], logger.failures())
def test_methodsCountedSeparately(self):
"""
The counts for one method do not affect the results of another method.
"""
logger = ReportStatistics()
for _ignore in range(99):
logger.observe(dict(
type='response', method='GET', success=True,
duration=2.5, user='******', client_type="test", client_id="1234"
))
logger.observe(dict(
type='response', method='POST', success=True,
duration=2.5, user='******', client_type="test", client_id="1234"
))
logger.observe(dict(
type='response', method='GET', success=False,
duration=2.5, user='******', client_type="test", client_id="1234"
))
logger.observe(dict(
type='response', method='POST', success=False,
duration=2.5, user='******', client_type="test", client_id="1234"
))
self.assertEqual([], logger.failures())
def test_requestFailures(self):
"""
If more than 1% of requests fail, L{ReportStatistics.failures} returns a
list containing a string describing this.
"""
logger = ReportStatistics()
for _ignore in range(98):
logger.observe(dict(
type='response', method='GET', success=True,
duration=2.5, user='******', client_type="test", client_id="1234"))
logger.observe(dict(
type='response', method='GET', success=False,
duration=2.5, user='******', client_type="test", client_id="1234"))
self.assertEqual(
["Greater than 1% GET failed"],
logger.failures())
def test_requestFailures(self):
"""
If more than 1% of requests fail, L{ReportStatistics.failures} returns a
list containing a string describing this.
"""
logger = ReportStatistics()
for _ignore in range(98):
logger.observe(dict(
type='response', method='GET', success=True,
duration=2.5, user='******', client_type="test", client_id="1234"))
logger.observe(dict(
type='response', method='GET', success=False,
duration=2.5, user='******', client_type="test", client_id="1234"))
self.assertEqual(
["Greater than 1% GET failed"],
logger.failures())
def test_fiveSecondFailure(self):
"""
If more than 1% of requests take longer than 5 seconds,
L{ReportStatistics.failures} returns a list containing a string
describing that.
"""
logger = ReportStatistics()
for _ignore in range(98):
logger.observe(dict(
type='response', method='GET', success=True,
duration=2.5, user='******', client_type="test", client_id="1234"))
logger.observe(dict(
type='response', method='GET', success=True,
duration=5.5, user='******', client_type="test", client_id="1234"))
self.assertEqual(
["Greater than 1% GET exceeded 5 second response time"],
logger.failures())
def test_bucketRequest(self):
"""
PUT(xxx-huge/large/medium/small} have different thresholds. Test that requests straddling
each of those are correctly determined to be failures or not.
"""
_thresholds = {
"requests": {
"limits": [0.1, 0.5, 1.0, 3.0, 5.0, 10.0, 30.0],
"thresholds": {
"default": [100.0, 100.0, 100.0, 5.0, 1.0, 0.5, 0.0],
"PUT{organizer-small}": [100.0, 50.0, 25.0, 5.0, 1.0, 0.5, 0.0],
"PUT{organizer-medium}": [100.0, 100.0, 50.0, 25.0, 5.0, 1.0, 0.5],
"PUT{organizer-large}": [100.0, 100.0, 100.0, 50.0, 25.0, 5.0, 1.0],
"PUT{organizer-huge}": [100.0, 100.0, 100.0, 100.0, 100.0, 50.0, 25.0],
}
}
}
# -small below threshold
logger = ReportStatistics(thresholds=_thresholds)
logger.observe(dict(
type='response', method='PUT{organizer-small}', success=True,
duration=0.2, user='******', client_type="test", client_id="1234"
))
logger.observe(dict(
type='response', method='PUT{organizer-small}', success=True,
duration=0.2, user='******', client_type="test", client_id="1234"
))
logger.observe(dict(
type='response', method='PUT{organizer-small}', success=True,
duration=0.2, user='******', client_type="test", client_id="1234"
))
logger.observe(dict(
type='response', method='PUT{organizer-small}', success=True,
duration=0.2, user='******', client_type="test", client_id="1234"
))
self.assertEqual([], logger.failures())
# -small above 0.5 threshold
logger = ReportStatistics(thresholds=_thresholds)
logger.observe(dict(
type='response', method='PUT{organizer-small}', success=True,
duration=0.2, user='******', client_type="test", client_id="1234"
))
logger.observe(dict(
type='response', method='PUT{organizer-small}', success=True,
duration=0.6, user='******', client_type="test", client_id="1234"
))
logger.observe(dict(
type='response', method='PUT{organizer-small}', success=True,
duration=0.6, user='******', client_type="test", client_id="1234"
))
logger.observe(dict(
type='response', method='PUT{organizer-small}', success=True,
duration=0.6, user='******', client_type="test", client_id="1234"
))
self.assertEqual(
["Greater than 50% PUT{organizer-small} exceeded 0.5 second response time"],
logger.failures()
)
# -medium below 0.5 threshold
logger = ReportStatistics(thresholds=_thresholds)
logger.observe(dict(
type='response', method='PUT{organizer-medium}', success=True,
duration=0.2, user='******', client_type="test", client_id="1234"
))
logger.observe(dict(
type='response', method='PUT{organizer-medium}', success=True,
duration=0.6, user='******', client_type="test", client_id="1234"
))
logger.observe(dict(
type='response', method='PUT{organizer-medium}', success=True,
duration=0.6, user='******', client_type="test", client_id="1234"
))
logger.observe(dict(
type='response', method='PUT{organizer-medium}', success=True,
duration=0.6, user='******', client_type="test", client_id="1234"
))
self.assertEqual(
[],
logger.failures()
)
# -medium above 1.0 threshold
logger = ReportStatistics(thresholds=_thresholds)
logger.observe(dict(
type='response', method='PUT{organizer-medium}', success=True,
duration=0.2, user='******', client_type="test", client_id="1234"
))
logger.observe(dict(
type='response', method='PUT{organizer-medium}', success=True,
duration=1.6, user='******', client_type="test", client_id="1234"
))
logger.observe(dict(
type='response', method='PUT{organizer-medium}', success=True,
duration=1.6, user='******', client_type="test", client_id="1234"
))
logger.observe(dict(
type='response', method='PUT{organizer-medium}', success=True,
duration=1.6, user='******', client_type="test", client_id="1234"
))
self.assertEqual(
["Greater than 50% PUT{organizer-medium} exceeded 1 second response time"],
logger.failures()
)
# -large below 1.0 threshold
logger = ReportStatistics(thresholds=_thresholds)
logger.observe(dict(
type='response', method='PUT{organizer-large}', success=True,
duration=0.2, user='******', client_type="test", client_id="1234"
))
logger.observe(dict(
type='response', method='PUT{organizer-large}', success=True,
duration=1.6, user='******', client_type="test", client_id="1234"
))
logger.observe(dict(
type='response', method='PUT{organizer-large}', success=True,
duration=1.6, user='******', client_type="test", client_id="1234"
))
logger.observe(dict(
type='response', method='PUT{organizer-large}', success=True,
duration=1.6, user='******', client_type="test", client_id="1234"
))
self.assertEqual(
[],
logger.failures()
)
# -large above 3.0 threshold
logger = ReportStatistics(thresholds=_thresholds)
logger.observe(dict(
type='response', method='PUT{organizer-large}', success=True,
duration=0.2, user='******', client_type="test", client_id="1234"
))
logger.observe(dict(
type='response', method='PUT{organizer-large}', success=True,
duration=3.6, user='******', client_type="test", client_id="1234"
))
logger.observe(dict(
type='response', method='PUT{organizer-large}', success=True,
duration=3.6, user='******', client_type="test", client_id="1234"
))
logger.observe(dict(
type='response', method='PUT{organizer-large}', success=True,
duration=3.6, user='******', client_type="test", client_id="1234"
))
self.assertEqual(
["Greater than 50% PUT{organizer-large} exceeded 3 second response time"],
logger.failures()
)
# -huge below 10.0 threshold
logger = ReportStatistics(thresholds=_thresholds)
logger.observe(dict(
type='response', method='PUT{organizer-huge}', success=True,
duration=12.0, user='******', client_type="test", client_id="1234"
))
logger.observe(dict(
type='response', method='PUT{organizer-huge}', success=True,
duration=8, user='******', client_type="test", client_id="1234"
))
logger.observe(dict(
type='response', method='PUT{organizer-huge}', success=True,
duration=11.0, user='******', client_type="test", client_id="1234"
))
logger.observe(dict(
type='response', method='PUT{organizer-huge}', success=True,
duration=9.0, user='******', client_type="test", client_id="1234"
))
self.assertEqual(
[],
logger.failures()
)
# -huge above 10.0 threshold
logger = ReportStatistics(thresholds=_thresholds)
logger.observe(dict(
type='response', method='PUT{organizer-huge}', success=True,
duration=12.0, user='******', client_type="test", client_id="1234"
))
logger.observe(dict(
type='response', method='PUT{organizer-huge}', success=True,
duration=9.0, user='******', client_type="test", client_id="1234"
))
logger.observe(dict(
type='response', method='PUT{organizer-huge}', success=True,
duration=12.0, user='******', client_type="test", client_id="1234"
))
logger.observe(dict(
type='response', method='PUT{organizer-huge}', success=True,
duration=42.42, user='******', client_type="test", client_id="1234"
))
self.assertEqual(
["Greater than 50% PUT{organizer-huge} exceeded 10 second response time"],
logger.failures()
)
def test_bucketRequest(self):
"""
PUT(xxx-huge/large/medium/small} have different thresholds. Test that requests straddling
each of those are correctly determined to be failures or not.
"""
_thresholds = {
"requests":{
"limits": [ 0.1, 0.5, 1.0, 3.0, 5.0, 10.0, 30.0],
"thresholds":{
"default": [ 100.0, 100.0, 100.0, 5.0, 1.0, 0.5, 0.0],
"PUT{organizer-small}": [ 100.0, 50.0, 25.0, 5.0, 1.0, 0.5, 0.0],
"PUT{organizer-medium}":[ 100.0, 100.0, 50.0, 25.0, 5.0, 1.0, 0.5],
"PUT{organizer-large}": [ 100.0, 100.0, 100.0, 50.0, 25.0, 5.0, 1.0],
"PUT{organizer-huge}": [ 100.0, 100.0, 100.0, 100.0, 100.0, 50.0, 25.0],
}
}
}
# -small below threshold
logger = ReportStatistics(thresholds=_thresholds)
logger.observe(dict(
type='response', method='PUT{organizer-small}', success=True,
duration=0.2, user='******', client_type="test", client_id="1234"))
logger.observe(dict(
type='response', method='PUT{organizer-small}', success=True,
duration=0.2, user='******', client_type="test", client_id="1234"))
logger.observe(dict(
type='response', method='PUT{organizer-small}', success=True,
duration=0.2, user='******', client_type="test", client_id="1234"))
logger.observe(dict(
type='response', method='PUT{organizer-small}', success=True,
duration=0.2, user='******', client_type="test", client_id="1234"))
self.assertEqual([], logger.failures())
# -small above 0.5 threshold
logger = ReportStatistics(thresholds=_thresholds)
logger.observe(dict(
type='response', method='PUT{organizer-small}', success=True,
duration=0.2, user='******', client_type="test", client_id="1234"))
logger.observe(dict(
type='response', method='PUT{organizer-small}', success=True,
duration=0.6, user='******', client_type="test", client_id="1234"))
logger.observe(dict(
type='response', method='PUT{organizer-small}', success=True,
duration=0.6, user='******', client_type="test", client_id="1234"))
logger.observe(dict(
type='response', method='PUT{organizer-small}', success=True,
duration=0.6, user='******', client_type="test", client_id="1234"))
self.assertEqual(
["Greater than 50% PUT{organizer-small} exceeded 0.5 second response time"],
logger.failures()
)
# -medium below 0.5 threshold
logger = ReportStatistics(thresholds=_thresholds)
logger.observe(dict(
type='response', method='PUT{organizer-medium}', success=True,
duration=0.2, user='******', client_type="test", client_id="1234"))
logger.observe(dict(
type='response', method='PUT{organizer-medium}', success=True,
duration=0.6, user='******', client_type="test", client_id="1234"))
logger.observe(dict(
type='response', method='PUT{organizer-medium}', success=True,
duration=0.6, user='******', client_type="test", client_id="1234"))
logger.observe(dict(
type='response', method='PUT{organizer-medium}', success=True,
duration=0.6, user='******', client_type="test", client_id="1234"))
self.assertEqual(
[],
logger.failures()
)
# -medium above 1.0 threshold
logger = ReportStatistics(thresholds=_thresholds)
logger.observe(dict(
type='response', method='PUT{organizer-medium}', success=True,
duration=0.2, user='******', client_type="test", client_id="1234"))
logger.observe(dict(
type='response', method='PUT{organizer-medium}', success=True,
duration=1.6, user='******', client_type="test", client_id="1234"))
logger.observe(dict(
type='response', method='PUT{organizer-medium}', success=True,
duration=1.6, user='******', client_type="test", client_id="1234"))
logger.observe(dict(
type='response', method='PUT{organizer-medium}', success=True,
duration=1.6, user='******', client_type="test", client_id="1234"))
self.assertEqual(
["Greater than 50% PUT{organizer-medium} exceeded 1 second response time"],
logger.failures()
)
# -large below 1.0 threshold
logger = ReportStatistics(thresholds=_thresholds)
logger.observe(dict(
type='response', method='PUT{organizer-large}', success=True,
duration=0.2, user='******', client_type="test", client_id="1234"))
logger.observe(dict(
type='response', method='PUT{organizer-large}', success=True,
duration=1.6, user='******', client_type="test", client_id="1234"))
logger.observe(dict(
type='response', method='PUT{organizer-large}', success=True,
duration=1.6, user='******', client_type="test", client_id="1234"))
logger.observe(dict(
type='response', method='PUT{organizer-large}', success=True,
duration=1.6, user='******', client_type="test", client_id="1234"))
self.assertEqual(
[],
logger.failures()
)
# -large above 3.0 threshold
logger = ReportStatistics(thresholds=_thresholds)
logger.observe(dict(
type='response', method='PUT{organizer-large}', success=True,
duration=0.2, user='******', client_type="test", client_id="1234"))
logger.observe(dict(
type='response', method='PUT{organizer-large}', success=True,
duration=3.6, user='******', client_type="test", client_id="1234"))
logger.observe(dict(
type='response', method='PUT{organizer-large}', success=True,
duration=3.6, user='******', client_type="test", client_id="1234"))
logger.observe(dict(
type='response', method='PUT{organizer-large}', success=True,
duration=3.6, user='******', client_type="test", client_id="1234"))
self.assertEqual(
["Greater than 50% PUT{organizer-large} exceeded 3 second response time"],
logger.failures()
)
# -huge below 10.0 threshold
logger = ReportStatistics(thresholds=_thresholds)
logger.observe(dict(
type='response', method='PUT{organizer-huge}', success=True,
duration=12.0, user='******', client_type="test", client_id="1234"))
logger.observe(dict(
type='response', method='PUT{organizer-huge}', success=True,
duration=8, user='******', client_type="test", client_id="1234"))
logger.observe(dict(
type='response', method='PUT{organizer-huge}', success=True,
duration=11.0, user='******', client_type="test", client_id="1234"))
logger.observe(dict(
type='response', method='PUT{organizer-huge}', success=True,
duration=9.0, user='******', client_type="test", client_id="1234"))
self.assertEqual(
[],
logger.failures()
)
# -huge above 10.0 threshold
logger = ReportStatistics(thresholds=_thresholds)
logger.observe(dict(
type='response', method='PUT{organizer-huge}', success=True,
duration=12.0, user='******', client_type="test", client_id="1234"))
logger.observe(dict(
type='response', method='PUT{organizer-huge}', success=True,
duration=9.0, user='******', client_type="test", client_id="1234"))
logger.observe(dict(
type='response', method='PUT{organizer-huge}', success=True,
duration=12.0, user='******', client_type="test", client_id="1234"))
logger.observe(dict(
type='response', method='PUT{organizer-huge}', success=True,
duration=42.42, user='******', client_type="test", client_id="1234"))
self.assertEqual(
["Greater than 50% PUT{organizer-huge} exceeded 10 second response time"],
logger.failures()
)