def run_testsets(testsets):
""" Execute a set of tests, using given TestSet list input """
group_results = dict() # results, by group
group_failure_counts = dict()
total_failures = 0
myinteractive = False
curl_handle = pycurl.Curl()
for testset in testsets:
mytests = testset.tests
myconfig = testset.config
mybenchmarks = testset.benchmarks
context = Context()
# Bind variables & add generators if pertinent
if myconfig.variable_binds:
context.bind_variables(myconfig.variable_binds)
if myconfig.generators:
for key, value in myconfig.generators.items():
context.add_generator(key, value)
# Make sure we actually have tests to execute
if not mytests and not mybenchmarks:
# no tests in this test set, probably just imports.. skip to next
# test set
break
myinteractive = True if myinteractive or myconfig.interactive else False
# Run tests, collecting statistics as needed
for test in mytests:
# Initialize the dictionaries to store test fail counts and results
if test.group not in group_results:
group_results[test.group] = list()
group_failure_counts[test.group] = 0
result = run_test(test, test_config=myconfig, context=context, curl_handle=curl_handle)
result.body = None # Remove the body, save some memory!
if not result.passed: # Print failure, increase failure counts for that test group
# Use result test URL to allow for templating
logger.error('Test Failed: ' + test.name + " URL=" + result.test.url +
" Group=" + test.group + " HTTP Status Code: " + str(result.response_code))
# Print test failure reasons
if result.failures:
for failure in result.failures:
log_failure(failure, context=context,
test_config=myconfig)
# Increment test failure counts for that group (adding an entry
# if not present)
failures = group_failure_counts[test.group]
failures = failures + 1
group_failure_counts[test.group] = failures
else: # Test passed, print results
logger.info('Test Succeeded: ' + test.name +
" URL=" + test.url + " Group=" + test.group)
# Add results for this test group to the resultset
group_results[test.group].append(result)
# handle stop_on_failure flag
if not result.passed and test.stop_on_failure is not None and test.stop_on_failure:
print(
'STOP ON FAILURE! stopping test set execution, continuing with other test sets')
break
for benchmark in mybenchmarks: # Run benchmarks, analyze, write
if not benchmark.metrics:
logger.debug('Skipping benchmark, no metrics to collect')
continue
logger.info("Benchmark Starting: " + benchmark.name +
" Group: " + benchmark.group)
benchmark_result = run_benchmark(
benchmark, myconfig, context=context)
print(benchmark_result)
logger.info("Benchmark Done: " + benchmark.name +
" Group: " + benchmark.group)
if benchmark.output_file: # Write file
logger.debug(
'Writing benchmark to file in format: ' + benchmark.output_format)
write_method = OUTPUT_METHODS[benchmark.output_format]
my_file = open(benchmark.output_file, 'w') # Overwrites file
logger.debug("Benchmark writing to file: " +
benchmark.output_file)
write_method(my_file, benchmark_result,
benchmark, test_config=myconfig)
my_file.close()
if myinteractive:
# a break for when interactive bits are complete, before summary data
print("===================================")
# Print summary results
for group in sorted(group_results.keys()):
test_count = len(group_results[group])
failures = group_failure_counts[group]
total_failures = total_failures + failures
passfail = {True: u'SUCCEEDED: ', False: u'FAILED: '}
output_string = "Test Group {0} {1}: {2}/{3} Tests Passed!".format(group, passfail[failures == 0], str(test_count - failures), str(test_count))
if myconfig.skip_term_colors:
print(output_string)
else:
if failures > 0:
print('\033[91m' + output_string + '\033[0m')
else:
print('\033[92m' + output_string + '\033[0m')
return total_failures
def run_benchmark(benchmark, test_config=TestConfig(), context=None, *args, **kwargs):
""" Perform a benchmark, (re)using a given, configured CURL call to do so
The actual analysis of metrics is performed separately, to allow for testing
"""
# Context handling
my_context = context
if my_context is None:
my_context = Context()
warmup_runs = benchmark.warmup_runs
benchmark_runs = benchmark.benchmark_runs
message = '' # Message is name of benchmark... print it?
if (benchmark_runs <= 0):
raise Exception(
"Invalid number of benchmark runs, must be > 0 :" + benchmark_runs)
result = TestResponse()
# TODO create and use a curl-returning configuration function
# TODO create and use a post-benchmark cleanup function
# They should use is_dynamic/is_context_modifier to determine if they need to
# worry about context and re-reading/retemplating and only do it if needed
# - Also, they will need to be smart enough to handle extraction functions
# For performance reasons, we don't want to re-run templating/extraction if
# we do not need to, and do not want to save request bodies.
# Initialize variables to store output
output = BenchmarkResult()
output.name = benchmark.name
output.group = benchmark.group
metricnames = list(benchmark.metrics)
# Metric variable for curl, to avoid hash lookup for every metric name
metricvalues = [METRICS[name] for name in metricnames]
# Initialize arrays to store results for each metric
results = [list() for x in xrange(0, len(metricnames))]
curl = pycurl.Curl()
# Benchmark warm-up to allow for caching, JIT compiling, on client
logger.info('Warmup: ' + message + ' started')
for x in xrange(0, warmup_runs):
benchmark.update_context_before(my_context)
templated = benchmark.realize(my_context)
curl = templated.configure_curl(
timeout=test_config.timeout, context=my_context, curl_handle=curl)
# Do not store actual response body at all.
curl.setopt(pycurl.WRITEFUNCTION, lambda x: None)
curl.perform()
logger.info('Warmup: ' + message + ' finished')
logger.info('Benchmark: ' + message + ' starting')
for x in xrange(0, benchmark_runs): # Run the actual benchmarks
# Setup benchmark
benchmark.update_context_before(my_context)
templated = benchmark.realize(my_context)
curl = templated.configure_curl(
timeout=test_config.timeout, context=my_context, curl_handle=curl)
# Do not store actual response body at all.
curl.setopt(pycurl.WRITEFUNCTION, lambda x: None)
try: # Run the curl call, if it errors, then add to failure counts for benchmark
curl.perform()
except Exception:
output.failures = output.failures + 1
curl.close()
curl = pycurl.Curl()
continue # Skip metrics collection
# Get all metrics values for this run, and store to metric lists
for i in xrange(0, len(metricnames)):
results[i].append(curl.getinfo(metricvalues[i]))
logger.info('Benchmark: ' + message + ' ending')
temp_results = dict()
for i in xrange(0, len(metricnames)):
temp_results[metricnames[i]] = results[i]
output.results = temp_results
return analyze_benchmark_results(output, benchmark)
def run_test(mytest, test_config=TestConfig(), context=None, curl_handle=None, *args, **kwargs):
""" Put together test pieces: configure & run actual test, return results """
# Initialize a context if not supplied
my_context = context
if my_context is None:
my_context = Context()
mytest.update_context_before(my_context)
templated_test = mytest.realize(my_context)
curl = templated_test.configure_curl(
timeout=test_config.timeout, context=my_context, curl_handle=curl_handle)
result = TestResponse()
result.test = templated_test
# reset the body, it holds values from previous runs otherwise
headers = MyIO()
body = MyIO()
curl.setopt(pycurl.WRITEFUNCTION, body.write)
curl.setopt(pycurl.HEADERFUNCTION, headers.write)
if test_config.verbose:
curl.setopt(pycurl.VERBOSE, True)
if test_config.ssl_insecure:
curl.setopt(pycurl.SSL_VERIFYPEER, 0)
curl.setopt(pycurl.SSL_VERIFYHOST, 0)
result.passed = None
if test_config.interactive:
print("===================================")
print("%s" % mytest.name)
print("-----------------------------------")
print("REQUEST:")
print("%s %s" % (templated_test.method, templated_test.url))
print("HEADERS:")
print("%s" % (templated_test.headers))
if mytest.body is not None:
print("\n%s" % templated_test.body)
raw_input("Press ENTER when ready (%d): " % (mytest.delay))
if mytest.delay > 0:
print("Delaying for %ds" % mytest.delay)
time.sleep(mytest.delay)
try:
curl.perform() # Run the actual call
except Exception as e:
# Curl exception occurred (network error), do not pass go, do not
# collect $200
trace = traceback.format_exc()
result.failures.append(Failure(message="Curl Exception: {0}".format(
e), details=trace, failure_type=validators.FAILURE_CURL_EXCEPTION))
result.passed = False
curl.close()
return result
# Retrieve values
result.body = body.getvalue()
body.close()
result.response_headers = text_type(headers.getvalue(), HEADER_ENCODING) # Per RFC 2616
headers.close()
response_code = curl.getinfo(pycurl.RESPONSE_CODE)
result.response_code = response_code
logger.debug("Initial Test Result, based on expected response code: " +
str(response_code in mytest.expected_status))
if response_code in mytest.expected_status:
result.passed = True
else:
# Invalid response code
result.passed = False
failure_message = "Invalid HTTP response code: response code {0} not in expected codes [{1}]".format(
response_code, mytest.expected_status)
result.failures.append(Failure(
message=failure_message, details=None, failure_type=validators.FAILURE_INVALID_RESPONSE))
# Parse HTTP headers
try:
result.response_headers = parse_headers(result.response_headers)
except Exception as e:
trace = traceback.format_exc()
result.failures.append(Failure(message="Header parsing exception: {0}".format(
e), details=trace, failure_type=validators.FAILURE_TEST_EXCEPTION))
result.passed = False
curl.close()
return result
# print str(test_config.print_bodies) + ',' + str(not result.passed) + ' ,
# ' + str(test_config.print_bodies or not result.passed)
head = result.response_headers
# execute validator on body
if result.passed is True:
body = result.body
if mytest.validators is not None and isinstance(mytest.validators, list):
logger.debug("executing this many validators: " +
str(len(mytest.validators)))
failures = result.failures
for validator in mytest.validators:
validate_result = validator.validate(
body=body, headers=head, context=my_context)
if not validate_result:
result.passed = False
# Proxy for checking if it is a Failure object, because of
# import issues with isinstance there
if hasattr(validate_result, 'details'):
failures.append(validate_result)
# TODO add printing of validation for interactive mode
else:
logger.debug("no validators found")
# Only do context updates if test was successful
mytest.update_context_after(result.body, head, my_context)
# Print response body if override is set to print all *OR* if test failed
# (to capture maybe a stack trace)
if test_config.print_bodies or not result.passed:
if test_config.interactive:
print("RESPONSE:")
print(result.body.decode(ESCAPE_DECODING))
if test_config.print_headers or not result.passed:
if test_config.interactive:
print("RESPONSE HEADERS:")
print(result.response_headers)
# TODO add string escape on body output
logger.debug(result)
return result
from pyresttest.generators import factory_generate_ids
import cProfile
test = Benchmark()
test.warmup_runs = 0
test.benchmark_runs = 1000
test.raw_metrics = set()
test.metrics = {'total_time'}
test.aggregated_metrics = {'total_time': ['total', 'mean']}
# Basic get test
test.url = 'http://localhost:8000/api/person/'
test.name = 'Basic GET'
print 'Basic GET test'
#cProfile.run('resttest.run_benchmark(test)', sort='cumtime')
# Test a generator PUT method
test.method = 'PUT'
test.set_url('http://localhost:8000/api/person/$id/', isTemplate=True)
test.headers = {'Content-Type': 'application/json'}
handler = ContentHandler()
handler.setup(
'{"first_name": "Gaius","id": "$id","last_name": "Baltar","login": "******"}',
is_template_content=True)
test.body = handler
context = Context()
context.add_generator('gen', factory_generate_ids(starting_id=10)())
test.generator_binds = {'id': 'gen'}
print 'Running templated PUT test'
cProfile.run('resttest.run_benchmark(test, context=context)', sort='cumtime')
import cProfile
test = Benchmark()
test.warmup_runs = 0
test.benchmark_runs = 1000
test.raw_metrics = set()
test.metrics = {'total_time'}
test.aggregated_metrics = {'total_time': ['total', 'mean']}
# Basic get test
test.url = 'http://localhost:8000/api/person/'
test.name = 'Basic GET'
print 'Basic GET test'
#cProfile.run('resttest.run_benchmark(test)', sort='cumtime')
# Test a generator PUT method
test.method = 'PUT'
test.set_url('http://localhost:8000/api/person/$id/', isTemplate=True)
test.headers = {'Content-Type': 'application/json'}
handler = ContentHandler()
handler.setup('{"first_name": "Gaius","id": "$id","last_name": "Baltar","login": "******"}',
is_template_content=True)
test.body = handler
context = Context()
context.add_generator('gen', factory_generate_ids(starting_id=10)())
test.generator_binds = {'id': 'gen'}
print 'Running templated PUT test'
cProfile.run('test.execute_macro(context=context)', sort='cumtime')
def run_test(mytest,
test_config=TestConfig(),
context=None,
curl_handle=None,
*args,
**kwargs):
""" Put together test pieces: configure & run actual test, return results """
# Initialize a context if not supplied
my_context = context
if my_context is None:
my_context = Context()
mytest.update_context_before(my_context)
templated_test = mytest.realize(my_context)
result = TestResponse()
result.test = templated_test
session = requests.Session()
# generate and attach signature to header
req = templated_test.configure_request(timeout=test_config.timeout,
context=my_context,
curl_handle=curl_handle)
if test_config.verbose:
session.verbose = True
if test_config.ssl_insecure:
session.verify = False
headers = MyIO()
body = MyIO()
prepped = req.prepare()
prepped = signer.request_signer(prepped, TestConfig.key)
result.passed = None
if test_config.interactive:
LOGGER.debug("===================================")
LOGGER.debug("%s" % mytest.name)
LOGGER.debug("-----------------------------------")
LOGGER.debug("REQUEST:")
LOGGER.debug("%s %s" % (templated_test.method, templated_test.url))
LOGGER.debug("HEADERS:")
LOGGER.debug("%s" % prepped.headers)
if mytest.body is not None:
LOGGER.debug("\n%s" % templated_test.body)
if sys.version_info >= (3, 0):
input("Press ENTER when ready (%d): " % (mytest.delay))
else:
raw_input("Press ENTER when ready (%d): " % (mytest.delay))
if mytest.delay > 0:
LOGGER.info("Delaying for %ds" % mytest.delay)
time.sleep(mytest.delay)
try:
response = session.send(prepped)
except Exception as error:
# exception occurred (network error), do not pass go, do not
# collect $200
trace = traceback.format_exc()
result.failures.append(
Failure(message="Request Exception: {0}".format(error),
details=trace,
failure_type=validators.FAILURE_CURL_EXCEPTION))
result.passed = False
session.close()
return result
# Retrieve values
result.body = response.content
body.close()
result.response_headers = response.headers # Per RFC 2616
headers.close()
response_code = response.status_code
result.response_code = response_code
LOGGER.debug("Initial Test Result, based on expected response code: " +
str(response_code in mytest.expected_status))
if response_code in mytest.expected_status:
result.passed = True
else:
# Invalid response code
result.passed = False
failure_message = \
"Invalid HTTP response code: response code " \
"{0} not in expected codes [{1}]".format(response_code,
mytest.expected_status)
result.failures.append(
Failure(message=failure_message,
details=None,
failure_type=validators.FAILURE_INVALID_RESPONSE))
# Parse HTTP headers
try:
result.response_headers = parse_headers(result.response_headers)
except Exception as error:
trace = traceback.format_exc()
result.failures.append(
Failure(message="Header parsing exception: {0}".format(error),
details=trace,
failure_type=validators.FAILURE_TEST_EXCEPTION))
result.passed = False
session.close()
return result
head = result.response_headers
# execute validator on body
if result.passed is True:
body = result.body
if mytest.validators is not None and isinstance(
mytest.validators, list):
LOGGER.debug("executing this many validators: " +
str(len(mytest.validators)))
failures = result.failures
for validator in mytest.validators:
validate_result = validator.validate(body=body,
headers=head,
context=my_context)
if not validate_result:
result.passed = False
# Proxy for checking if it is a Failure object, because of
# import issues with isinstance there
if hasattr(validate_result, 'details'):
failures.append(validate_result)
# TODO add printing of validation for interactive mode
else:
LOGGER.debug("no validators found")
# Only do context updates if test was successful
mytest.update_context_after(result.body, head, my_context)
# Print response body if override is set to print all *OR* if test failed
# (to capture maybe a stack trace)
if test_config.print_bodies or not result.passed:
if test_config.interactive:
LOGGER.info("RESPONSE:")
LOGGER.info(result.body.decode(ESCAPE_DECODING))
if test_config.print_headers or not result.passed:
if test_config.interactive:
LOGGER.info("RESPONSE HEADERS:")
LOGGER.info(result.response_headers)
# TODO add string escape on body output
LOGGER.debug(result)
return result
