def run_testsets(testsets): """ Execute a set of tests, using given TestSet list input """ group_results = dict() # results, by group group_failure_counts = dict() total_failures = 0 myinteractive = False curl_handle = pycurl.Curl() for testset in testsets: mytests = testset.tests myconfig = testset.config mybenchmarks = testset.benchmarks context = Context() # Bind variables & add generators if pertinent if myconfig.variable_binds: context.bind_variables(myconfig.variable_binds) if myconfig.generators: for key, value in myconfig.generators.items(): context.add_generator(key, value) # Make sure we actually have tests to execute if not mytests and not mybenchmarks: # no tests in this test set, probably just imports.. skip to next # test set break myinteractive = True if myinteractive or myconfig.interactive else False # Run tests, collecting statistics as needed for test in mytests: # Initialize the dictionaries to store test fail counts and results if test.group not in group_results: group_results[test.group] = list() group_failure_counts[test.group] = 0 result = run_test(test, test_config=myconfig, context=context, curl_handle=curl_handle) result.body = None # Remove the body, save some memory! if not result.passed: # Print failure, increase failure counts for that test group # Use result test URL to allow for templating logger.error('Test Failed: ' + test.name + " URL=" + result.test.url + " Group=" + test.group + " HTTP Status Code: " + str(result.response_code)) # Print test failure reasons if result.failures: for failure in result.failures: log_failure(failure, context=context, test_config=myconfig) # Increment test failure counts for that group (adding an entry # if not present) failures = group_failure_counts[test.group] failures = failures + 1 group_failure_counts[test.group] = failures else: # Test passed, print results logger.info('Test Succeeded: ' + test.name + " URL=" + test.url + " Group=" + test.group) # Add results for this test group to the resultset group_results[test.group].append(result) # handle stop_on_failure flag if not result.passed and test.stop_on_failure is not None and test.stop_on_failure: print( 'STOP ON FAILURE! stopping test set execution, continuing with other test sets') break for benchmark in mybenchmarks: # Run benchmarks, analyze, write if not benchmark.metrics: logger.debug('Skipping benchmark, no metrics to collect') continue logger.info("Benchmark Starting: " + benchmark.name + " Group: " + benchmark.group) benchmark_result = run_benchmark( benchmark, myconfig, context=context) print(benchmark_result) logger.info("Benchmark Done: " + benchmark.name + " Group: " + benchmark.group) if benchmark.output_file: # Write file logger.debug( 'Writing benchmark to file in format: ' + benchmark.output_format) write_method = OUTPUT_METHODS[benchmark.output_format] my_file = open(benchmark.output_file, 'w') # Overwrites file logger.debug("Benchmark writing to file: " + benchmark.output_file) write_method(my_file, benchmark_result, benchmark, test_config=myconfig) my_file.close() if myinteractive: # a break for when interactive bits are complete, before summary data print("===================================") # Print summary results for group in sorted(group_results.keys()): test_count = len(group_results[group]) failures = group_failure_counts[group] total_failures = total_failures + failures passfail = {True: u'SUCCEEDED: ', False: u'FAILED: '} output_string = "Test Group {0} {1}: {2}/{3} Tests Passed!".format(group, passfail[failures == 0], str(test_count - failures), str(test_count)) if myconfig.skip_term_colors: print(output_string) else: if failures > 0: print('\033[91m' + output_string + '\033[0m') else: print('\033[92m' + output_string + '\033[0m') return total_failures
def run_benchmark(benchmark, test_config=TestConfig(), context=None, *args, **kwargs): """ Perform a benchmark, (re)using a given, configured CURL call to do so The actual analysis of metrics is performed separately, to allow for testing """ # Context handling my_context = context if my_context is None: my_context = Context() warmup_runs = benchmark.warmup_runs benchmark_runs = benchmark.benchmark_runs message = '' # Message is name of benchmark... print it? if (benchmark_runs <= 0): raise Exception( "Invalid number of benchmark runs, must be > 0 :" + benchmark_runs) result = TestResponse() # TODO create and use a curl-returning configuration function # TODO create and use a post-benchmark cleanup function # They should use is_dynamic/is_context_modifier to determine if they need to # worry about context and re-reading/retemplating and only do it if needed # - Also, they will need to be smart enough to handle extraction functions # For performance reasons, we don't want to re-run templating/extraction if # we do not need to, and do not want to save request bodies. # Initialize variables to store output output = BenchmarkResult() output.name = benchmark.name output.group = benchmark.group metricnames = list(benchmark.metrics) # Metric variable for curl, to avoid hash lookup for every metric name metricvalues = [METRICS[name] for name in metricnames] # Initialize arrays to store results for each metric results = [list() for x in xrange(0, len(metricnames))] curl = pycurl.Curl() # Benchmark warm-up to allow for caching, JIT compiling, on client logger.info('Warmup: ' + message + ' started') for x in xrange(0, warmup_runs): benchmark.update_context_before(my_context) templated = benchmark.realize(my_context) curl = templated.configure_curl( timeout=test_config.timeout, context=my_context, curl_handle=curl) # Do not store actual response body at all. curl.setopt(pycurl.WRITEFUNCTION, lambda x: None) curl.perform() logger.info('Warmup: ' + message + ' finished') logger.info('Benchmark: ' + message + ' starting') for x in xrange(0, benchmark_runs): # Run the actual benchmarks # Setup benchmark benchmark.update_context_before(my_context) templated = benchmark.realize(my_context) curl = templated.configure_curl( timeout=test_config.timeout, context=my_context, curl_handle=curl) # Do not store actual response body at all. curl.setopt(pycurl.WRITEFUNCTION, lambda x: None) try: # Run the curl call, if it errors, then add to failure counts for benchmark curl.perform() except Exception: output.failures = output.failures + 1 curl.close() curl = pycurl.Curl() continue # Skip metrics collection # Get all metrics values for this run, and store to metric lists for i in xrange(0, len(metricnames)): results[i].append(curl.getinfo(metricvalues[i])) logger.info('Benchmark: ' + message + ' ending') temp_results = dict() for i in xrange(0, len(metricnames)): temp_results[metricnames[i]] = results[i] output.results = temp_results return analyze_benchmark_results(output, benchmark)
def run_test(mytest, test_config=TestConfig(), context=None, curl_handle=None, *args, **kwargs): """ Put together test pieces: configure & run actual test, return results """ # Initialize a context if not supplied my_context = context if my_context is None: my_context = Context() mytest.update_context_before(my_context) templated_test = mytest.realize(my_context) curl = templated_test.configure_curl( timeout=test_config.timeout, context=my_context, curl_handle=curl_handle) result = TestResponse() result.test = templated_test # reset the body, it holds values from previous runs otherwise headers = MyIO() body = MyIO() curl.setopt(pycurl.WRITEFUNCTION, body.write) curl.setopt(pycurl.HEADERFUNCTION, headers.write) if test_config.verbose: curl.setopt(pycurl.VERBOSE, True) if test_config.ssl_insecure: curl.setopt(pycurl.SSL_VERIFYPEER, 0) curl.setopt(pycurl.SSL_VERIFYHOST, 0) result.passed = None if test_config.interactive: print("===================================") print("%s" % mytest.name) print("-----------------------------------") print("REQUEST:") print("%s %s" % (templated_test.method, templated_test.url)) print("HEADERS:") print("%s" % (templated_test.headers)) if mytest.body is not None: print("\n%s" % templated_test.body) raw_input("Press ENTER when ready (%d): " % (mytest.delay)) if mytest.delay > 0: print("Delaying for %ds" % mytest.delay) time.sleep(mytest.delay) try: curl.perform() # Run the actual call except Exception as e: # Curl exception occurred (network error), do not pass go, do not # collect $200 trace = traceback.format_exc() result.failures.append(Failure(message="Curl Exception: {0}".format( e), details=trace, failure_type=validators.FAILURE_CURL_EXCEPTION)) result.passed = False curl.close() return result # Retrieve values result.body = body.getvalue() body.close() result.response_headers = text_type(headers.getvalue(), HEADER_ENCODING) # Per RFC 2616 headers.close() response_code = curl.getinfo(pycurl.RESPONSE_CODE) result.response_code = response_code logger.debug("Initial Test Result, based on expected response code: " + str(response_code in mytest.expected_status)) if response_code in mytest.expected_status: result.passed = True else: # Invalid response code result.passed = False failure_message = "Invalid HTTP response code: response code {0} not in expected codes [{1}]".format( response_code, mytest.expected_status) result.failures.append(Failure( message=failure_message, details=None, failure_type=validators.FAILURE_INVALID_RESPONSE)) # Parse HTTP headers try: result.response_headers = parse_headers(result.response_headers) except Exception as e: trace = traceback.format_exc() result.failures.append(Failure(message="Header parsing exception: {0}".format( e), details=trace, failure_type=validators.FAILURE_TEST_EXCEPTION)) result.passed = False curl.close() return result # print str(test_config.print_bodies) + ',' + str(not result.passed) + ' , # ' + str(test_config.print_bodies or not result.passed) head = result.response_headers # execute validator on body if result.passed is True: body = result.body if mytest.validators is not None and isinstance(mytest.validators, list): logger.debug("executing this many validators: " + str(len(mytest.validators))) failures = result.failures for validator in mytest.validators: validate_result = validator.validate( body=body, headers=head, context=my_context) if not validate_result: result.passed = False # Proxy for checking if it is a Failure object, because of # import issues with isinstance there if hasattr(validate_result, 'details'): failures.append(validate_result) # TODO add printing of validation for interactive mode else: logger.debug("no validators found") # Only do context updates if test was successful mytest.update_context_after(result.body, head, my_context) # Print response body if override is set to print all *OR* if test failed # (to capture maybe a stack trace) if test_config.print_bodies or not result.passed: if test_config.interactive: print("RESPONSE:") print(result.body.decode(ESCAPE_DECODING)) if test_config.print_headers or not result.passed: if test_config.interactive: print("RESPONSE HEADERS:") print(result.response_headers) # TODO add string escape on body output logger.debug(result) return result
from pyresttest.generators import factory_generate_ids import cProfile test = Benchmark() test.warmup_runs = 0 test.benchmark_runs = 1000 test.raw_metrics = set() test.metrics = {'total_time'} test.aggregated_metrics = {'total_time': ['total', 'mean']} # Basic get test test.url = 'http://localhost:8000/api/person/' test.name = 'Basic GET' print 'Basic GET test' #cProfile.run('resttest.run_benchmark(test)', sort='cumtime') # Test a generator PUT method test.method = 'PUT' test.set_url('http://localhost:8000/api/person/$id/', isTemplate=True) test.headers = {'Content-Type': 'application/json'} handler = ContentHandler() handler.setup( '{"first_name": "Gaius","id": "$id","last_name": "Baltar","login": "******"}', is_template_content=True) test.body = handler context = Context() context.add_generator('gen', factory_generate_ids(starting_id=10)()) test.generator_binds = {'id': 'gen'} print 'Running templated PUT test' cProfile.run('resttest.run_benchmark(test, context=context)', sort='cumtime')
import cProfile test = Benchmark() test.warmup_runs = 0 test.benchmark_runs = 1000 test.raw_metrics = set() test.metrics = {'total_time'} test.aggregated_metrics = {'total_time': ['total', 'mean']} # Basic get test test.url = 'http://localhost:8000/api/person/' test.name = 'Basic GET' print 'Basic GET test' #cProfile.run('resttest.run_benchmark(test)', sort='cumtime') # Test a generator PUT method test.method = 'PUT' test.set_url('http://localhost:8000/api/person/$id/', isTemplate=True) test.headers = {'Content-Type': 'application/json'} handler = ContentHandler() handler.setup('{"first_name": "Gaius","id": "$id","last_name": "Baltar","login": "******"}', is_template_content=True) test.body = handler context = Context() context.add_generator('gen', factory_generate_ids(starting_id=10)()) test.generator_binds = {'id': 'gen'} print 'Running templated PUT test' cProfile.run('test.execute_macro(context=context)', sort='cumtime')
def run_test(mytest, test_config=TestConfig(), context=None, curl_handle=None, *args, **kwargs): """ Put together test pieces: configure & run actual test, return results """ # Initialize a context if not supplied my_context = context if my_context is None: my_context = Context() mytest.update_context_before(my_context) templated_test = mytest.realize(my_context) result = TestResponse() result.test = templated_test session = requests.Session() # generate and attach signature to header req = templated_test.configure_request(timeout=test_config.timeout, context=my_context, curl_handle=curl_handle) if test_config.verbose: session.verbose = True if test_config.ssl_insecure: session.verify = False headers = MyIO() body = MyIO() prepped = req.prepare() prepped = signer.request_signer(prepped, TestConfig.key) result.passed = None if test_config.interactive: LOGGER.debug("===================================") LOGGER.debug("%s" % mytest.name) LOGGER.debug("-----------------------------------") LOGGER.debug("REQUEST:") LOGGER.debug("%s %s" % (templated_test.method, templated_test.url)) LOGGER.debug("HEADERS:") LOGGER.debug("%s" % prepped.headers) if mytest.body is not None: LOGGER.debug("\n%s" % templated_test.body) if sys.version_info >= (3, 0): input("Press ENTER when ready (%d): " % (mytest.delay)) else: raw_input("Press ENTER when ready (%d): " % (mytest.delay)) if mytest.delay > 0: LOGGER.info("Delaying for %ds" % mytest.delay) time.sleep(mytest.delay) try: response = session.send(prepped) except Exception as error: # exception occurred (network error), do not pass go, do not # collect $200 trace = traceback.format_exc() result.failures.append( Failure(message="Request Exception: {0}".format(error), details=trace, failure_type=validators.FAILURE_CURL_EXCEPTION)) result.passed = False session.close() return result # Retrieve values result.body = response.content body.close() result.response_headers = response.headers # Per RFC 2616 headers.close() response_code = response.status_code result.response_code = response_code LOGGER.debug("Initial Test Result, based on expected response code: " + str(response_code in mytest.expected_status)) if response_code in mytest.expected_status: result.passed = True else: # Invalid response code result.passed = False failure_message = \ "Invalid HTTP response code: response code " \ "{0} not in expected codes [{1}]".format(response_code, mytest.expected_status) result.failures.append( Failure(message=failure_message, details=None, failure_type=validators.FAILURE_INVALID_RESPONSE)) # Parse HTTP headers try: result.response_headers = parse_headers(result.response_headers) except Exception as error: trace = traceback.format_exc() result.failures.append( Failure(message="Header parsing exception: {0}".format(error), details=trace, failure_type=validators.FAILURE_TEST_EXCEPTION)) result.passed = False session.close() return result head = result.response_headers # execute validator on body if result.passed is True: body = result.body if mytest.validators is not None and isinstance( mytest.validators, list): LOGGER.debug("executing this many validators: " + str(len(mytest.validators))) failures = result.failures for validator in mytest.validators: validate_result = validator.validate(body=body, headers=head, context=my_context) if not validate_result: result.passed = False # Proxy for checking if it is a Failure object, because of # import issues with isinstance there if hasattr(validate_result, 'details'): failures.append(validate_result) # TODO add printing of validation for interactive mode else: LOGGER.debug("no validators found") # Only do context updates if test was successful mytest.update_context_after(result.body, head, my_context) # Print response body if override is set to print all *OR* if test failed # (to capture maybe a stack trace) if test_config.print_bodies or not result.passed: if test_config.interactive: LOGGER.info("RESPONSE:") LOGGER.info(result.body.decode(ESCAPE_DECODING)) if test_config.print_headers or not result.passed: if test_config.interactive: LOGGER.info("RESPONSE HEADERS:") LOGGER.info(result.response_headers) # TODO add string escape on body output LOGGER.debug(result) return result