def test_content_file_template(self):
""" Test file read and templating of read files in this directory """
variables = {'id':1, 'login':'******'}
context = Context()
file_path = os.path.dirname(os.path.realpath(__file__))
file_path = os.path.join(file_path, 'person_body_template.json')
file_content = None
with open(file_path, 'r') as f:
file_content = f.read()
# Test basic read
handler = ContentHandler()
handler.setup(file_path, is_file=True)
self.assertEqual(file_content, handler.get_content())
# Test templating of read content
handler.setup(file_path, is_file=True, is_template_content=True)
self.assertEqual(file_content, handler.get_content())
self.assertEqual(file_content, handler.get_content(context)) # No substitution
substituted = string.Template(file_content).safe_substitute(variables)
context.bind_variables(variables)
self.assertEqual(substituted, handler.get_content(context))
# Test path templating
templated_file_path = '$filepath'
context.bind_variable('filepath', file_path)
handler.setup(file_path, is_file=True, is_template_path=True)
self.assertEqual(file_content, handler.get_content(context))
# Test double templating with files
handler.setup(file_path, is_file=True, is_template_path=True, is_template_content=True)
self.assertEqual(substituted, handler.get_content(context=context))
def test_header_templating(self):
test = Test()
head_templated = {'$key': "$val"}
context = Context()
context.bind_variables({'key': 'cheese', 'val': 'gouda'})
# No templating applied
test.headers = head_templated
head = test.get_headers()
self.assertEqual(1, len(head))
self.assertEqual('$val', head['$key'])
test.set_headers(head_templated, isTemplate=True)
self.assertTrue(test.templates)
self.assertTrue(test.NAME_HEADERS in test.templates)
# No context, no templating
head = test.headers
self.assertEqual(1, len(head))
self.assertEqual('$val', head['$key'])
# Templated with context
head = test.get_headers(context=context)
self.assertEqual(1, len(head))
self.assertEqual('gouda', head['cheese'])
def test_header_templating(self):
test = Test()
head_templated = {'$key': "$val"}
context = Context()
context.bind_variables({'key': 'cheese', 'val': 'gouda'})
# No templating applied
test.headers = head_templated
head = test.get_headers()
self.assertEqual(1, len(head))
self.assertEqual('$val', head['$key'])
test.set_headers(head_templated, isTemplate=True)
self.assertTrue(test.templates)
self.assertTrue(test.NAME_HEADERS in test.templates)
# No context, no templating
head = test.headers
self.assertEqual(1, len(head))
self.assertEqual('$val', head['$key'])
# Templated with context
head = test.get_headers(context=context)
self.assertEqual(1, len(head))
self.assertEqual('gouda', head['cheese'])
def test_parse_test_templated_headers(self):
""" Test parsing with templated headers """
heads = {"Accept": "Application/json", "$AuthHeader": "$AuthString"}
templated_heads = {
"Accept": "Application/json",
"apikey": "magic_passWord"
}
context = Context()
context.bind_variables({
'AuthHeader': 'apikey',
'AuthString': 'magic_passWord'
})
# If this doesn't throw errors we have silent failures
input_invalid = {
"url": "/ping",
"method": "DELETE",
"NAME": "foo",
"group": "bar",
"body": "<xml>input</xml>",
"headers": 'goat'
}
try:
test = Test.parse_test('', input_invalid)
test.fail("Expected error not thrown")
except TypeError:
pass
def assert_dict_eq(dict1, dict2):
""" Test dicts are equal """
self.assertEqual(2, len(set(dict1.items()) & set(dict2.items())))
# Before templating is used
input = {
"url": "/ping",
"method": "DELETE",
"NAME": "foo",
"group": "bar",
"body": "<xml>input</xml>",
"headers": heads
}
test = Test.parse_test('', input)
assert_dict_eq(heads, test.headers)
assert_dict_eq(heads, test.get_headers(context=context))
# After templating applied
input_templated = {
"url": "/ping",
"method": "DELETE",
"NAME": "foo",
"group": "bar",
"body": "<xml>input</xml>",
"headers": {
'tEmplate': heads
}
}
test2 = Test.parse_test('', input_templated)
assert_dict_eq(heads, test2.get_headers())
assert_dict_eq(templated_heads, test2.get_headers(context=context))
def test_parse_test_templated_headers(self):
""" Test parsing with templated headers """
heads = {"Accept":"Application/json", "$AuthHeader":"$AuthString"}
templated_heads = {"Accept":"Application/json", "apikey":"magic_passWord"}
context = Context()
context.bind_variables({'AuthHeader': 'apikey', 'AuthString':'magic_passWord'})
# If this doesn't throw errors we have silent failures
input_invalid = {"url": "/ping", "method": "DELETE", "NAME":"foo", "group":"bar", "body":"<xml>input</xml>","headers": 'goat'}
try:
test = Test.parse_test('', input_invalid)
test.fail("Expected error not thrown")
except TypeError:
pass
def assert_dict_eq(dict1, dict2):
""" Test dicts are equal """
self.assertEqual(2, len(set(dict1.items()) & set(dict2.items())))
# Before templating is used
input = {"url": "/ping", "method": "DELETE", "NAME":"foo", "group":"bar", "body":"<xml>input</xml>","headers": heads}
test = Test.parse_test('', input)
assert_dict_eq(heads, test.headers)
assert_dict_eq(heads, test.get_headers(context=context))
# After templating applied
input_templated = {"url": "/ping", "method": "DELETE", "NAME":"foo", "group":"bar", "body":"<xml>input</xml>","headers": {'tEmplate': heads}}
test2 = Test.parse_test('', input_templated)
assert_dict_eq(heads, test2.get_headers())
assert_dict_eq(templated_heads, test2.get_headers(context=context))
def test_test_content_templating(self):
test = Test()
handler = ContentHandler()
handler.is_template_content = True
handler.content = '{"first_name": "Gaius","id": "$id","last_name": "Baltar","login": "******"}'
context = Context()
context.bind_variables({'id': 9, 'login': '******'})
test.set_body(handler)
templated = test.realize(context=context)
self.assertEqual(string.Template(handler.content).safe_substitute(context.get_values()),
templated.body)
def test_test_content_templating(self):
test = Test()
handler = ContentHandler()
handler.is_template_content = True
handler.content = '{"first_name": "Gaius","id": "$id","last_name": "Baltar","login": "******"}'
context = Context()
context.bind_variables({'id':9, 'login':'******'})
test.set_body(handler)
templated = test.realize(context=context)
self.assertEqual(string.Template(handler.content).safe_substitute(context.get_values()),
templated.body)
def test_content_file_template(self):
""" Test file read and templating of read files in this directory """
variables = {'id': 1, 'login': '******'}
context = Context()
file_path = os.path.dirname(os.path.realpath(__file__))
file_path = os.path.join(file_path, 'person_body_template.json')
file_content = None
with open(file_path, 'r') as f:
file_content = f.read()
# Test basic read
handler = ContentHandler()
handler.setup(file_path, is_file=True)
self.assertEqual(file_content, handler.get_content())
# Test templating of read content
handler.setup(file_path, is_file=True, is_template_content=True)
self.assertEqual(file_content, handler.get_content())
self.assertEqual(file_content,
handler.get_content(context)) # No substitution
substituted = string.Template(file_content).safe_substitute(variables)
context.bind_variables(variables)
self.assertEqual(substituted, handler.get_content(context))
# Test path templating
templated_file_path = '$filepath'
context.bind_variable('filepath', file_path)
handler.setup(file_path, is_file=True, is_template_path=True)
self.assertEqual(file_content, handler.get_content(context))
# Test double templating with files
handler.setup(file_path,
is_file=True,
is_template_path=True,
is_template_content=True)
self.assertEqual(substituted, handler.get_content(context=context))
def run_testsets(testsets):
""" Execute a set of tests, using given TestSet list input """
group_results = dict() # results, by group
group_failure_counts = dict()
total_failures = 0
myinteractive = False
for testset in testsets:
mytests = testset.tests
myconfig = testset.config
mybenchmarks = testset.benchmarks
context = Context()
# Bind variables & add generators if pertinent
if myconfig.variable_binds:
context.bind_variables(myconfig.variable_binds)
if myconfig.generators:
for key, value in myconfig.generators.items():
context.add_generator(key, value)
# Make sure we actually have tests to execute
if not mytests and not mybenchmarks:
# no tests in this test set, probably just imports.. skip to next test set
break
myinteractive = True if myinteractive or myconfig.interactive else False
# Run tests, collecting statistics as needed
for test in mytests:
# Initialize the dictionaries to store test fail counts and results
if test.group not in group_results:
group_results[test.group] = list()
group_failure_counts[test.group] = 0
result = run_test(test, test_config = myconfig, context=context)
result.body = None # Remove the body, save some memory!
if not result.passed: # Print failure, increase failure counts for that test group
# Use result test URL to allow for templating
logger.error('Test Failed: '+test.name+" URL="+result.test.url+" Group="+test.group+" HTTP Status Code: "+str(result.response_code))
# Print test failure reasons
if result.failures:
for failure in result.failures:
log_failure(failure, context=context, test_config=myconfig)
# Increment test failure counts for that group (adding an entry if not present)
failures = group_failure_counts[test.group]
failures = failures + 1
group_failure_counts[test.group] = failures
else: # Test passed, print results
logger.info('Test Succeeded: '+test.name+" URL="+test.url+" Group="+test.group)
# Add results for this test group to the resultset
group_results[test.group].append(result)
# handle stop_on_failure flag
if not result.passed and test.stop_on_failure is not None and test.stop_on_failure:
print 'STOP ON FAILURE! stopping test set execution, continuing with other test sets'
break
for benchmark in mybenchmarks: # Run benchmarks, analyze, write
if not benchmark.metrics:
logger.debug('Skipping benchmark, no metrics to collect')
continue
logger.info("Benchmark Starting: "+benchmark.name+" Group: "+benchmark.group)
benchmark_result = run_benchmark(benchmark, myconfig, context=context)
print benchmark_result
logger.info("Benchmark Done: "+benchmark.name+" Group: "+benchmark.group)
if benchmark.output_file: # Write file
logger.debug('Writing benchmark to file in format: '+benchmark.output_format)
write_method = OUTPUT_METHODS[benchmark.output_format]
my_file = open(benchmark.output_file, 'w') # Overwrites file
logger.debug("Benchmark writing to file: " + benchmark.output_file)
write_method(my_file, benchmark_result, benchmark, test_config = myconfig)
my_file.close()
if myinteractive:
# a break for when interactive bits are complete, before summary data
print "==================================="
# Print summary results
for group in sorted(group_results.keys()):
test_count = len(group_results[group])
failures = group_failure_counts[group]
total_failures = total_failures + failures
if (failures > 0):
print u'Test Group '+group+u' FAILED: '+ str((test_count-failures))+'/'+str(test_count) + u' Tests Passed!'
else:
print u'Test Group '+group+u' SUCCEEDED: '+ str((test_count-failures))+'/'+str(test_count) + u' Tests Passed!'
return total_failures
def run_testsets(testsets):
""" Execute a set of tests, using given TestSet list input """
group_results = dict() #results, by group
group_failure_counts = dict()
total_failures = 0
myinteractive = False
for testset in testsets:
mytests = testset.tests
myconfig = testset.config
mybenchmarks = testset.benchmarks
context = Context()
# Bind variables & add generators if pertinent
if myconfig.variable_binds:
context.bind_variables(myconfig.variable_binds)
if myconfig.generators:
for key, value in myconfig.generators.items():
context.add_generator(key, value)
#Make sure we actually have tests to execute
if not mytests and not mybenchmarks:
# no tests in this test set, probably just imports.. skip to next test set
break
myinteractive = True if myinteractive or myconfig.interactive else False
#Run tests, collecting statistics as needed
for test in mytests:
#Initialize the dictionaries to store test fail counts and results
if test.group not in group_results:
group_results[test.group] = list()
group_failure_counts[test.group] = 0
result = run_test(test, test_config = myconfig, context=context)
result.body = None # Remove the body, save some memory!
if not result.passed: #Print failure, increase failure counts for that test group
# Use result test URL to allow for templating
logger.error('Test Failed: '+test.name+" URL="+result.test.url+" Group="+test.group+" HTTP Status Code: "+str(result.response_code))
# Print test failure reasons
if result.failures:
for failure in result.failures:
log_failure(failure, context=context, test_config=myconfig)
#Increment test failure counts for that group (adding an entry if not present)
failures = group_failure_counts[test.group]
failures = failures + 1
group_failure_counts[test.group] = failures
else: #Test passed, print results
logger.info('Test Succeeded: '+test.name+" URL="+test.url+" Group="+test.group)
#Add results for this test group to the resultset
group_results[test.group].append(result)
# handle stop_on_failure flag
if not result.passed and test.stop_on_failure is not None and test.stop_on_failure:
print 'STOP ON FAILURE! stopping test set execution, continuing with other test sets'
break
for benchmark in mybenchmarks: # Run benchmarks, analyze, write
if not benchmark.metrics:
logger.debug('Skipping benchmark, no metrics to collect')
continue
logger.info("Benchmark Starting: "+benchmark.name+" Group: "+benchmark.group)
benchmark_result = run_benchmark(benchmark, myconfig, context=context)
print benchmark_result
logger.info("Benchmark Done: "+benchmark.name+" Group: "+benchmark.group)
if benchmark.output_file: # Write file
logger.debug('Writing benchmark to file in format: '+benchmark.output_format)
write_method = OUTPUT_METHODS[benchmark.output_format]
my_file = open(benchmark.output_file, 'w') # Overwrites file
logger.debug("Benchmark writing to file: " + benchmark.output_file)
write_method(my_file, benchmark_result, benchmark, test_config = myconfig)
my_file.close()
if myinteractive:
# a break for when interactive bits are complete, before summary data
print "==================================="
#Print summary results
for group in sorted(group_results.keys()):
test_count = len(group_results[group])
failures = group_failure_counts[group]
total_failures = total_failures + failures
if (failures > 0):
print u'Test Group '+group+u' FAILED: '+ str((test_count-failures))+'/'+str(test_count) + u' Tests Passed!'
else:
print u'Test Group '+group+u' SUCCEEDED: '+ str((test_count-failures))+'/'+str(test_count) + u' Tests Passed!'
return total_failures
