def test_header_templating(self):
test = Test()
head_templated = {'$key': "$val"}
context = Context()
context.bind_variables({'key': 'cheese', 'val': 'gouda'})
# No templating applied
test.headers = head_templated
head = test.get_headers()
self.assertEqual(1, len(head))
self.assertEqual('$val', head['$key'])
test.set_headers(head_templated, isTemplate=True)
self.assertTrue(test.templates)
self.assertTrue(test.NAME_HEADERS in test.templates)
# No context, no templating
head = test.headers
self.assertEqual(1, len(head))
self.assertEqual('$val', head['$key'])
# Templated with context
head = test.get_headers(context=context)
self.assertEqual(1, len(head))
self.assertEqual('gouda', head['cheese'])
def test_parse_test_templated_headers(self):
""" Test parsing with templated headers """
heads = {"Accept": "Application/json", "$AuthHeader": "$AuthString"}
templated_heads = {
"Accept": "Application/json",
"apikey": "magic_passWord"
}
context = Context()
context.bind_variables({
'AuthHeader': 'apikey',
'AuthString': 'magic_passWord'
})
# If this doesn't throw errors we have silent failures
input_invalid = {
"url": "/ping",
"method": "DELETE",
"NAME": "foo",
"group": "bar",
"body": "<xml>input</xml>",
"headers": 'goat'
}
try:
test = Test.parse_test('', input_invalid)
test.fail("Expected error not thrown")
except TypeError:
pass
def assert_dict_eq(dict1, dict2):
""" Test dicts are equal """
self.assertEqual(2, len(set(dict1.items()) & set(dict2.items())))
# Before templating is used
input = {
"url": "/ping",
"method": "DELETE",
"NAME": "foo",
"group": "bar",
"body": "<xml>input</xml>",
"headers": heads
}
test = Test.parse_test('', input)
assert_dict_eq(heads, test.headers)
assert_dict_eq(heads, test.get_headers(context=context))
# After templating applied
input_templated = {
"url": "/ping",
"method": "DELETE",
"NAME": "foo",
"group": "bar",
"body": "<xml>input</xml>",
"headers": {
'tEmplate': heads
}
}
test2 = Test.parse_test('', input_templated)
assert_dict_eq(heads, test2.get_headers())
assert_dict_eq(templated_heads, test2.get_headers(context=context))
def test_content_file_template(self):
""" Test file read and templating of read files in this directory """
variables = {'id':1, 'login':'******'}
context = Context()
file_path = os.path.dirname(os.path.realpath(__file__))
file_path = os.path.join(file_path, 'person_body_template.json')
file_content = None
with open(file_path, 'r') as f:
file_content = f.read()
# Test basic read
handler = ContentHandler()
handler.setup(file_path, is_file=True)
self.assertEqual(file_content, handler.get_content())
# Test templating of read content
handler.setup(file_path, is_file=True, is_template_content=True)
self.assertEqual(file_content, handler.get_content())
self.assertEqual(file_content, handler.get_content(context)) # No substitution
substituted = string.Template(file_content).safe_substitute(variables)
context.bind_variables(variables)
self.assertEqual(substituted, handler.get_content(context))
# Test path templating
templated_file_path = '$filepath'
context.bind_variable('filepath', file_path)
handler.setup(file_path, is_file=True, is_template_path=True)
self.assertEqual(file_content, handler.get_content(context))
# Test double templating with files
handler.setup(file_path, is_file=True, is_template_path=True, is_template_content=True)
self.assertEqual(substituted, handler.get_content(context=context))
def test_parse_validator_comparator(self):
""" Test parsing a comparator validator """
test_config = {
'name':
'Default',
'url':
'/api',
'validators': [{
'comparator': {
'jsonpath_mini': 'id',
'comparator': 'eq',
'expected': {
'template': '$id'
}
}
}]
}
test = Test.parse_test('', test_config)
self.assertTrue(test.validators)
self.assertEqual(1, len(test.validators))
context = Context()
context.bind_variable('id', 3)
myjson = '{"id": "3"}'
failure = test.validators[0].validate(myjson, context=context)
self.assertTrue(test.validators[0].validate(myjson, context=context))
self.assertFalse(test.validators[0].validate(myjson))
def test_header_templating(self):
test = Test()
head_templated = {'$key': "$val"}
context = Context()
context.bind_variables({'key': 'cheese', 'val': 'gouda'})
# No templating applied
test.headers = head_templated
head = test.get_headers()
self.assertEqual(1, len(head))
self.assertEqual('$val', head['$key'])
test.set_headers(head_templated, isTemplate=True)
self.assertTrue(test.templates)
self.assertTrue(test.NAME_HEADERS in test.templates)
# No context, no templating
head = test.headers
self.assertEqual(1, len(head))
self.assertEqual('$val', head['$key'])
# Templated with context
head = test.get_headers(context=context)
self.assertEqual(1, len(head))
self.assertEqual('gouda', head['cheese'])
def test_parse_test_templated_headers(self):
""" Test parsing with templated headers """
heads = {"Accept":"Application/json", "$AuthHeader":"$AuthString"}
templated_heads = {"Accept":"Application/json", "apikey":"magic_passWord"}
context = Context()
context.bind_variables({'AuthHeader': 'apikey', 'AuthString':'magic_passWord'})
# If this doesn't throw errors we have silent failures
input_invalid = {"url": "/ping", "method": "DELETE", "NAME":"foo", "group":"bar", "body":"<xml>input</xml>","headers": 'goat'}
try:
test = Test.parse_test('', input_invalid)
test.fail("Expected error not thrown")
except TypeError:
pass
def assert_dict_eq(dict1, dict2):
""" Test dicts are equal """
self.assertEqual(2, len(set(dict1.items()) & set(dict2.items())))
# Before templating is used
input = {"url": "/ping", "method": "DELETE", "NAME":"foo", "group":"bar", "body":"<xml>input</xml>","headers": heads}
test = Test.parse_test('', input)
assert_dict_eq(heads, test.headers)
assert_dict_eq(heads, test.get_headers(context=context))
# After templating applied
input_templated = {"url": "/ping", "method": "DELETE", "NAME":"foo", "group":"bar", "body":"<xml>input</xml>","headers": {'tEmplate': heads}}
test2 = Test.parse_test('', input_templated)
assert_dict_eq(heads, test2.get_headers())
assert_dict_eq(templated_heads, test2.get_headers(context=context))
def test_update_context_variables(self):
test = Test()
context = Context()
context.bind_variable('foo', 'broken')
test.variable_binds = {'foo': 'correct', 'test': 'value'}
test.update_context_before(context)
self.assertEqual('correct', context.get_value('foo'))
self.assertEqual('value', context.get_value('test'))
def test_abstract_extractor_templating(self):
""" Test that abstract extractors template the query """
ext = validators.AbstractExtractor()
ext.query = '$val.vee'
ext.is_templated = True
context = Context()
context.bind_variable('val', 'foo')
self.assertEqual('$val.vee', ext.templated_query())
self.assertEqual('foo.vee', ext.templated_query(context=context))
ext.is_templated = False
self.assertEqual('$val.vee', ext.templated_query(context=context))
def test_test_content_templating(self):
test = Test()
handler = ContentHandler()
handler.is_template_content = True
handler.content = '{"first_name": "Gaius","id": "$id","last_name": "Baltar","login": "******"}'
context = Context()
context.bind_variables({'id': 9, 'login': '******'})
test.set_body(handler)
templated = test.realize(context=context)
self.assertEqual(string.Template(handler.content).safe_substitute(context.get_values()),
templated.body)
def test_parse_content_templated(self):
""" Test parsing of templated content """
node = {'template': 'myval $var'}
handler = ContentHandler.parse_content(node)
context = Context()
context.bind_variable('var', 'cheese')
self.assertEqual(node['template'], handler.content)
self.assertEqual('myval cheese', handler.get_content(context))
self.assertTrue(handler.is_dynamic())
self.assertFalse(handler.is_file)
self.assertFalse(handler.is_template_path)
self.assertTrue(handler.is_template_content)
def test_parse_content_templated(self):
""" Test parsing of templated content """
node = {'template':'myval $var'}
handler = ContentHandler.parse_content(node)
context = Context()
context.bind_variable('var','cheese')
self.assertEqual(node['template'], handler.content)
self.assertEqual('myval cheese', handler.get_content(context))
self.assertTrue(handler.is_dynamic())
self.assertFalse(handler.is_file)
self.assertFalse(handler.is_template_path)
self.assertTrue(handler.is_template_content)
def test_test_content_templating(self):
test = Test()
handler = ContentHandler()
handler.is_template_content = True
handler.content = '{"first_name": "Gaius","id": "$id","last_name": "Baltar","login": "******"}'
context = Context()
context.bind_variables({'id':9, 'login':'******'})
test.set_body(handler)
templated = test.realize(context=context)
self.assertEqual(string.Template(handler.content).safe_substitute(context.get_values()),
templated.body)
def test_mixing_binds(self):
""" Ensure that variables are set correctly when mixing explicit declaration and variables """
context = Context()
context.add_generator('gen', count_gen())
context.bind_variable('foo', '100')
self.assertEqual(1, context.mod_count)
context.bind_generator_next('foo', 'gen')
self.assertEqual(1, context.get_value('foo'))
self.assertEqual(2, context.mod_count)
def test_test_url_templating(self):
test = Test()
test.set_url('$cheese', isTemplate=True)
self.assertTrue(test.is_dynamic())
self.assertEqual('$cheese', test.get_url())
self.assertTrue(test.templates['url'])
context = Context()
context.bind_variable('cheese', 'stilton')
self.assertEqual('stilton', test.get_url(context=context))
realized = test.realize(context)
self.assertEqual('stilton', realized.url)
def test_test_url_templating(self):
test = Test()
test.set_url('$cheese', isTemplate=True)
self.assertTrue(test.is_dynamic())
self.assertEqual('$cheese', test.get_url())
self.assertTrue(test.templates['url'])
context = Context()
context.bind_variable('cheese', 'stilton')
self.assertEqual('stilton', test.get_url(context=context))
realized = test.realize(context)
self.assertEqual('stilton', realized.url)
def test_parse_validator(self):
""" Test basic parsing using registry """
config = {"jsonpath_mini": "key.val", "comparator": "eq", "expected": 3}
validator = validators.parse_validator("comparator", config)
myjson = '{"key": {"val": 3}}'
comp = validator.validate(body=myjson)
# Try it with templating
config["jsonpath_mini"] = {"template": "key.$node"}
validator = validators.parse_validator("comparator", config)
context = Context()
context.bind_variable("node", "val")
comp = validator.validate(myjson, context=context)
def test_content_templating(self):
""" Test content and templating of it """
handler = ContentHandler()
body = '$variable value'
context = Context()
context.bind_variable('variable', 'bar')
# No templating
handler.setup(body, is_template_content=False)
self.assertEqual(body, handler.get_content())
self.assertEqual(body, handler.get_content(context))
# Templating
handler.setup(body, is_template_content=True)
self.assertEqual(body, handler.get_content())
def test_content_templating(self):
""" Test content and templating of it """
handler = ContentHandler()
body = '$variable value'
context = Context()
context.bind_variable('variable', 'bar')
# No templating
handler.setup(body, is_template_content=False)
self.assertEqual(body, handler.get_content())
self.assertEqual(body, handler.get_content(context))
# Templating
handler.setup(body, is_template_content=True)
self.assertEqual(body, handler.get_content())
def test_variables(self):
""" Test bind/return of variables """
context = Context()
self.assertTrue(context.get_value('foo') is None)
self.assertEqual(0, context.mod_count)
context.bind_variable('foo', 'bar')
self.assertEqual('bar', context.get_value('foo'))
self.assertEqual('bar', context.get_values()['foo'])
self.assertEqual(1, context.mod_count)
context.bind_variable('foo', 'bar2')
self.assertEqual('bar2', context.get_value('foo'))
self.assertEqual(2, context.mod_count)
def test_generator_bind(self):
""" Test generator setting to variables """
context = Context()
self.assertEqual(0, len(context.get_generators()))
my_gen = count_gen()
context.add_generator('gen', my_gen)
context.bind_generator_next('foo', 'gen')
self.assertEqual(1, context.mod_count)
self.assertEqual(1, context.get_value('foo'))
self.assertTrue(2, context.get_generator('gen').next())
self.assertTrue(3, my_gen.next())
def test_variable_binding(self):
""" Test that tests successfully bind variables """
element = 3
input = [{"url": "/ping"},{"name": "cheese"},{"expected_status":["200",204,"202"]}]
input.append({"variable_binds":{'var':'value'}})
test = Test.parse_test('', input)
binds = test.variable_binds
self.assertEqual(1, len(binds))
self.assertEqual('value', binds['var'])
# Test that updates context correctly
context = Context()
test.update_context_before(context)
self.assertEqual('value', context.get_value('var'))
self.assertTrue(test.is_context_modifier())
def test_mixing_binds(self):
""" Ensure that variables are set correctly when mixing explicit declaration and variables """
context = Context()
context.add_generator('gen', count_gen())
context.bind_variable('foo', '100')
self.assertEqual(1, context.mod_count)
context.bind_generator_next('foo', 'gen')
self.assertEqual(1, context.get_value('foo'))
self.assertEqual(2, context.mod_count)
def test_validator_comparator_templating(self):
""" Try templating comparator validator """
config = {"jsonpath_mini": {"template": "key.$node"}, "comparator": "eq", "expected": 3}
context = Context()
context.bind_variable("node", "val")
myjson_pass = '******'
myjson_fail = '{"id": 3, "key": {"val": 4}}'
comp = validators.ComparatorValidator.parse(config)
self.assertTrue(comp.validate(body=myjson_pass, context=context))
self.assertFalse(comp.validate(body=myjson_fail, context=context))
# Template expected
config["expected"] = {"template": "$id"}
context.bind_variable("id", 3)
self.assertTrue(comp.validate(body=myjson_pass, context=context))
self.assertFalse(comp.validate(body=myjson_fail, context=context))
def test_parse_validator(self):
""" Test basic parsing using registry """
config = {
'jsonpath_mini': 'key.val',
'comparator': 'eq',
'expected': 3
}
validator = validators.parse_validator('comparator', config)
myjson = '{"key": {"val": 3}}'
comp = validator.validate(body=myjson)
# Try it with templating
config['jsonpath_mini']={'template':'key.$node'}
validator = validators.parse_validator('comparator', config)
context = Context()
context.bind_variable('node','val')
comp = validator.validate(myjson, context=context)
def test_generator(self):
""" Test adding a generator """
context = Context()
self.assertEqual(0, len(context.get_generators()))
my_gen = count_gen()
context.add_generator('gen', my_gen)
self.assertEqual(1, len(context.get_generators()))
self.assertTrue('gen' in context.get_generators())
self.assertTrue(context.get_generator('gen') is not None)
def test_header_extraction(self):
test = Test()
test.url = self.prefix + '/api/person/1/'
key1 = 'server-header'
key2 = 'server-header-mixedcase'
test.extract_binds = {
key1: validators.HeaderExtractor.parse('server'),
# Verify case-insensitive behavior
key2: validators.HeaderExtractor.parse('sErVer')
}
my_context = Context()
test_response = resttest.run_test(test, context=my_context)
val1 = my_context.get_value(key1)
val2 = my_context.get_value(key2)
self.assertEqual(val1, val2)
self.assertTrue('wsgi' in val1.lower())
self.assertTrue('wsgi' in val2.lower())
def test_header_extraction(self):
test = Test()
test.url = self.prefix + "/api/person/1/"
key1 = "server-header"
key2 = "server-header-mixedcase"
test.extract_binds = {
key1: validators.HeaderExtractor.parse("server"),
# Verify case-insensitive behavior
key2: validators.HeaderExtractor.parse("sErVer"),
}
my_context = Context()
test_response = resttest.run_test(test, context=my_context)
val1 = my_context.get_value(key1)
val2 = my_context.get_value(key2)
self.assertEqual(val1, val2)
self.assertTrue("wsgi" in val1.lower())
self.assertTrue("wsgi" in val2.lower())
def run_test(mytest, test_config = TestConfig(), context = None):
""" Put together test pieces: configure & run actual test, return results """
# Initialize a context if not supplied
my_context = context
if my_context is None:
my_context = Context()
mytest.update_context_before(my_context)
templated_test = mytest.realize(my_context)
curl = templated_test.configure_curl(timeout=test_config.timeout, context=my_context)
result = TestResponse()
result.test = templated_test
# reset the body, it holds values from previous runs otherwise
headers = StringIO()
body = StringIO()
curl.setopt(pycurl.WRITEDATA, body)
curl.setopt(pycurl.HEADERFUNCTION, headers.write)
result.passed = None
if test_config.interactive:
print "==================================="
print "%s" % mytest.name
print "-----------------------------------"
print "REQUEST:"
print "%s %s" % (mytest.method, mytest.url)
if mytest.body is not None:
print "\n%s" % mytest.body
raw_input("Press ENTER when ready: ")
if (mytest.delay > 0.0):
logger.debug("Delay sleeping " + str(mytest.delay) + " before HTTP request")
time.sleep(mytest.delay)
retries = test_config.retries
retry_sleep = 1.0
while True:
try:
curl.perform() #Run the actual call
except Exception, e:
# Curl exception occurred (network error), do not pass go, do not collect $200
trace = traceback.format_exc()
result.failures.append(Failure(message="Curl Exception: {0}".format(e), details=trace, failure_type=validators.FAILURE_CURL_EXCEPTION))
result.passed = False
curl.close()
return result
response_code = curl.getinfo(pycurl.RESPONSE_CODE)
if (response_code == 503 and retries > 0):
logger.debug("Retry sleeping " + str(retry_sleep) + " seconds after HTTP 503")
time.sleep(retry_sleep)
retry_sleep = retry_sleep * 2.0
retries = retries - 1
else:
break
def test_variables(self):
""" Test bind/return of variables """
context = Context()
self.assertTrue(context.get_value('foo') is None)
self.assertEqual(0, context.mod_count)
context.bind_variable('foo','bar')
self.assertEqual('bar', context.get_value('foo'))
self.assertEqual('bar', context.get_values()['foo'])
self.assertEqual(1, context.mod_count)
context.bind_variable('foo','bar2')
self.assertEqual('bar2', context.get_value('foo'))
self.assertEqual(2, context.mod_count)
def run_test(mytest, test_config=TestConfig(), context=None):
""" Put together test pieces: configure & run actual test, return results """
# Initialize a context if not supplied
my_context = context
if my_context is None:
my_context = Context()
mytest.update_context_before(my_context)
templated_test = mytest.realize(my_context)
curl = templated_test.configure_curl(timeout=test_config.timeout,
context=my_context)
result = TestResponse()
result.test = templated_test
# reset the body, it holds values from previous runs otherwise
headers = StringIO()
body = StringIO()
curl.setopt(pycurl.WRITEDATA, body)
curl.setopt(pycurl.HEADERFUNCTION, headers.write)
if test_config.verbose:
curl.setopt(pycurl.VERBOSE, True)
if test_config.ssl_insecure:
curl.setopt(pycurl.SSL_VERIFYPEER, 0)
curl.setopt(pycurl.SSL_VERIFYHOST, 0)
result.passed = None
if test_config.interactive:
print "==================================="
print "%s" % mytest.name
print "-----------------------------------"
print "REQUEST:"
print "%s %s" % (templated_test.method, templated_test.url)
print "HEADERS:"
print "%s" % (templated_test.headers)
if mytest.body is not None:
print "\n%s" % templated_test.body
raw_input("Press ENTER when ready (%d): " % (mytest.delay))
if mytest.delay > 0:
print "Delaying for %ds" % mytest.delay
time.sleep(mytest.delay)
try:
curl.perform() # Run the actual call
except Exception, e:
# Curl exception occurred (network error), do not pass go, do not collect $200
trace = traceback.format_exc()
result.failures.append(
Failure(message="Curl Exception: {0}".format(e),
details=trace,
failure_type=validators.FAILURE_CURL_EXCEPTION))
result.passed = False
curl.close()
return result
def test_generator_bind(self):
""" Test generator setting to variables """
context = Context()
self.assertEqual(0, len(context.get_generators()))
my_gen = count_gen()
context.add_generator('gen', my_gen)
context.bind_generator_next('foo', 'gen')
self.assertEqual(1, context.mod_count)
self.assertEqual(1, context.get_value('foo'))
self.assertTrue(2, context.get_generator('gen').next())
self.assertTrue(3, my_gen.next())
def test_update_context_generators(self):
""" Test updating context variables using generator """
test = Test()
context = Context()
context.bind_variable('foo', 'broken')
test.variable_binds = {'foo': 'initial_value'}
test.generator_binds = {'foo': 'gen'}
context.add_generator('gen', generators.generator_basic_ids())
test.update_context_before(context)
self.assertEqual(1, context.get_value('foo'))
test.update_context_before(context)
self.assertEqual(2, context.get_value('foo'))
def test_parse_extractor_minijson(self):
config = 'key.val'
extractor = validators.MiniJsonExtractor.parse(config)
myjson = '{"key": {"val": 3}}'
context = Context()
context.bind_variable('node', 'val')
extracted = extractor.extract(body=myjson)
self.assertEqual(3, extracted)
self.assertEqual(extracted, extractor.extract(body=myjson, context=context))
try:
val = extractor.extract(body='[31{]')
self.fail("Should throw exception on invalid JSON")
except ValueError:
pass
# Templating
config = {'template': 'key.$node'}
extract = validators.MiniJsonExtractor.parse(config)
self.assertEqual(3, extract.extract(myjson, context=context))
def test_generator(self):
""" Test adding a generator """
context = Context()
self.assertEqual(0, len(context.get_generators()))
my_gen = count_gen()
context.add_generator('gen', my_gen)
self.assertEqual(1, len(context.get_generators()))
self.assertTrue('gen' in context.get_generators())
self.assertTrue(context.get_generator('gen') is not None)
def test_abstract_extractor_readableconfig(self):
""" Test human-readable extractor config string output """
config = 'key.val'
extractor = validators.parse_extractor('jsonpath_mini', config)
expected_string = 'Extractor Type: jsonpath_mini, Query: "key.val", Templated?: False'
self.assertEqual(expected_string, extractor.get_readable_config())
# Check empty context & args uses okay
context = Context()
self.assertEqual(expected_string, extractor.get_readable_config(context=context))
context.bind_variable('foo', 'bar')
self.assertEqual(expected_string, extractor.get_readable_config(context=context))
extractor.args = dict()
self.assertEqual(expected_string, extractor.get_readable_config(context=context))
# Check args output is handled correctly
extractor.args = {'caseSensitive': True}
self.assertEqual(expected_string+", Args: "+str(extractor.args), extractor.get_readable_config(context=context))
# Check template handling is okay
config = {'template': 'key.$templated'}
context.bind_variable('templated', 'val')
extractor = validators.parse_extractor('jsonpath_mini', config)
expected_string = 'Extractor Type: jsonpath_mini, Query: "key.val", Templated?: True'
self.assertEqual(expected_string, extractor.get_readable_config(context=context))
def test_parse_validator_comparator(self):
""" Test parsing a comparator validator """
test_config = {
'name': 'Default',
'url': '/api',
'validators': [
{'comparator': {'jsonpath_mini': 'id',
'comparator': 'eq',
'expected': {'template': '$id'}}}
]
}
test = Test.parse_test('', test_config)
self.assertTrue(test.validators)
self.assertEqual(1, len(test.validators))
context = Context()
context.bind_variable('id', 3)
myjson = '{"id": "3"}'
failure = test.validators[0].validate(myjson, context=context)
self.assertTrue(test.validators[0].validate(myjson, context=context))
self.assertFalse(test.validators[0].validate(myjson))
def test_update_context_variables(self):
test = Test()
context = Context()
context.bind_variable('foo', 'broken')
test.variable_binds = {'foo': 'correct', 'test': 'value'}
test.update_context_before(context)
self.assertEqual('correct', context.get_value('foo'))
self.assertEqual('value', context.get_value('test'))
def test_update_context_generators(self):
""" Test updating context variables using generator """
test = Test()
context = Context()
context.bind_variable('foo', 'broken')
test.variable_binds = {'foo': 'initial_value'}
test.generator_binds = {'foo': 'gen'}
context.add_generator('gen', generators.generator_basic_ids())
test.update_context_before(context)
self.assertEqual(1, context.get_value('foo'))
test.update_context_before(context)
self.assertEqual(2, context.get_value('foo'))
def test_validator_comparator_templating(self):
""" Try templating comparator validator """
config = {
'jsonpath_mini': {'template': 'key.$node'},
'comparator': 'eq',
'expected': 3
}
context = Context()
context.bind_variable('node', 'val')
myjson_pass = '******'
myjson_fail = '{"id": 3, "key": {"val": 4}}'
comp = validators.ComparatorValidator.parse(config)
self.assertTrue(comp.validate(body=myjson_pass, context=context))
self.assertFalse(comp.validate(body=myjson_fail, context=context))
# Template expected
config['expected'] = {'template' : '$id'}
context.bind_variable('id', 3)
self.assertTrue(comp.validate(body=myjson_pass, context=context))
self.assertFalse(comp.validate(body=myjson_fail, context=context))
def test_content_file_template(self):
""" Test file read and templating of read files in this directory """
variables = {'id': 1, 'login': '******'}
context = Context()
file_path = os.path.dirname(os.path.realpath(__file__))
file_path = os.path.join(file_path, 'person_body_template.json')
file_content = None
with open(file_path, 'r') as f:
file_content = f.read()
# Test basic read
handler = ContentHandler()
handler.setup(file_path, is_file=True)
self.assertEqual(file_content, handler.get_content())
# Test templating of read content
handler.setup(file_path, is_file=True, is_template_content=True)
self.assertEqual(file_content, handler.get_content())
self.assertEqual(file_content,
handler.get_content(context)) # No substitution
substituted = string.Template(file_content).safe_substitute(variables)
context.bind_variables(variables)
self.assertEqual(substituted, handler.get_content(context))
# Test path templating
templated_file_path = '$filepath'
context.bind_variable('filepath', file_path)
handler.setup(file_path, is_file=True, is_template_path=True)
self.assertEqual(file_content, handler.get_content(context))
# Test double templating with files
handler.setup(file_path,
is_file=True,
is_template_path=True,
is_template_content=True)
self.assertEqual(substituted, handler.get_content(context=context))
def run_testsets(testsets):
""" Execute a set of tests, using given TestSet list input """
group_results = dict() # results, by group
group_failure_counts = dict()
total_failures = 0
myinteractive = False
for testset in testsets:
mytests = testset.tests
myconfig = testset.config
mybenchmarks = testset.benchmarks
context = Context()
# Bind variables & add generators if pertinent
if myconfig.variable_binds:
context.bind_variables(myconfig.variable_binds)
if myconfig.generators:
for key, value in myconfig.generators.items():
context.add_generator(key, value)
# Make sure we actually have tests to execute
if not mytests and not mybenchmarks:
# no tests in this test set, probably just imports.. skip to next test set
break
myinteractive = True if myinteractive or myconfig.interactive else False
# Run tests, collecting statistics as needed
for test in mytests:
# Initialize the dictionaries to store test fail counts and results
if test.group not in group_results:
group_results[test.group] = list()
group_failure_counts[test.group] = 0
result = run_test(test, test_config = myconfig, context=context)
result.body = None # Remove the body, save some memory!
if not result.passed: # Print failure, increase failure counts for that test group
# Use result test URL to allow for templating
logger.error('Test Failed: '+test.name+" URL="+result.test.url+" Group="+test.group+" HTTP Status Code: "+str(result.response_code))
# Print test failure reasons
if result.failures:
for failure in result.failures:
log_failure(failure, context=context, test_config=myconfig)
# Increment test failure counts for that group (adding an entry if not present)
failures = group_failure_counts[test.group]
failures = failures + 1
group_failure_counts[test.group] = failures
else: # Test passed, print results
logger.info('Test Succeeded: '+test.name+" URL="+test.url+" Group="+test.group)
# Add results for this test group to the resultset
group_results[test.group].append(result)
# handle stop_on_failure flag
if not result.passed and test.stop_on_failure is not None and test.stop_on_failure:
print 'STOP ON FAILURE! stopping test set execution, continuing with other test sets'
break
for benchmark in mybenchmarks: # Run benchmarks, analyze, write
if not benchmark.metrics:
logger.debug('Skipping benchmark, no metrics to collect')
continue
logger.info("Benchmark Starting: "+benchmark.name+" Group: "+benchmark.group)
benchmark_result = run_benchmark(benchmark, myconfig, context=context)
print benchmark_result
logger.info("Benchmark Done: "+benchmark.name+" Group: "+benchmark.group)
if benchmark.output_file: # Write file
logger.debug('Writing benchmark to file in format: '+benchmark.output_format)
write_method = OUTPUT_METHODS[benchmark.output_format]
my_file = open(benchmark.output_file, 'w') # Overwrites file
logger.debug("Benchmark writing to file: " + benchmark.output_file)
write_method(my_file, benchmark_result, benchmark, test_config = myconfig)
my_file.close()
if myinteractive:
# a break for when interactive bits are complete, before summary data
print "==================================="
# Print summary results
for group in sorted(group_results.keys()):
test_count = len(group_results[group])
failures = group_failure_counts[group]
total_failures = total_failures + failures
if (failures > 0):
print u'Test Group '+group+u' FAILED: '+ str((test_count-failures))+'/'+str(test_count) + u' Tests Passed!'
else:
print u'Test Group '+group+u' SUCCEEDED: '+ str((test_count-failures))+'/'+str(test_count) + u' Tests Passed!'
return total_failures
def run_testsets(testsets):
""" Execute a set of tests, using given TestSet list input """
group_results = dict() #results, by group
group_failure_counts = dict()
total_failures = 0
myinteractive = False
for testset in testsets:
mytests = testset.tests
myconfig = testset.config
mybenchmarks = testset.benchmarks
context = Context()
# Bind variables & add generators if pertinent
if myconfig.variable_binds:
context.bind_variables(myconfig.variable_binds)
if myconfig.generators:
for key, value in myconfig.generators.items():
context.add_generator(key, value)
#Make sure we actually have tests to execute
if not mytests and not mybenchmarks:
# no tests in this test set, probably just imports.. skip to next test set
break
myinteractive = True if myinteractive or myconfig.interactive else False
#Run tests, collecting statistics as needed
for test in mytests:
#Initialize the dictionaries to store test fail counts and results
if test.group not in group_results:
group_results[test.group] = list()
group_failure_counts[test.group] = 0
result = run_test(test, test_config = myconfig, context=context)
result.body = None # Remove the body, save some memory!
if not result.passed: #Print failure, increase failure counts for that test group
# Use result test URL to allow for templating
logger.error('Test Failed: '+test.name+" URL="+result.test.url+" Group="+test.group+" HTTP Status Code: "+str(result.response_code))
# Print test failure reasons
if result.failures:
for failure in result.failures:
log_failure(failure, context=context, test_config=myconfig)
#Increment test failure counts for that group (adding an entry if not present)
failures = group_failure_counts[test.group]
failures = failures + 1
group_failure_counts[test.group] = failures
else: #Test passed, print results
logger.info('Test Succeeded: '+test.name+" URL="+test.url+" Group="+test.group)
#Add results for this test group to the resultset
group_results[test.group].append(result)
# handle stop_on_failure flag
if not result.passed and test.stop_on_failure is not None and test.stop_on_failure:
print 'STOP ON FAILURE! stopping test set execution, continuing with other test sets'
break
for benchmark in mybenchmarks: # Run benchmarks, analyze, write
if not benchmark.metrics:
logger.debug('Skipping benchmark, no metrics to collect')
continue
logger.info("Benchmark Starting: "+benchmark.name+" Group: "+benchmark.group)
benchmark_result = run_benchmark(benchmark, myconfig, context=context)
print benchmark_result
logger.info("Benchmark Done: "+benchmark.name+" Group: "+benchmark.group)
if benchmark.output_file: # Write file
logger.debug('Writing benchmark to file in format: '+benchmark.output_format)
write_method = OUTPUT_METHODS[benchmark.output_format]
my_file = open(benchmark.output_file, 'w') # Overwrites file
logger.debug("Benchmark writing to file: " + benchmark.output_file)
write_method(my_file, benchmark_result, benchmark, test_config = myconfig)
my_file.close()
if myinteractive:
# a break for when interactive bits are complete, before summary data
print "==================================="
#Print summary results
for group in sorted(group_results.keys()):
test_count = len(group_results[group])
failures = group_failure_counts[group]
total_failures = total_failures + failures
if (failures > 0):
print u'Test Group '+group+u' FAILED: '+ str((test_count-failures))+'/'+str(test_count) + u' Tests Passed!'
else:
print u'Test Group '+group+u' SUCCEEDED: '+ str((test_count-failures))+'/'+str(test_count) + u' Tests Passed!'
return total_failures
def run_benchmark(benchmark, test_config = TestConfig(), context = None):
""" Perform a benchmark, (re)using a given, configured CURL call to do so
The actual analysis of metrics is performed separately, to allow for testing
"""
# Context handling
my_context = context
if my_context is None:
my_context = Context()
warmup_runs = benchmark.warmup_runs
benchmark_runs = benchmark.benchmark_runs
message = '' #Message is name of benchmark... print it?
if (benchmark_runs <= 0):
raise Exception("Invalid number of benchmark runs, must be > 0 :" + benchmark_runs)
result = TestResponse()
# TODO create and use a curl-returning configuration function
# TODO create and use a post-benchmark cleanup function
# They should use is_dynamic/is_context_modifier to determine if they need to
# worry about context and re-reading/retemplating and only do it if needed
# - Also, they will need to be smart enough to handle extraction functions
# For performance reasons, we don't want to re-run templating/extraction if
# we do not need to, and do not want to save request bodies.
#Initialize variables to store output
output = BenchmarkResult()
output.name = benchmark.name
output.group = benchmark.group
metricnames = list(benchmark.metrics)
metricvalues = [METRICS[name] for name in metricnames] # Metric variable for curl, to avoid hash lookup for every metric name
results = [list() for x in xrange(0, len(metricnames))] # Initialize arrays to store results for each metric
curl = pycurl.Curl()
#Benchmark warm-up to allow for caching, JIT compiling, on client
logger.info('Warmup: ' + message + ' started')
for x in xrange(0, warmup_runs):
benchmark.update_context_before(my_context)
templated = benchmark.realize(my_context)
curl = templated.configure_curl(timeout=test_config.timeout, context=my_context, curl_handle=curl)
curl.setopt(pycurl.WRITEFUNCTION, lambda x: None) #Do not store actual response body at all.
curl.perform()
logger.info('Warmup: ' + message + ' finished')
logger.info('Benchmark: ' + message + ' starting')
for x in xrange(0, benchmark_runs): # Run the actual benchmarks
# Setup benchmark
benchmark.update_context_before(my_context)
templated = benchmark.realize(my_context)
curl = templated.configure_curl(timeout=test_config.timeout, context=my_context, curl_handle=curl)
curl.setopt(pycurl.WRITEFUNCTION, lambda x: None) #Do not store actual response body at all.
try: # Run the curl call, if it errors, then add to failure counts for benchmark
curl.perform()
except Exception:
output.failures = output.failures + 1
curl.close()
curl = pycurl.Curl()
continue # Skip metrics collection
# Get all metrics values for this run, and store to metric lists
for i in xrange(0, len(metricnames)):
results[i].append( curl.getinfo(metricvalues[i]) )
curl.close()
logger.info('Benchmark: ' + message + ' ending')
temp_results = dict()
for i in xrange(0, len(metricnames)):
temp_results[metricnames[i]] = results[i]
output.results = temp_results
return analyze_benchmark_results(output, benchmark)
def run_test(mytest, test_config = TestConfig(), context = None):
""" Put together test pieces: configure & run actual test, return results """
# Initialize a context if not supplied
my_context = context
if my_context is None:
my_context = Context()
mytest.update_context_before(my_context)
templated_test = mytest.realize(my_context)
curl = templated_test.configure_curl(timeout=test_config.timeout, context=my_context)
result = TestResponse()
result.test = templated_test
# reset the body, it holds values from previous runs otherwise
headers = StringIO()
body = StringIO()
curl.setopt(pycurl.WRITEDATA, body)
curl.setopt(pycurl.HEADERFUNCTION, headers.write)
if test_config.verbose:
curl.setopt(pycurl.VERBOSE,True)
if test_config.ssl_insecure:
curl.setopt(pycurl.SSL_VERIFYPEER,0)
curl.setopt(pycurl.SSL_VERIFYHOST,0)
result.passed = None
if test_config.interactive:
print("===================================")
print("%s" % mytest.name)
print("-----------------------------------")
print("REQUEST:")
print("%s %s" % (templated_test.method, templated_test.url))
print("HEADERS:")
print("%s" % (templated_test.headers))
if mytest.body is not None:
print("\n%s" % templated_test.body)
raw_input("Press ENTER when ready (%d): " % (mytest.delay))
if mytest.delay > 0:
print("Delaying for %ds" % mytest.delay)
time.sleep(mytest.delay)
try:
curl.perform() # Run the actual call
except Exception as e:
# Curl exception occurred (network error), do not pass go, do not collect $200
trace = traceback.format_exc()
result.failures.append(Failure(message="Curl Exception: {0}".format(e), details=trace, failure_type=validators.FAILURE_CURL_EXCEPTION))
result.passed = False
curl.close()
return result
# Retrieve values
result.body = body.getvalue()
body.close()
result.response_headers = headers.getvalue()
headers.close()
response_code = curl.getinfo(pycurl.RESPONSE_CODE)
result.response_code = response_code
logger.debug("Initial Test Result, based on expected response code: "+str(response_code in mytest.expected_status))
if response_code in mytest.expected_status:
result.passed = True
else:
# Invalid response code
result.passed = False
failure_message = "Invalid HTTP response code: response code {0} not in expected codes [{1}]".format(response_code, mytest.expected_status)
result.failures.append(Failure(message=failure_message, details=None, failure_type=validators.FAILURE_INVALID_RESPONSE))
# Parse HTTP headers
try:
result.response_headers = parse_headers(result.response_headers)
except Exception as e:
trace = traceback.format_exc()
result.failures.append(Failure(message="Header parsing exception: {0}".format(e), details=trace, failure_type=validators.FAILURE_TEST_EXCEPTION))
result.passed = False
curl.close()
return result
# print str(test_config.print_bodies) + ',' + str(not result.passed) + ' , ' + str(test_config.print_bodies or not result.passed)
head = result.response_headers
# execute validator on body
if result.passed is True:
body = result.body
if mytest.validators is not None and isinstance(mytest.validators, list):
logger.debug("executing this many validators: " + str(len(mytest.validators)))
failures = result.failures
for validator in mytest.validators:
validate_result = validator.validate(body=body, headers=head, context=my_context)
if not validate_result:
result.passed = False
# Proxy for checking if it is a Failure object, because of import issues with isinstance there
if hasattr(validate_result, 'details'):
failures.append(validate_result)
# TODO add printing of validation for interactive mode
else:
logger.debug("no validators found")
# Only do context updates if test was successful
mytest.update_context_after(result.body, my_context)
# Print response body if override is set to print all *OR* if test failed (to capture maybe a stack trace)
if test_config.print_bodies or not result.passed:
if test_config.interactive:
print("RESPONSE:")
print(result.body.decode("string-escape"))
# TODO add string escape on body output
logger.debug(result)
curl.close()
return result
