def parse_generator(configuration):
""" Parses a configuration built from yaml and returns a generator
Configuration should be a map
"""
from py3resttest.utils import Parser
configuration = Parser.lowercase_keys(Parser.flatten_dictionaries(configuration))
gen_type = str(configuration.get(u'type')).lower()
if gen_type not in GENERATOR_TYPES:
raise ValueError(
'Generator type given {0} is not valid '.format(gen_type))
# Do the easy parsing, delegate more complex logic to parsing functions
if gen_type == 'env_variable':
return factory_env_variable(configuration['variable_name'])()
elif gen_type == 'env_string':
return factory_env_string(configuration['string'])()
elif gen_type == 'number_sequence':
start = int(configuration.get('start', 1))
increment = int(configuration.get('increment', 1))
return factory_generate_ids(start, increment)()
elif gen_type == 'random_int':
return generator_random_int32()
elif gen_type == 'random_text':
return parse_random_text_generator(configuration)
elif gen_type in GENERATOR_TYPES:
return GENERATOR_PARSING[gen_type](configuration)
raise Exception("Unknown generator type: {0}".format('gen_type'))
def test_encode_unicode_bytes(self):
decode_str = Parser.encode_unicode_bytes('my😽')
self.assertEqual(decode_str, bytes('my😽', 'utf-8'))
decode_str = Parser.encode_unicode_bytes(bytes('my😽', 'utf-8'))
self.assertEqual(decode_str, bytes('my😽', 'utf-8'))
decode_str = Parser.encode_unicode_bytes('hello')
self.assertEqual(decode_str, bytes('hello', 'utf-8'))
def url(self, value):
if isinstance(value, dict):
# this is the templated url , we need to convert it into actual URL
template_str = Parser.lowercase_keys(value)['template']
self.set_template("url", Parser.coerce_to_string(template_str))
self.__url = value
else:
self.__url = value
def parse(config):
from py3resttest.utils import Parser
output = ExtractTestValidator()
config = Parser.lowercase_keys(Parser.flatten_dictionaries(config))
output.config = config
extractor = _get_extractor(config)
output.extractor = extractor
test_name = config['test']
output.test_name = test_name
test_fn = VALIDATOR_TESTS[test_name]
output.test_fn = test_fn
return output
def parse(self, testcase_dict):
testcase_dict = Parser.flatten_lowercase_keys_dict(testcase_dict)
for keyword in TestCase.KEYWORD_DICT.keys():
value = testcase_dict.get(keyword)
if value is None:
continue
if keyword == TestCaseKeywords.auth_username:
self.auth_username = value
elif keyword == TestCaseKeywords.auth_password:
self.auth_password = value
elif keyword == TestCaseKeywords.method:
self.http_method = value
elif keyword == TestCaseKeywords.delay:
self.__delay = int(value)
elif keyword == TestCaseKeywords.group:
self.__group = value
elif keyword == TestCaseKeywords.name:
self.__name = value
elif keyword == TestCaseKeywords.url:
self.url = value
elif keyword == TestCaseKeywords.extract_binds:
self.extract_binds = value
elif keyword == TestCaseKeywords.validators:
self.validators = value
elif keyword == TestCaseKeywords.headers:
self.headers = value
elif keyword == TestCaseKeywords.variable_binds:
self.__variable_binds_dict = Parser.flatten_dictionaries(value)
elif keyword == TestCaseKeywords.generator_binds:
self.__generator_binds_dict = {
str(k): str(v)
for k, v in Parser.flatten_dictionaries(value)
}
elif keyword == TestCaseKeywords.options:
raise NotImplementedError("Yet to Support")
elif keyword == TestCaseKeywords.body:
self.body = value
elif keyword == TestCaseKeywords.absolute_urls:
self.__abs_url = Parser.safe_to_bool(value)
expected_status = testcase_dict.get(TestCaseKeywords.expected_status,
[])
if expected_status:
self.expected_http_status_code_list = expected_status
else:
if self.http_method in ["POST", "PUT", "DELETE"]:
self.expected_http_status_code_list = [200, 201, 204]
def test_safe_to_json(self):
class Example:
x = 1
y = 2
result = Parser.safe_to_json(Example)
self.assertEqual(result, {'x': 1, 'y': 2})
result = Parser.safe_to_json("Example1")
self.assertEqual(result, "Example1")
result = Parser.safe_to_json(bytes("Example", "utf-8"))
self.assertEqual(result, "Example")
result = Parser.safe_to_json(1)
self.assertEqual(result, "1")
def test_parse_headers(self):
request_text = (b'GET /who/ken/trust.html HTTP/1.1\r\n'
b'Host: cm.bell-labs.com\r\n'
b'Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.3\r\n'
b'Accept: text/html;q=0.9,text/plain\r\n'
b'\r\n')
result_list = Parser.parse_headers(request_text)
self.assertEqual(3, len(result_list))
self.assertEqual(('host', 'cm.bell-labs.com'), result_list[0])
request_text = ""
result_list = Parser.parse_headers(request_text)
self.assertEqual(0, len(result_list))
request_text = '\r\n'
result_list = Parser.parse_headers(request_text)
self.assertEqual(0, len(result_list))
def parse(cls, config):
validator = JsonSchemaValidator()
config = Parser.lowercase_keys(config)
if 'schema' not in config:
raise ValueError(
"Cannot create schema validator without a 'schema' configuration element!")
validator.schema_context = ContentHandler.parse_content(config['schema'])
return validator
def test_flatten_lowercase_keys_dict(self):
input_dict = {"x": 1, "y": 2}
result_dict = Parser.flatten_lowercase_keys_dict([{
'x': 2
}, input_dict])
self.assertEqual(input_dict, result_dict)
input_dict = {"X": 1, "y": 2}
result_dict = Parser.flatten_lowercase_keys_dict([{
'x': 2
}, input_dict])
self.assertEqual({'x': 1, 'y': 2}, result_dict)
input_dict = {"X": 1, "y": 2}
result_dict = Parser.flatten_lowercase_keys_dict(input_dict)
self.assertEqual({'x': 1, 'y': 2}, result_dict)
input_dict = 22 # unexpected
result_dict = Parser.flatten_lowercase_keys_dict(input_dict)
self.assertEqual(22, result_dict)
def headers(self, headers):
config_value = Parser.flatten_dictionaries(headers)
if isinstance(config_value, dict):
for key, value in config_value.items():
if isinstance(value, dict):
if value.get('template'):
self.set_template("headers", value.get('template'))
self.__header_dict.update(config_value)
else:
raise ValidatorError(
"Illegal header type: headers must be a dictionary or list of dictionary keys"
)
def parse(self, config_node):
node = Parser.flatten_lowercase_keys_dict(config_node)
for key, value in node.items():
if key == 'timeout':
self.timeout = int(value)
elif key == u'print_bodies':
self.print_bodies = Parser.safe_to_bool(value)
elif key == 'retries':
self.retries = int(value)
elif key == 'variable_binds':
self.variable_binds = value
elif key == u'generators':
if not isinstance(value, list):
raise TypeError(
"generators in config should defined as list(array).")
flat = Parser.flatten_dictionaries(value)
gen_dict = {}
for generator_name, generator_config in flat.items():
gen = parse_generator(generator_config)
gen_dict[str(generator_name)] = gen
self.generators = gen_dict
def extract_binds(self, bind_dict):
bind_dict = Parser.flatten_dictionaries(bind_dict)
for variable_name, extractor in bind_dict.items():
if not isinstance(extractor, dict) or len(extractor) == 0:
raise BindError(
"Extractors must be defined as maps of extractorType:{configs} with 1 entry"
)
if len(extractor) > 1:
raise BindError(
"Cannot define multiple extractors for given variable name"
)
for extractor_type, extractor_config in extractor.items():
self.__extract_binds_dict[variable_name] = parse_extractor(
extractor_type, extractor_config)
def get_content(self, context=None):
""" Does all context binding and pathing to get content, templated out """
if self.is_file:
path = self.content
if self.is_template_path and context:
path = string.Template(path).safe_substitute(
context.get_values())
with open(path, 'r') as f:
data = f.read()
if self.is_template_content and context:
return string.Template(data).safe_substitute(context.get_values())
else:
return data
else:
if self.is_template_content and context:
return Parser.safe_substitute_unicode_template(self.content, context.get_values())
else:
return self.content
def test_flatten_dictionaries(self):
input_dict = {"x": 1, "y": 2}
result_dict = Parser.flatten_dictionaries(input_dict)
self.assertEqual(input_dict, result_dict)
input_dict.update({"y": {"a": 1}})
result_dict = Parser.flatten_dictionaries(input_dict)
self.assertEqual(input_dict, result_dict)
result_dict = Parser.flatten_dictionaries(
[input_dict, input_dict, input_dict])
self.assertEqual(input_dict, result_dict)
result_dict = Parser.flatten_dictionaries([input_dict])
self.assertEqual(input_dict, result_dict)
result_dict = Parser.flatten_dictionaries([{'x': 1}, input_dict])
self.assertEqual(input_dict, result_dict)
result_dict = Parser.flatten_dictionaries([input_dict, {'x': 2}])
self.assertNotEqual(input_dict, result_dict)
result_dict = Parser.flatten_dictionaries([{'x': 2}, input_dict])
self.assertEqual(input_dict, result_dict)
def parse(config):
""" Create a validator that does an extract from body and applies a comparator,
Then does comparison vs expected value
Syntax sample:
{ jsonpath_mini: 'node.child',
operator: 'eq',
expected: 'myValue'
}
"""
from py3resttest.utils import Parser
output = ComparatorValidator()
config = Parser.lowercase_keys(Parser.flatten_dictionaries(config))
output.config = config
output.extractor = _get_extractor(config)
if output.extractor is None:
raise ValueError(
"Extract function for comparison is not valid or not found!")
if 'comparator' not in config: # Equals comparator if unspecified
output.comparator_name = 'eq'
else:
output.comparator_name = config['comparator'].lower()
try:
output.comparator = COMPARATORS[output.comparator_name]
except KeyError:
raise ValueError("Invalid comparator given! %s "
"available options are %s" %
(output.comparator_name, COMPARATORS.keys()))
if not output.comparator:
raise ValueError("Invalid comparator given!")
try:
expected = config['expected']
except KeyError:
raise ValueError(
"No expected value found in comparator validator config, one must be!"
)
# Expected value can be another extractor query, or a single value, or
# a templated value
if isinstance(expected, str) or isinstance(expected,
(int, float, complex)):
output.expected = expected
elif isinstance(expected, dict):
expected = Parser.lowercase_keys(expected)
template = expected.get('template')
if template: # Templated string
if not isinstance(template, str):
raise ValueError(
"Can't template a comparator-validator unless template value is a string"
)
output.is_template_expected = True
output.expected = template
else: # Extractor to compare against
output.expected = _get_extractor(expected)
if not output.expected:
raise ValueError(
"Can't supply a non-template, non-extract dictionary to comparator-validator"
)
return output
def test_coerce_string_to_ascii(self):
result = Parser.coerce_string_to_ascii(bytes("Hello", 'utf-8'))
self.assertEqual(result, "Hello".encode('ascii'))
def test_lowercase_keys(self):
input_val = 23
result_dict = Parser.lowercase_keys(input_val)
self.assertEqual(23, result_dict)
def variable_binds(self, variable_dict):
"""Variable binding """
if isinstance(variable_dict, dict):
self.__variable_binds_dict.update(
Parser.flatten_dictionaries(variable_dict))
def parse_content(node):
""" Parse content from input node and returns ContentHandler object
it'll look like:
- template:
- file:
- temple: path
or something
"""
# Tread carefully, this one is a bit narly because of nesting
output = ContentHandler()
is_template_path = False
is_template_content = False
is_file = False
is_done = False
if not isinstance(node, (str, dict, list)):
raise TypeError(
"Content must be a string, dictionary, or list of dictionaries"
)
while node and not is_done: # Dive through the configuration tree
# Finally we've found the value!
if isinstance(node, str):
output.content = node
output.setup(node,
is_file=is_file,
is_template_path=is_template_path,
is_template_content=is_template_content)
return output
is_done = True
# Dictionary or list of dictionaries
flat_dict = Parser.flatten_lowercase_keys_dict(node)
for key, value in flat_dict.items():
if key == 'template':
if isinstance(value, str):
if is_file:
value = os.path.abspath(value)
output.content = value
is_template_content = is_template_content or not is_file
output.is_template_content = is_template_content
output.is_template_path = is_file
output.is_file = is_file
return output
else:
is_template_content = True
node = value
is_done = False
break
elif key == 'file':
if isinstance(value, str):
output.content = os.path.abspath(value)
output.is_file = True
output.is_template_content = is_template_content
return output
else:
is_file = True
node = value
is_done = False
break
raise Exception("Invalid configuration for content.")
def generator_binds(self, value: Dict):
binds_dict = Parser.flatten_dictionaries(value)
__binds_dict = {str(k): str(v) for k, v in binds_dict.items()}
self.__generator_binds_dict.update(__binds_dict)
def test_safe_substitute_unicode_template(self):
result = Parser.safe_substitute_unicode_template(
"This is $x test", {'x': 'unit'})
self.assertEqual("This is unit test", result)
def auth_password(self, password):
self.__auth_password = Parser.coerce_string_to_ascii(password)
def auth_username(self, username):
self.__auth_username = Parser.coerce_string_to_ascii(username)
def run(self, context=None, timeout=None, curl_handler=None):
if context is None:
context = self.__context
self.pre_update(context)
self.render()
if timeout is None:
timeout = DEFAULT_TIMEOUT
if curl_handler:
try: # Check the curl handle isn't closed, and reuse it if possible
curl_handler.getinfo(curl_handler.HTTP_CODE)
# Below clears the cookies & curl options for clean run
# But retains the DNS cache and connection pool
curl_handler.reset()
curl_handler.setopt(curl_handler.COOKIELIST, "ALL")
except pycurl.error:
curl_handler = pycurl.Curl()
else:
curl_handler = pycurl.Curl()
body_byte, header_byte = self.__default_curl_config(
curl_handler, timeout)
if self.config.timeout:
curl_handler.setopt(pycurl.CONNECTTIMEOUT, self.config.timeout)
if self.__ssl_insecure:
curl_handler.setopt(pycurl.SSL_VERIFYPEER, 0)
curl_handler.setopt(pycurl.SSL_VERIFYHOST, 0)
if self.body:
logger.debug("Request body %s" % self.body)
curl_handler.setopt(curl_handler.READFUNCTION,
BytesIO(bytes(self.body, 'utf-8')).read)
if self.auth_username and self.auth_password:
curl_handler.setopt(pycurl.USERPWD,
self.auth_username + ':' + self.auth_password)
self.__configure_curl_method(curl_handler)
head = self.headers
self.__configure_curl_headers(curl_handler, head)
if self.__delay:
time.sleep(self.__delay)
try:
logger.info("Hitting %s" % self.url)
curl_handler.perform()
except pycurl.error as e:
logger.error("Unknown Exception", exc_info=True)
self.__passed = False
curl_handler.close()
trace = traceback.format_exc()
self.__failure_list.append(
Failure(message="Curl Exception: {0}".format(e),
details=trace,
failure_type=FAILURE_CURL_EXCEPTION))
return
self.body = body_byte.getvalue()
body_byte.close()
response_code = curl_handler.getinfo(pycurl.RESPONSE_CODE)
self.__response_code = int(response_code)
if self.config.print_bodies:
print(self.body)
try:
response_headers = Parser.parse_headers(header_byte.getvalue())
self.__response_headers = response_headers
logger.debug("RESPONSE HEADERS: %s" % self.__response_headers)
header_byte.close()
except Exception as e: # Need to catch the expected exception
trace = traceback.format_exc()
self.__failure_list.append(
Failure(message="Header parsing exception: {0}".format(e),
details=trace,
failure_type=FAILURE_TEST_EXCEPTION))
self.__passed = False
curl_handler.close()
return
if self.__response_code in self.expected_http_status_code_list:
self.__passed = True
self.__failure_list.extend(self.__perform_validation())
self.post_update(context)
else:
self.__passed = False
failure_message = "Invalid HTTP response code: response code {0} not in expected codes {1}".format(
self.__response_code, self.expected_http_status_code_list)
self.__failure_list.append(
Failure(message=failure_message,
details=None,
failure_type=FAILURE_INVALID_RESPONSE))
curl_handler.close()
def parse(self,
base_url: str,
testcase_list: List,
test_file=None,
working_directory=None,
variable_dict=None):
if working_directory is None:
working_directory = Path(os.path.abspath(os.getcwd()))
else:
working_directory = Path(working_directory)
if variable_dict is None:
self.config.variable_binds = variable_dict
if test_file:
self.__testcase_file.add(test_file)
testcase_config_object = TestCaseConfig()
for testcase_node in testcase_list:
if not isinstance(testcase_node, dict):
logger.warning("Skipping the configuration %s" % testcase_node)
continue
testcase_node = Parser.lowercase_keys(testcase_node)
for key in testcase_node:
sub_testcase_node = testcase_node[key]
if key == YamlKeyWords.INCLUDE:
if not isinstance(sub_testcase_node, list):
raise ValueError("include should be list not %s" %
type(sub_testcase_node))
for testcase_file_path in sub_testcase_node:
testcase_file_path = testcase_file_path.replace(
'.', '/')
testcase_file = str(
working_directory.joinpath(
"%s.yaml" % testcase_file_path).resolve())
if testcase_file not in self.__testcase_file:
self.__testcase_file.add(testcase_file)
import_testcase_list = read_testcase_file(
testcase_file)
with ChangeDir(working_directory):
self.parse(base_url,
import_testcase_list,
variable_dict=variable_dict)
elif key == YamlKeyWords.IMPORT:
if sub_testcase_node not in self.__testcase_file:
testcase_file_path = sub_testcase_node
logger.debug("Importing testcase from %s",
testcase_file_path)
testcase_file_path = str(
working_directory.joinpath(
"%s" % testcase_file_path).resolve())
self.__testcase_file.add(sub_testcase_node)
import_testcase_list = read_testcase_file(
testcase_file_path)
with ChangeDir(working_directory):
self.parse(base_url,
import_testcase_list,
variable_dict=variable_dict)
elif key == YamlKeyWords.URL:
__group_name = TestCaseGroup.DEFAULT_GROUP
group_object = TestSet.__create_test(
__group_name, testcase_config_object)
testcase_object = TestCase(
base_url=base_url,
extract_binds=group_object.extract_binds,
variable_binds=group_object.variable_binds,
context=group_object.context,
config=group_object.config)
testcase_object.url = testcase_node[key]
group_object.testcase_list = testcase_object
elif key == YamlKeyWords.TEST:
with ChangeDir(working_directory):
self.parse_test(base_url, sub_testcase_node,
testcase_config_object)
elif key == YamlKeyWords.CONFIG:
testcase_config_object.parse(sub_testcase_node)
self.config = testcase_config_object
def test_coerce_to_string(self):
result = Parser.coerce_to_string(bytes("Hello", 'utf-8'))
self.assertEqual(result, "Hello")