def __form_body(self, variables) -> str or dict:
if self.method == 'get':
return False, None
body = self.body
if body is None:
body = read_file(fill_template_str(self.file, variables))
if isinstance(body, dict): # dump body to json to be able fill templates in
body = json.dumps(body)
isjson = 'tojson' in body
return isjson, fill_template(body, variables, isjson=isjson)
def _prepare_include_vars(test, global_variables):
if test.include:
include_vars = try_get_object(
fill_template_str(test.include.variables, global_variables))
else:
include_vars = {}
override_keys = report_override(test.variables, include_vars)
if override_keys:
debug('Include variables override these variables: ' +
str(override_keys))
return include_vars
def populate(self,
variables,
conf=None,
schema=None,
data: dict = None,
use_json=False,
**kwargs):
"""
:Input: Populate database with prepared scripts (DDL or CSV with data).
:populate: - populate a database with the data and/or run DDL to create schema.
:conf: postgres configuration. Can be a single line string or object. **Required**.
- dbname: name of the database to connect to
- user: database user
- host: database host
- password: user's password
- port: database port
:schema: path to the schema file. *Optional*
:data: dictionary with keys = tables and values - paths to csv files with data. *Optional*
:use_json: try to recognize json strings and convert them to json. *Optional*, default is false.
:F.e.:
populate postgres
::
variables:
pg_schema: schema.sql
pg_data:
foo: foo.csv
bar: bar.csv
steps:
- prepare:
populate:
postgres:
conf: {{ pg_conf }}
schema: {{ pg_schema }}
data: {{ pg_data }}
use_json: true
"""
resources = variables['RESOURCES_DIR']
if schema is not None:
with open(os.path.join(resources, schema)) as fd:
ddl_sql = fill_template_str(fd.read(), variables)
self.__execute(conf, ddl_sql)
if data is not None and data:
for table_name, path_to_csv in data.items():
self.__populate_csv(conf, table_name,
os.path.join(resources, path_to_csv),
variables, use_json)
def action(self, includes: dict, variables: dict) -> dict:
filled_vars = dict([(k, fill_template_str(v, variables))
for (k, v) in self.variables.items()])
out = fill_template_str(self.include, variables)
test, tag = get_tag(out, self.tag)
if test not in includes:
error('No include registered for name ' + test)
raise Exception('No include registered for name ' + test)
include = includes[test]
variables = merge_two_dicts(include.variables,
merge_two_dicts(variables, filled_vars))
include.variables = variables
try:
variables = include.run(tag=tag, raise_stop=True)
except StopException as e:
raise e
except Exception as e:
if not self.ignore_errors:
raise Exception('Step run ' + test + ' failed: ' + str(e))
return variables
def simple_input(self, variables):
"""
Use this method to get simple input as python object, with all
templates filled in
:param variables:
:return: python object
"""
json_args = fill_template_str(json.dumps(self.data), variables)
return try_get_objects(json_args)
def __run_actions(self, includes, variables: dict) -> dict:
output = variables
for action in self.do_action:
try:
output = action.action(includes, output)
except Exception as e:
if action.ignore_errors:
debug('{} got {} but we ignore it'.format(
fill_template_str(action.name, variables), e))
break
raise e
return output
def _form_request(self, url, variables: dict) -> dict:
headers = dict([(fill_template_str(k, variables),
fill_template_str(v, variables))
for k, v in self.headers.items()])
rq = dict(verify=self.verify,
headers=headers,
files=self.__form_files(variables))
isjson, body = self.__form_body(variables)
debug('http ' + str(self.method) + ' ' + str(url) + ', ' +
str(headers) + ', ' + str(body))
content_type = self.__get_content_type(headers)
if isjson or isinstance(body,
dict): # contains tojson or dict supplied
if isinstance(body, dict) and content_type == 'application/json':
# json body formed manually via python dict
rq['json'] = body
else: # json string or form-data dict
rq['data'] = body
else: # raw body (or body is None)
rq['data'] = body
rq['timeout'] = self.timeout
return rq
def action(self, includes: dict, variables: dict) -> any:
# if virtual host is not specified default it to /
config = try_get_objects(fill_template_str(self.config, variables))
if config.get('virtualhost') is None:
config['virtualhost'] = ''
disconnect_timeout = int(config.get('disconnect_timeout', 10)) # 10 sec for connection closed exception
connection_parameters = self._get_connection_parameters(config)
if self.method == 'publish':
message = self.form_body(self.message, self.file, variables)
return variables, self.publish(connection_parameters,
fill_template_str(self.exchange, variables),
fill_template_str(self.routing_key, variables),
fill_template(self.headers, variables),
message,
disconnect_timeout)
elif self.method == 'consume':
return variables, self.consume(connection_parameters,
fill_template_str(self.queue, variables),
disconnect_timeout)
else:
raise AttributeError('unknown method: ' + self.method)
def action(self, includes: dict, variables: dict) -> tuple:
if self.source_file: # read from file
resources = variables['RESOURCES_DIR']
out = fill_template_str(
read_file(os.path.join(resources, self.source_file)),
variables)
else:
out = fill_template(self.source, variables)
if self.dst is None:
info(out)
else:
dst = fill_template(self.dst, variables)
with open(join(self.path, dst), 'w') as f:
f.write(str(out))
return variables, out
def __form_body(self, variables) -> tuple:
if self.method == 'get':
return False, None
body = self.body
if body is None and self.file is not None:
resources = variables['RESOURCES_DIR']
body = file_utils.read_file(
fill_template_str(os.path.join(resources, self.file),
variables))
if isinstance(body, dict) or isinstance(
body, list): # dump body to json to be able fill templates in
body = json.dumps(body)
if body is None:
return False, None
isjson = 'tojson' in body
return isjson, fill_template(body, variables, isjson=isjson)
def prepare_variables(self, test, global_variables: dict):
"""
Create variables for test
:param global_variables:
:param test: test with test variables
"""
# system env + inventory
# (test template is filled in based on system, inventory & cmd vars)
test.variables = try_get_object(
fill_template_str(test.variables,
merge_two_dicts(global_variables,
self._cmd_env)))
# test local
global_variables.update(test.variables)
# include vars override test local
global_variables.update(
self._prepare_include_vars(test, global_variables))
# cmd_env override everything
global_variables.update(self._cmd_env)
test.variables = global_variables
def __init__(self,
path: str,
system_environment=None,
inventory_vars: dict = None,
cmd_env: Optional[dict] = None,
resources: Optional[dict] = None) -> None:
system_vars = system_environment or {}
if system_vars:
debug('Use system variables: ' + str(list(system_vars.keys())))
self._variables = system_vars
else:
self._variables = {}
inventory = try_get_object(
fill_template_str(inventory_vars,
self._variables)) # fill env vars
self._cmd_env = cmd_env or {}
self._variables.update(inventory)
self._variables['CURRENT_DIR'] = path
self._variables['RESOURCES_DIR'] = resources or os.path.join(
path, 'resources')
def action(self, includes: dict, variables: dict) -> dict:
output = variables
if self.type == 'while':
operator = Operator.find_operator(self.if_clause)
while operator.operation(output):
output = self.__run_actions(includes, output)
if self.max_cycle is not None:
if self.max_cycle == 0:
break
self.max_cycle = self.max_cycle - 1
return output
elif self.type == 'foreach':
loop_var = try_get_objects(
fill_template_str(json.dumps(self.in_var), variables))
if not isinstance(loop_var, collections.Iterable):
raise ValueError(str(loop_var) + ' is not iterable')
for entry in loop_var:
output['ITEM'] = entry
output = self.__run_actions(includes, output)
return output
def run_tests(self) -> bool:
try:
self._compose.up()
if self.system_vars:
debug('Use system variables: ' +
str(list(self.system_vars.keys())))
variables = self.system_vars
else:
variables = {}
if self.inventory is not None:
inv_vars = read_source_file(self.inventory)
inv_vars['INVENTORY'] = get_filename(self.inventory)
variables = try_get_object(
fill_template_str(inv_vars, variables)) # fill env vars
variables['CURRENT_DIR'] = self.path
variables[
'RESOURCES_DIR'] = self.resources or self.path + '/resources'
test_files = get_files(self.tests_path)
results = []
for file in test_files:
self.all_includes = []
try:
variables['TEST_NAME'] = file
test = self.prepare_test(file, variables)
logger.log_storage.test_start(file)
test.run()
results.append(True)
info('Test ' + file + ' passed.')
logger.log_storage.test_end(file, True)
except Exception as e:
warning('Test ' + file + ' failed: ' + str(e))
results.append(False)
logger.log_storage.test_end(file, False, str(e))
return all(results)
finally:
logger.log_storage.write_report(self.path)
self._compose.down()
def __read_n_fill_csv(cls, csv_path, variables):
with open(csv_path) as csv_file:
csv_content = fill_template_str(csv_file.read(),
variables).replace('\n\n', '\n')
return StringIO(csv_content)
def get_action_name(action_type: str, action: Step, variables: dict):
if action.name is not None:
return fill_template_str(action.name, variables)
return action_type
def __form_body(self, variables):
data = self.data
if data is None:
data = read_file(fill_template_str(self.file, variables))
return fill_template_str(data, variables)
