def fill_connections(inventory, conf, dialect, fernet_key):
"""
Populate airflow connections based on catcher's inventory file
:param inventory: path to inventory file
:param conf: db configuration
:param dialect: dialect
:param fernet_key: is used in passwords encryption
:return:
"""
inv_dict = file_utils.read_source_file(inventory)
engine = db_utils.get_engine(conf, dialect)
with engine.connect() as connection:
for name, value in inv_dict.items():
try:
if isinstance(value, dict) and 'type' in value:
if _check_conn_id_exists(name, connection):
debug('{} already exists'.format(name))
continue
query, params = _prepare_connection(value, fernet_key)
from sqlalchemy.sql import text
connection.execute(text(query), name=name, **params)
debug('{} added'.format(name))
else:
debug('{} ignored. No type specified'.format(name))
except Exception as e:
warning('Can\'t add {}:{} - {}'.format(name, value, e))
def run_tests(self) -> bool:
try:
self._compose.up()
variables = {}
if self.inventory is not None:
variables = read_source_file(self.inventory)
variables['INVENTORY'] = get_filename(self.inventory)
variables = try_get_object(fill_template_str(
variables, {})) # fill env vars
variables['CURRENT_DIR'] = self.path
test_files = get_files(self.tests_path)
results = []
for file in test_files:
self.all_includes = []
try:
variables['TEST_NAME'] = file
test = self.prepare_test(file, variables)
test.run()
results.append(True)
info('Test ' + file + ' passed.')
except Exception as e:
warning('Test ' + file + ' failed: ' + str(e))
results.append(False)
return all(results)
finally:
self._compose.down()
def read_inventory(self) -> dict:
if self.inventory is not None:
inv_vars = read_source_file(self.inventory)
inv_vars['INVENTORY'] = get_filename(self.inventory)
inv_vars['INVENTORY_FILE'] = self.inventory
return inv_vars
return {}
def read_test(self, test_file: str):
body = read_source_file(test_file)
return Test(self.path,
test_file,
includes=body.get('include', []),
variables=body.get('variables', {}),
config=body.get('config', {}),
steps=body.get('steps', []),
final=body.get('finally', []),
ignore=body.get('ignore', False))
def prepare_test(self,
file: str,
variables: dict,
override_vars: None or dict = None) -> Test:
body = read_source_file(file)
registered_includes = self.process_includes(body.get('include', []),
variables)
variables = merge_two_dicts(variables, body.get(
'variables', {})) # override variables with test's variables
if override_vars: # TODO warn when overriding inventory vars?
variables = merge_two_dicts(variables, override_vars)
return Test(self.path, registered_includes, variables,
body.get('config', {}), body.get('steps', []),
self.modules, self.environment)
def prepare_test(self,
file: str,
variables: dict,
override_vars: None or dict = None) -> Test:
body = read_source_file(file)
registered_includes = self.process_includes(body.get('include', []),
variables)
variables = merge_two_dicts(variables, body.get(
'variables', {})) # override variables with test's variables
if override_vars:
override_keys = report_override(variables, override_vars)
if override_keys:
debug('Overriding these variables: ' + str(override_keys))
variables = merge_two_dicts(variables, override_vars)
return Test(
self.path,
registered_includes,
deepcopy(variables), # each test has independent variables
body.get('config', {}),
body.get('steps', []),
self.modules,
self.environment)
def run_tests(self) -> bool:
try:
self._compose.up()
if self.system_vars:
debug('Use system variables: ' +
str(list(self.system_vars.keys())))
variables = self.system_vars
else:
variables = {}
if self.inventory is not None:
inv_vars = read_source_file(self.inventory)
inv_vars['INVENTORY'] = get_filename(self.inventory)
variables = try_get_object(
fill_template_str(inv_vars, variables)) # fill env vars
variables['CURRENT_DIR'] = self.path
variables[
'RESOURCES_DIR'] = self.resources or self.path + '/resources'
test_files = get_files(self.tests_path)
results = []
for file in test_files:
self.all_includes = []
try:
variables['TEST_NAME'] = file
test = self.prepare_test(file, variables)
logger.log_storage.test_start(file)
test.run()
results.append(True)
info('Test ' + file + ' passed.')
logger.log_storage.test_end(file, True)
except Exception as e:
warning('Test ' + file + ' failed: ' + str(e))
results.append(False)
logger.log_storage.test_end(file, False, str(e))
return all(results)
finally:
logger.log_storage.write_report(self.path)
self._compose.down()
def test_read_yaml_simple(self):
self.__populate_files(['test_file.yaml'])
file = join(self.test_dir, 'test_file.yaml')
result = read_source_file(file)
self.assertEqual(self.yaml_parsed, result)