def _create_ansible_config(self):
"""
Create ansible configuration file
:return: None
"""
try:
method = self._cli_conf.get(CTFCliConfig.ANSIBLE_SECTION_NAME, CTFCliConfig.CONFIG_ANSIBLE_METHOD)
host = self._cli_conf.get(CTFCliConfig.ANSIBLE_SECTION_NAME, CTFCliConfig.CONFIG_ANSIBLE_HOST)
user = self._cli_conf.get(CTFCliConfig.ANSIBLE_SECTION_NAME, CTFCliConfig.CONFIG_ANSIBLE_USER)
except NoSectionError as e:
raise CTFCliError("No configuration for 'ansible' provided!")
except NoOptionError as e:
raise CTFCliError("Wrong ansible configuration: {0}".format(str(e)))
# Optional parameters
try:
sudo = self._cli_conf.get(CTFCliConfig.ANSIBLE_SECTION_NAME, CTFCliConfig.CONFIG_ANSIBLE_SUDO)
except NoOptionError as e:
sudo = False
ansible_conf_path = os.path.join(self._working_dir, 'ansible.conf')
ansible_conf_content = "[ctf]\n{host} ansible_connection={method} ansible_ssh_user={user} ansible_sudo={sudo}\n".format(
host=host, method=method, user=user, sudo=sudo
)
logger.debug("Writing ansible configuration to '%s'\n%s", ansible_conf_path, ansible_conf_content)
with open(ansible_conf_path, 'w') as f:
f.write(ansible_conf_content)
self._exec_type_conf_path = ansible_conf_path
def check_and_add_init_py(path, skip_root=False):
"""
Checks if __init__.py is in all subdirs under the given path
:param path: root of the path where to begin
:param skip_root: Whether to skip the root directory
:return: list of paths for created __init__.py files
"""
files = []
for (dirpath, dirnames, filenames) in os.walk(path, followlinks=True):
if skip_root and dirpath == path:
continue
# skip hidden directories and their subdirs
dirs = [x for x in dirpath.replace(path, '').split(os.sep) if x.startswith('.')]
if dirs:
logger.debug("Skipping, since dirpath '%s' contains hidden dir: %s", dirpath, str(dirs))
continue
if '__init__.py' not in filenames:
new_file = os.path.join(dirpath, '__init__.py')
logger.debug("Creating __init__.py inside '%s'", dirpath)
files.append(new_file)
with open(new_file, 'w') as f:
f.write('# -*- coding: utf-8 -*-\n# File generated by Containers Testing Framework\n')
return files
def get_import_statements(path):
"""
Generate import statements
:param path: path to dir from which to generate import statements
:return: list of strings containing import statement
"""
imports = []
for (dirpath, dirnames, filenames) in os.walk(path, followlinks=True):
module = dirpath.replace(path, '').strip(os.sep).replace(os.sep, '.')
# generate imports for the *.py files in the current dir
# skip hidden directories and their subdirs
dirs = [x for x in dirpath.replace(path, '').split(os.sep) if x.startswith('.')]
if dirs:
logger.debug("Skipping, since dirpath '%s' contains hidden dir: %s", dirpath, str(dirs))
continue
for f in filenames:
# skip NON Python files
if not f.endswith('.py'):
continue
if f == '__init__.py':
continue
logger.debug("Adding import for '%s'", os.path.join(dirpath, f))
imports.append('from {0}.{1} import *'.format(module, f.replace('.py', '')))
return imports
def __init__(self, cli_args=None):
"""
The Application implementation.
"""
self._execution_dir_path = os.getcwd()
self._working_dir_path = os.path.join(
self._execution_dir_path, '{0}-behave-working-dir'.format(
os.path.basename(self._execution_dir_path)))
self._working_dir = None
self._behave_runner = None
if not cli_args.cli_config_path:
cli_args.cli_config_path = CTFCliConfig.find_cli_config(
self._execution_dir_path)
self._cli_conf = CTFCliConfig(cli_args)
# If no Dockerfile passed on the cli, try to use one from the execution directory
if not self._cli_conf.get(CTFCliConfig.GLOBAL_SECTION_NAME,
CTFCliConfig.CONFIG_DOCKERFILE):
local_file = os.path.join(self._execution_dir_path, 'Dockerfile')
if not os.path.isfile(local_file):
raise CTFCliError(
"No Dockerfile passed on the cli and no Dockerfile "
"is present in the current directory!")
logger.debug("Using Dockerfile from the current directory.")
self._cli_conf.set(CTFCliConfig.GLOBAL_SECTION_NAME,
CTFCliConfig.CONFIG_DOCKERFILE, local_file)
# TODO: Remove this or rework, once more types are implemented
if self._cli_conf.get(CTFCliConfig.GLOBAL_SECTION_NAME,
CTFCliConfig.CONFIG_EXEC_TYPE) != 'ansible':
raise CTFCliError(
"Wrong ExecType configured. Currently only 'ansible' is supported!"
)
def run():
try:
# add application-wide debug log
LoggerHelper.add_debug_log_file(os.getcwd())
args = ArgumentsParser(sys.argv[1:])
if args.verbose is True:
LoggerHelper.add_stream_handler(logger,
logging.Formatter('%(levelname)s:\t%(message)s'),
logging.DEBUG)
else:
LoggerHelper.add_stream_handler(logger,
logging.Formatter('%(levelname)s:\t%(message)s'),
logging.INFO)
app = Application(args)
app.run()
except KeyboardInterrupt:
logger.info('Interrupted by user')
except CTFCliError as e:
logger.error('%s', str(e))
sys.exit(1)
else:
sys.exit(0)
finally:
logger.debug('Exiting...')
def get_import_statements(path):
"""
Generate import statements
:param path: path to dir from which to generate import statements
:return: list of strings containing import statement
"""
imports = []
for (dirpath, dirnames, filenames) in os.walk(path, followlinks=True):
module = dirpath.replace(path, '').strip(os.sep).replace(os.sep, '.')
# generate imports for the *.py files in the current dir
# skip hidden directories and their subdirs
dirs = [x for x in dirpath.replace(path, '').split(os.sep) if x.startswith('.')]
if dirs:
logger.debug("Skipping, since dirpath '%s' contains hidden dir: %s", dirpath, str(dirs))
continue
for f in filenames:
# skip NON Python files
if not f.endswith('.py'):
continue
if f == '__init__.py':
continue
logger.debug("Adding import for '%s'", os.path.join(dirpath, f))
imports.append('from {0}.{1} import *'.format(module, f.replace('.py', '')))
return imports
def check_and_add_init_py(path, skip_root=False):
"""
Checks if __init__.py is in all subdirs under the given path
:param path: root of the path where to begin
:param skip_root: Whether to skip the root directory
:return: list of paths for created __init__.py files
"""
files = []
for (dirpath, _, filenames) in os.walk(path, followlinks=True):
if skip_root and dirpath == path:
continue
# skip hidden directories and their subdirs
dirs = [
x for x in dirpath.replace(path, '').split(os.sep)
if x.startswith('.')
]
if dirs:
logger.debug(
"Skipping, since dirpath '%s' contains hidden dir: %s",
dirpath, str(dirs))
continue
if '__init__.py' not in filenames:
new_file = os.path.join(dirpath, '__init__.py')
logger.debug("Creating __init__.py inside '%s'", dirpath)
files.append(new_file)
with open(new_file, 'w') as f:
f.write(
'# -*- coding: utf-8 -*-\n# File generated by Containers Testing Framework\n'
)
return files
def _create_ansible_config(self):
"""
Create ansible configuration file
:return: None
"""
try:
method = self._cli_conf.get(CTFCliConfig.ANSIBLE_SECTION_NAME, CTFCliConfig.CONFIG_ANSIBLE_METHOD)
host = self._cli_conf.get(CTFCliConfig.ANSIBLE_SECTION_NAME, CTFCliConfig.CONFIG_ANSIBLE_HOST)
user = self._cli_conf.get(CTFCliConfig.ANSIBLE_SECTION_NAME, CTFCliConfig.CONFIG_ANSIBLE_USER)
sudo = self._cli_conf.get(CTFCliConfig.ANSIBLE_SECTION_NAME, CTFCliConfig.CONFIG_ANSIBLE_SUDO)
except NoSectionError as e:
raise CTFCliError("No configuration for 'ansible' provided!")
except NoOptionError as e:
raise CTFCliError("Wrong ansible configuration: {0}".format(str(e)))
ansible_conf_path = os.path.join(self._working_dir, 'ansible.conf')
ansible_conf_content = "[ctf]\n{host} ansible_connection={method} ansible_ssh_user={user} ansible_sudo={sudo}\n".format(
host=host, method=method, user=user, sudo=sudo
)
logger.debug("Writing ansible configuration to '%s'\n%s", ansible_conf_path, ansible_conf_content)
with open(ansible_conf_path, 'w') as f:
f.write(ansible_conf_content)
self._exec_type_conf_path = ansible_conf_path
def _write_environment_py(self):
"""
Writes common environment.py inside working_dir/. The file will import
the copied project specific environment file if present.
:return:
"""
project_env_py = glob.glob(
os.path.join(self._working_dir, '*environment.py'))
if project_env_py:
module_name = os.path.basename(project_env_py[0]).replace(
'.py', '').replace('-', '_')
import_statement = 'from {0} import *\n'.format(module_name)
import_statement += 'import {0}'.format(module_name)
else:
import_statement = ''
# create the environment.py
with open(os.path.join(self._working_dir, 'environment.py'), 'w') as f, \
open(os.path.join(__path__[0], 'common_environment_content.py'), 'r') as src:
logger.debug("Writing '%s'",
os.path.join(self._working_dir, 'environment.py'))
f.write(
common_environment_py_header.format(
project_env_py_import=import_statement))
f.write(src.read())
def run(self):
"""
The main application execution method
"""
logger.info("Running Containers Testing Framework cli")
# If no Dockerfile passed on the cli, try to use one from the execution directory
if not self._cli_conf.get(CTFCliConfig.GLOBAL_SECTION_NAME, CTFCliConfig.CONFIG_DOCKERFILE):
local_file = os.path.join(self._execution_dir_path, 'Dockerfile')
if not os.path.isfile(local_file):
raise CTFCliError("No Dockerfile passed on the cli and no Dockerfile "
"is present in the current directory!")
logger.debug("Using Dockerfile from the current directory.")
self._cli_conf.set(CTFCliConfig.GLOBAL_SECTION_NAME, CTFCliConfig.CONFIG_DOCKERFILE, local_file)
# TODO: Remove this or rework, once more types are implemented
if self._cli_conf.get(CTFCliConfig.GLOBAL_SECTION_NAME, CTFCliConfig.CONFIG_EXEC_TYPE) != 'ansible':
raise CTFCliError("Wrong ExecType configured. Currently only 'ansible' is supported!")
self._working_dir = BehaveWorkingDirectory(self._working_dir_path, self._cli_conf)
# Setup Behave structure inside working directory
# Clone common Features and steps into the working dir
# Add the project specific Features and steps
# Prepare the steps.py in the Steps dir that combines all the other
self._working_dir.setup()
# Execute Behave
self._behave_runner = BehaveRunner(self._working_dir, self._cli_conf)
return self._behave_runner.run()
def _setup_dir_structure(self):
"""
Create directory structure and create git repo where appropriate
:return:
"""
logger.debug("Setting up Behave working directory")
for d in (self._features_dir, self._steps_dir):
os.mkdir(d)
def _setup_dir_structure(self):
"""
Create directory structure and create git repo where appropriate
:return:
"""
logger.debug("Setting up Behave working directory")
for d in (self._features_dir, self._steps_dir):
os.mkdir(d)
def _check_working_dir(self):
"""
Check if working directory exists. Remove it if it exists and then recreate. Create it if it does not exist
"""
if os.path.exists(self._working_dir):
logger.debug("Working directory '%s' exists. Removing it!", self._working_dir)
shutil.rmtree(self._working_dir)
logger.debug("Creating working directory '%s'.", self._working_dir)
os.mkdir(self._working_dir)
def __init__(self, conf_path):
self._config = ConfigParser()
self._config_path = conf_path
try:
self._config.read(self._config_path)[0]
except IndexError:
logger.warning("Tests configuration '%s' can not be read!", conf_path)
else:
logger.debug("Using Tests configuration from '%s'.", conf_path)
def _check_working_dir(self):
"""
Check if working directory exists. Remove it if it exists and then recreate. Create it if it does not exist
"""
if os.path.exists(self._working_dir):
logger.debug("Working directory '%s' exists. Removing it!", self._working_dir)
shutil.rmtree(self._working_dir)
logger.debug("Creating working directory '%s'.", self._working_dir)
os.mkdir(self._working_dir)
def __init__(self, conf_path):
self._config = ConfigParser()
self._config_path = conf_path
try:
self._config.read(self._config_path)[0]
except IndexError:
logger.warning("Tests configuration '%s' can not be read!", conf_path)
else:
logger.debug("Using Tests configuration from '%s'.", conf_path)
def __init__(self, cli_conf):
self._config = ConfigParser()
self._add_commandline_arguments(cli_conf)
if cli_conf.cli_config_path:
config_abs_path = os.path.abspath(cli_conf.cli_config_path)
try:
self._config.read(config_abs_path)[0]
except IndexError:
logger.warning("Configuration file '%s' could not be read... "
"Using ONLY default settings", config_abs_path)
else:
logger.debug("Using configuration from '%s'", config_abs_path)
def _add_remote_features(self):
"""
Add all remote features
:return:
"""
for test in self._tests_conf.get_tests():
remote_repo = self._tests_conf.get_test_features(test)
local_dir = os.path.join(self._features_dir, '{0}_features'.format(test).replace('-', '_'))
logger.info("Using remote Features from '%s'", remote_repo)
logger.debug("Cloning remote test Features from '%s' to '%s'", remote_repo, local_dir)
try:
check_call(['git', 'clone', '-q', remote_repo, local_dir])
except CalledProcessError as e:
raise CTFCliError("Cloning of {0} failed!\n{1}".format(remote_repo, str(e)))
def __init__(self, cli_conf):
self._config = ConfigParser()
self._add_commandline_arguments(cli_conf)
if cli_conf.cli_config_path:
config_abs_path = os.path.abspath(cli_conf.cli_config_path)
try:
self._config.read(config_abs_path)[0]
except IndexError:
logger.warning(
"Configuration file '%s' could not be read... "
"Using ONLY default settings", config_abs_path)
else:
logger.debug("Using configuration from '%s'", config_abs_path)
def _add_remote_features(self):
"""
Add all remote features
:return:
"""
for test in self._tests_conf.get_tests():
remote_repo = self._tests_conf.get_test_features(test)
local_dir = os.path.join(self._features_dir, '{0}_features'.format(test).replace('-', '_'))
logger.info("Using remote Features from '%s'", remote_repo)
logger.debug("Cloning remote test Features from '%s' to '%s'", remote_repo, local_dir)
try:
check_call(['git', 'clone', '-q', remote_repo, local_dir])
except CalledProcessError as e:
raise CTFCliError("Cloning of {0} failed!\n{1}".format(remote_repo, str(e)))
def _create_ansible_config(self):
"""
Create ansible configuration file
:return: None
"""
script = None
method = None
host = None
user = None
try:
script = self._cli_conf.get(CTFCliConfig.ANSIBLE_SECTION_NAME,
CTFCliConfig.CONFIG_ANSIBLE_DYNAMIC_SCRIPT)
except NoSectionError:
raise CTFCliError("No configuration for 'ansible' provided!")
except NoOptionError:
logger.debug("No dynamic provision script found")
script = None
if not script:
try:
method = self._cli_conf.get(CTFCliConfig.ANSIBLE_SECTION_NAME,
CTFCliConfig.CONFIG_ANSIBLE_METHOD)
host = self._cli_conf.get(CTFCliConfig.ANSIBLE_SECTION_NAME,
CTFCliConfig.CONFIG_ANSIBLE_HOST)
user = self._cli_conf.get(CTFCliConfig.ANSIBLE_SECTION_NAME,
CTFCliConfig.CONFIG_ANSIBLE_USER)
except NoOptionError:
logger.debug("No dynamic provision script found")
# Optional parameters
try:
sudo = self._cli_conf.get(CTFCliConfig.ANSIBLE_SECTION_NAME,
CTFCliConfig.CONFIG_ANSIBLE_SUDO)
except NoOptionError:
sudo = False
ansible_conf_path = None
if script:
ansible_conf_path = os.path.join(self._working_dir, script)
shutil.copy(os.path.abspath(script), self._working_dir)
logger.debug("Using ansible dynamic configuration from '%s'", ansible_conf_path)
else:
ansible_conf_path = os.path.join(self._working_dir, 'ansible.conf')
ansible_conf_content = "[ctf]\n{host} ansible_connection={method} ansible_ssh_user={user} ansible_sudo={sudo}\n".format(
host=host, method=method, user=user, sudo=sudo
)
logger.debug("Writing ansible configuration to '%s'\n%s",
ansible_conf_path, ansible_conf_content)
with open(ansible_conf_path, 'w') as f:
f.write(ansible_conf_content)
self._exec_type_conf_path = ansible_conf_path
def _write_environment_py(self):
"""
Writes common environment.py inside working_dir/. The file will import
the copied project specific environment file if present.
:return:
"""
project_env_py = glob.glob(os.path.join(self._working_dir, '*environment.py'))
if project_env_py:
import_statement = 'from {0} import *'.format(os.path.basename(project_env_py[0]).replace('.py', ''))
else:
import_statement = ''
# create the environment.py
with open(os.path.join(self._working_dir, 'environment.py'), 'w') as f:
logger.debug("Writing '%s'", os.path.join(self._working_dir, 'environment.py'))
f.write(common_environment_py_header.format(project_env_py_import=import_statement))
f.write(common_environment_py_content)
def find_tests_config(tests_path):
"""
Find a tests config in the tests directory
:param tests_path: path to tests/ directory containing Features and Steps
:return: path to a config file or None if not found
"""
logger.debug("Looking for tests configuration inside '%s'", tests_path)
f = glob.glob(os.path.join(tests_path, 'test*.ini'))
logger.debug(str(f))
if not f:
f = glob.glob(os.path.join(tests_path, 'test*.conf'))
logger.debug(str(f))
if not f:
logger.debug("Didn't find any tests configuration file!")
return None
else:
logger.debug("Found configuration file: %s", str(f))
return os.path.join(tests_path, f[0])
def find_tests_config(tests_path):
"""
Find a tests config in the tests directory
:param tests_path: path to tests/ directory containing Features and Steps
:return: path to a config file or None if not found
"""
logger.debug("Looking for tests configuration inside '%s'", tests_path)
f = glob.glob(os.path.join(tests_path, 'test*.ini'))
logger.debug(str(f))
if not f:
f = glob.glob(os.path.join(tests_path, 'test*.conf'))
logger.debug(str(f))
if not f:
logger.debug("Didn't find any tests configuration file!")
return None
else:
logger.debug("Found configuration file: %s", str(f))
return os.path.join(tests_path, f[0])
def _combine_steps(self):
"""
Generate steps.py inside the working_dir/steps/ which imports everything from
:return:
"""
steps_py_content = """# -*- coding: utf-8 -*-
# This file is automatically generated by Containers Testing Framework
# Any changes to this file will be discarded
from behave import *
""".splitlines()
imports = self.get_import_statements(self._steps_dir)
logger.debug("Using steps imports: %s", str(imports))
steps_py_content.extend(imports)
added_files = self.check_and_add_init_py(self._steps_dir)
logger.debug("Created __init__.py files: %s", str(added_files))
# create the steps.py
with open(os.path.join(self._steps_dir, 'steps.py'), 'w') as f:
logger.debug("Writing '%s'",
os.path.join(self._steps_dir, 'steps.py'))
f.write('\n'.join(steps_py_content) + '\n')
def run():
try:
# add application-wide debug log
LoggerHelper.add_debug_log_file(os.getcwd())
args = ArgumentsParser(sys.argv[1:])
if args.verbose is True:
LoggerHelper.add_stream_handler(logger,
logging.Formatter('%(levelname)s:\t%(message)s'),
logging.DEBUG)
else:
LoggerHelper.add_stream_handler(logger,
logging.Formatter('%(levelname)s:\t%(message)s'),
logging.INFO)
app = Application(args)
if 'init' in args.cli_action:
app.init()
if 'run' in args.cli_action:
app.run()
if 'update' in args.cli_action:
app.update()
if 'remote' in args.cli_action:
if 'add' in args.remote_action:
app.add_remote()
if 'remove' in args.remote_action:
app.remove_remote()
if 'list' in args.remote_action:
app.list_remotes()
except KeyboardInterrupt:
logger.info('Interrupted by user')
except CTFCliError as e:
logger.error('%s', str(e))
sys.exit(1)
else:
sys.exit(0)
finally:
logger.debug('Exiting...')
def _combine_steps(self):
"""
Generate steps.py inside the working_dir/steps/ which imports everything from
:return:
"""
steps_py_content = """# -*- coding: utf-8 -*-
# This file is automatically generated by Containers Testing Framework
# Any changes to this file will be discarded
from behave import *
""".splitlines()
imports = self.get_import_statements(self._steps_dir)
logger.debug("Using steps imports: %s", str(imports))
steps_py_content.extend(imports)
added_files = self.check_and_add_init_py(self._steps_dir)
logger.debug("Created __init__.py files: %s", str(added_files))
# create the steps.py
with open(os.path.join(self._steps_dir, 'steps.py'), 'w') as f:
logger.debug("Writing '%s'", os.path.join(self._steps_dir, 'steps.py'))
f.write('\n'.join(steps_py_content) + '\n')