def __init__(self, conf_folder):
self._logger = logging.getLogger(self.__class__.__name__)
self._conf_files = list(
map(lambda x: os.path.join(conf_folder, x),
self.CONF_FILES + ['deployment_config.yaml']))
self._schemas = self.SCHEMAS + [DEPLOYMENTCONFIG_SCHEMA]
self._confreader = ConfReader(self._conf_files, self._schemas)
self._deployers = deployer_factory(self._confreader)
def test_conf_reader_valid_nested_extra_field(self):
"""This function tests behaviour of ConfReader when
the configuration schema allows a nested field that is not defined
in properties and the configuration contains
such a field (here: "author": "last_name")."""
book_schema = {
"type": "object",
"properties": {
"title": {
"type": "string"
},
"author": {
"type": "object",
"properties": {
"first_name": {
"type": "string"
},
},
"required": ["first_name"],
"maxProperties": 2
},
"isbn": {
"type": "string"
}
},
"required": ["title", "author", "isbn"],
"maxProperties": 3
}
cr_valid = ConfReader([os.path.join('tests', 'examples', 'book.yaml')],
[book_schema])
self.assertIsInstance(cr_valid, ConfReader)
self.assertEqual(cr_valid['book']['author']['last_name'], 'Waltari')
def test_conf_reader_invalid_field_typo(self):
"""This function tests behaviour of ConfReader when
the configuration schema requires exact fields and
the configuration contains a typo (here, schema requires
"deleted", but configuration has "delete")."""
deployment_config_schema = {
"type": "object",
"properties": {
"method": {
"type": "string"
},
"deleted": {
"type": "boolean"
},
"set_sbit": {
"type": "boolean"
},
"target_host": {
"type": "string"
},
},
"required": ["method", "deleted", "set_sbit", "target_host"],
"maxProperties": 4
}
with self.assertRaises(ValidationError):
cr_invalid = ConfReader(
[os.path.join('tests', 'examples', 'deployment_config.yaml')],
[deployment_config_schema])
print(cr_invalid)
def test_conf_reader_valid_nested_default(self):
"""This function tests behaviour of ConfReader when
configuration schema contains nested fields and the
schema matches the configuration."""
book_schema = {
"type": "object",
"properties": {
"title": {
"type": "string"
},
"author": {
"type": "object",
"properties": {
"first_name": {
"type": "string"
},
"last_name": {
"type": "string"
}
},
"required": ["first_name", "last_name"],
"maxProperties": 2
},
"isbn": {
"type": "string"
}
},
"required": ["title", "author", "isbn"],
"maxProperties": 3
}
cr_valid = ConfReader([os.path.join('tests', 'examples', 'book.yaml')],
[book_schema])
self.assertIsInstance(cr_valid, ConfReader)
self.assertEqual(cr_valid['book']['author']['last_name'], 'Waltari')
def test_conf_reader_invalid_extra_field(self):
"""This function tests behaviour of ConfReader when
the configuration schema doesn't allow fields other than
those defined in properties and the configuration contains
such a field (here: "method")."""
deployment_config_schema = {
"type": "object",
"properties": {
"delete": {
"type": "boolean"
},
"set_sbit": {
"type": "number"
},
"target_host": {
"type": "string"
},
},
"required": ["delete", "set_sbit", "target_host"],
"maxProperties": 3
}
with self.assertRaises(ValidationError):
cr_invalid = ConfReader(
[os.path.join('tests', 'examples', 'deployment_config.yaml')],
[deployment_config_schema])
print(cr_invalid)
def test_conf_reader_invalid_wrong_field_type(self):
"""This function tests behaviour of ConfReader when
a required field has the wrong type."""
deployment_config_schema = {
"type": "object",
"properties": {
"method": {
"type": "string"
},
"delete": {
"type": "boolean"
},
"set_sbit": {
"type": "number"
},
"target_host": {
"type": "string"
},
},
"required": ["method", "delete", "set_sbit", "target_host"],
"maxProperties": 4
}
with self.assertRaises(ValidationError):
cr_invalid = ConfReader(
[os.path.join('tests', 'examples', 'deployment_config.yaml')],
[deployment_config_schema])
print(cr_invalid)
def test_conf_reader_valid_default(self):
"""This function tests behaviour of ConfReader when
configuration schema matches the configuration."""
deployment_config = EXAMPLE_CONFIGS['deployment_config']
deployment_config_schema = copy.deepcopy(
EXAMPLE_SCHEMAS['deployment_config'])
cr_valid = ConfReader([deployment_config], [deployment_config_schema])
self.assertIsInstance(cr_valid, ConfReader)
self.assertEqual(cr_valid['deployment_config']['method'], 'rsync')
def _get_auths(self):
auths_file = os.path.expanduser(self._deployer_config.get(
'auths_file',
os.path.join('~', 'os_auths.yaml')))
auths_name = os.path.splitext(os.path.basename(auths_file))[0]
auths = {}
if os.path.isfile(auths_file):
auths.update(ConfReader([auths_file],[self.AUTH_SCHEMA])[auths_name]['auths'])
return auths
def test_conf_reader_valid_extra_field(self):
"""This function tests behaviour of ConfReader when
the configuration schema allows a field that is not defined
in properties and the configuration contains
such a field (here: "method")."""
deployment_config = EXAMPLE_CONFIGS['deployment_config']
deployment_config_schema = copy.deepcopy(
EXAMPLE_SCHEMAS['deployment_config'])
del deployment_config_schema['properties']['method']
deployment_config_schema['required'] = [
"delete", "set_sbit", "target_host"
]
cr_valid = ConfReader([deployment_config], [deployment_config_schema])
self.assertIsInstance(cr_valid, ConfReader)
self.assertEqual(cr_valid['deployment_config']['method'], 'rsync')
def test_conf_reader_invalid_missing_field(self):
"""This function tests behaviour of ConfReader when
a required field is missing from the configuration."""
deployment_config_schema = copy.deepcopy(
EXAMPLE_SCHEMAS['deployment_config'])
deployment_config_schema["properties"]["important_extra_field"] = {
"type": "string"
}
deployment_config_schema["required"] = [
"method", "delete", "set_sbit", "target_host",
"important_extra_field"
]
deployment_config_schema["maxProperties"] = 5
with self.assertRaises(ValidationError):
cr_invalid = ConfReader(
[os.path.join('tests', 'examples', 'deployment_config.yaml')],
[deployment_config_schema])
print(cr_invalid)
def test_conf_reader_invalid_nested_field_missing(self):
"""This function tests behaviour of ConfReader when
a required nested field is missing from the
configuration (here: "author": "year_of_birth" is missing)."""
book_schema = {
"type": "object",
"properties": {
"title": {
"type": "string"
},
"author": {
"type": "object",
"properties": {
"first_name": {
"type": "string"
},
"last_name": {
"type": "string"
},
"year_of_birth": {
"type": "number"
}
},
"required": ["first_name", "last_name", "year_of_birth"],
"maxProperties": 3
},
"isbn": {
"type": "string"
}
},
"required": ["title", "author", "isbn"],
"maxProperties": 3
}
with self.assertRaises(ValidationError):
cr_invalid = ConfReader(
[os.path.join('tests', 'examples', 'book.yaml')],
[book_schema])
print(cr_invalid)
class Builder:
"""This superclass will create a build based on buildrules.
Builder is initialized based on given configuration files and schemas of
said configurations. It checks the validity of the configurations using
ConfReader. Deployment strategy is based on 'deployment_config' and
created using deployer_factory.
'buildrules' are created using _get_rules-function. Subclasses of Builder
should overwrite this function.
An overview of the whole build can be obtained with describe-function.
Build is initialized by running the Builder. By specifying dry_run no
changes are made, but the output is presented.
Args:
conf_folder (str): Configuration folder that contains configuration
files.
"""
BUILDER_NAME = 'None'
CONF_FILES = []
SCHEMAS = []
@log_error_and_quit
def __init__(self, conf_folder):
self._logger = logging.getLogger(self.__class__.__name__)
self._conf_files = list(
map(lambda x: os.path.join(conf_folder, x),
self.CONF_FILES + ['deployment_config.yaml']))
self._schemas = self.SCHEMAS + [DEPLOYMENTCONFIG_SCHEMA]
self._confreader = ConfReader(self._conf_files, self._schemas)
self._deployers = deployer_factory(self._confreader)
def _skip_rule(self, step):
return step in self._confreader.get('build_config',
{}).get('skip_rules', [])
def __call__(self, dry_run=False):
"""This function will execute all _build_rules."""
rules = self._get_rules()
for deployer in self._deployers:
rules = rules + deployer.get_rules()
for rule in rules:
try:
rule(dry_run=dry_run)
except RuleError as e:
self._logger.error(
'Encountered an error while executing BuildRule: {0}: {1}'.
format(rule, e))
sys.exit(1)
def _get_rules(self):
""""""
return []
@log_error_and_quit
def describe(self):
""""""
self._logger.info('Builder: {0}'.format(self.BUILDER_NAME))
self._logger.info('Configuration files: {0}'.format(
' '.join(self.CONF_FILES + ['deployment_config.yaml'])))
self._logger.debug(str(self._confreader))
rules = self._get_rules()
self._logger.info('Build rule descriptions:')
for rule in rules:
self._logger.info(rule)
deployer_rules = []
deployers = deployer_factory(self._confreader)
for deployer in deployers:
deployer_rules = deployer_rules + deployer.get_rules()
self._logger.info('Deployment descriptions:')
for rule in deployer_rules:
self._logger.info(rule)
class Builder:
"""This superclass will create a build based on buildrules.
Builder is initialized based on given configuration files and schemas of
said configurations. It checks the validity of the configurations using
ConfReader. Deployment strategy is based on 'deployment_config' and
created using deployer_factory.
'buildrules' are created using _get_rules-function. Subclasses of Builder
should overwrite this function.
An overview of the whole build can be obtained with describe-function.
Build is initialized by running the Builder. By specifying dry_run no
changes are made, but the output is presented.
Args:
conf_folder (str): Configuration folder that contains configuration
files.
"""
BUILDER_NAME = 'None'
CONF_FILES = []
SCHEMAS = []
@log_error_and_quit
def __init__(self, conf_folder):
self._logger = logging.getLogger(self.__class__.__name__)
self._conf_files = list(
map(lambda x: os.path.join(conf_folder, x),
self.CONF_FILES + ['deployment_config.yaml']))
self._schemas = self.SCHEMAS + [DEPLOYMENTCONFIG_SCHEMA]
self._confreader = ConfReader(self._conf_files, self._schemas)
self._deployers = deployer_factory(self._confreader)
def _skip_rule(self, step):
return step in self._confreader.get('build_config',
{}).get('skip_rules', [])
def __call__(self, dry_run=False):
"""This function will execute all _build_rules."""
rules = self._get_rules()
for deployer in self._deployers:
rules = rules + deployer.get_rules()
for rule in rules:
try:
rule(dry_run=dry_run)
except RuleError as e:
self._logger.error(
'Encountered an error while executing BuildRule: {0}: {1}'.
format(rule, e))
sys.exit(1)
def _get_rules(self):
""""""
return []
@log_error_and_quit
def describe(self):
""""""
self._logger.info('Builder: {0}'.format(self.BUILDER_NAME))
self._logger.info('Configuration files: {0}'.format(
' '.join(self.CONF_FILES + ['deployment_config.yaml'])))
self._logger.debug(str(self._confreader))
rules = self._get_rules()
self._logger.info('Build rule descriptions:')
for rule in rules:
self._logger.info(rule)
deployer_rules = []
deployers = deployer_factory(self._confreader)
for deployer in deployers:
deployer_rules = deployer_rules + deployer.get_rules()
self._logger.info('Deployment descriptions:')
for rule in deployer_rules:
self._logger.info(rule)
@classmethod
def _makedirs(cls, path, chmod=None):
""" This function creates a folder with requested permissions
Args:
path (str): Folder to create.
chmod (str): Chmod permissions. Default is None.
"""
try:
os.makedirs(path)
if chmod:
os.chmod(path, chmod)
except FileExistsError:
pass
@classmethod
def _copy_file(cls, src, target, chmod=None):
""" This function copies a file from src to target with required
permissions.
Args:
src (str): File that will be copied.
target (str): Target folder / file.
chmod (str): Chmod permissions. Default is None.
"""
copy2(src, target)
if chmod:
os.chmod(target, chmod)
@classmethod
def _copy_dir(cls, src, target, chmod=None):
""" This function copies a folder from src to target with required
permissions.
Args:
src (str): Folder that will be copied.
target (str): Target folder.
chmod (str): Chmod permissions. Default is None.
"""
copytree(src, target, symlinks=True)
if chmod:
os.chmod(target, chmod)
def _write_template(self, target_path, template_path=None, template=None):
"""Writes a file based on jinja2-template.
Args:
target_path (str): Target path to fill.
template_path (str): Template file to use. Default None.
template (str): jinja2-template as a string. Default None.
"""
if not template:
with open(template_path, 'r') as template_file:
template = ''.join(template_file.readlines())
filled_template = self._fill_template(template)
with open(target_path, 'w') as target_file:
target_file.write(filled_template)
def _fill_template(self, template):
"""Fills a jinja2-template based on build_config.
Args:
template (str): jinja2-template as a string.
Returns:
str: Filled template.
"""
return Template(template).render(self._confreader['build_config'])
def _calculate_file_checksum(self, filename, hash_function='sha256'):
hash_functions = {'sha256': hashlib.sha256}
hash_function = hash_functions[hash_function]()
with open(filename, "rb") as input_file:
for byte_block in iter(lambda: input_file.read(4096), b""):
hash_function.update(byte_block)
return hash_function.hexdigest()
def _calculate_dict_checksum(self, dict_object, hash_function='sha256'):
hash_functions = {'sha256': hashlib.sha256}
hash_function = hash_functions[hash_function]()
json_dump = json.dumps(dict_object, ensure_ascii=False, sort_keys=True)
hash_function.update(json_dump.encode('utf-8'))
return hash_function.hexdigest()