def __init__(self, conf_folder): self._logger = logging.getLogger(self.__class__.__name__) self._conf_files = list( map(lambda x: os.path.join(conf_folder, x), self.CONF_FILES + ['deployment_config.yaml'])) self._schemas = self.SCHEMAS + [DEPLOYMENTCONFIG_SCHEMA] self._confreader = ConfReader(self._conf_files, self._schemas) self._deployers = deployer_factory(self._confreader)
def test_conf_reader_valid_nested_extra_field(self): """This function tests behaviour of ConfReader when the configuration schema allows a nested field that is not defined in properties and the configuration contains such a field (here: "author": "last_name").""" book_schema = { "type": "object", "properties": { "title": { "type": "string" }, "author": { "type": "object", "properties": { "first_name": { "type": "string" }, }, "required": ["first_name"], "maxProperties": 2 }, "isbn": { "type": "string" } }, "required": ["title", "author", "isbn"], "maxProperties": 3 } cr_valid = ConfReader([os.path.join('tests', 'examples', 'book.yaml')], [book_schema]) self.assertIsInstance(cr_valid, ConfReader) self.assertEqual(cr_valid['book']['author']['last_name'], 'Waltari')
def test_conf_reader_invalid_field_typo(self): """This function tests behaviour of ConfReader when the configuration schema requires exact fields and the configuration contains a typo (here, schema requires "deleted", but configuration has "delete").""" deployment_config_schema = { "type": "object", "properties": { "method": { "type": "string" }, "deleted": { "type": "boolean" }, "set_sbit": { "type": "boolean" }, "target_host": { "type": "string" }, }, "required": ["method", "deleted", "set_sbit", "target_host"], "maxProperties": 4 } with self.assertRaises(ValidationError): cr_invalid = ConfReader( [os.path.join('tests', 'examples', 'deployment_config.yaml')], [deployment_config_schema]) print(cr_invalid)
def test_conf_reader_valid_nested_default(self): """This function tests behaviour of ConfReader when configuration schema contains nested fields and the schema matches the configuration.""" book_schema = { "type": "object", "properties": { "title": { "type": "string" }, "author": { "type": "object", "properties": { "first_name": { "type": "string" }, "last_name": { "type": "string" } }, "required": ["first_name", "last_name"], "maxProperties": 2 }, "isbn": { "type": "string" } }, "required": ["title", "author", "isbn"], "maxProperties": 3 } cr_valid = ConfReader([os.path.join('tests', 'examples', 'book.yaml')], [book_schema]) self.assertIsInstance(cr_valid, ConfReader) self.assertEqual(cr_valid['book']['author']['last_name'], 'Waltari')
def test_conf_reader_invalid_extra_field(self): """This function tests behaviour of ConfReader when the configuration schema doesn't allow fields other than those defined in properties and the configuration contains such a field (here: "method").""" deployment_config_schema = { "type": "object", "properties": { "delete": { "type": "boolean" }, "set_sbit": { "type": "number" }, "target_host": { "type": "string" }, }, "required": ["delete", "set_sbit", "target_host"], "maxProperties": 3 } with self.assertRaises(ValidationError): cr_invalid = ConfReader( [os.path.join('tests', 'examples', 'deployment_config.yaml')], [deployment_config_schema]) print(cr_invalid)
def test_conf_reader_invalid_wrong_field_type(self): """This function tests behaviour of ConfReader when a required field has the wrong type.""" deployment_config_schema = { "type": "object", "properties": { "method": { "type": "string" }, "delete": { "type": "boolean" }, "set_sbit": { "type": "number" }, "target_host": { "type": "string" }, }, "required": ["method", "delete", "set_sbit", "target_host"], "maxProperties": 4 } with self.assertRaises(ValidationError): cr_invalid = ConfReader( [os.path.join('tests', 'examples', 'deployment_config.yaml')], [deployment_config_schema]) print(cr_invalid)
def test_conf_reader_valid_default(self): """This function tests behaviour of ConfReader when configuration schema matches the configuration.""" deployment_config = EXAMPLE_CONFIGS['deployment_config'] deployment_config_schema = copy.deepcopy( EXAMPLE_SCHEMAS['deployment_config']) cr_valid = ConfReader([deployment_config], [deployment_config_schema]) self.assertIsInstance(cr_valid, ConfReader) self.assertEqual(cr_valid['deployment_config']['method'], 'rsync')
def _get_auths(self): auths_file = os.path.expanduser(self._deployer_config.get( 'auths_file', os.path.join('~', 'os_auths.yaml'))) auths_name = os.path.splitext(os.path.basename(auths_file))[0] auths = {} if os.path.isfile(auths_file): auths.update(ConfReader([auths_file],[self.AUTH_SCHEMA])[auths_name]['auths']) return auths
def test_conf_reader_valid_extra_field(self): """This function tests behaviour of ConfReader when the configuration schema allows a field that is not defined in properties and the configuration contains such a field (here: "method").""" deployment_config = EXAMPLE_CONFIGS['deployment_config'] deployment_config_schema = copy.deepcopy( EXAMPLE_SCHEMAS['deployment_config']) del deployment_config_schema['properties']['method'] deployment_config_schema['required'] = [ "delete", "set_sbit", "target_host" ] cr_valid = ConfReader([deployment_config], [deployment_config_schema]) self.assertIsInstance(cr_valid, ConfReader) self.assertEqual(cr_valid['deployment_config']['method'], 'rsync')
def test_conf_reader_invalid_missing_field(self): """This function tests behaviour of ConfReader when a required field is missing from the configuration.""" deployment_config_schema = copy.deepcopy( EXAMPLE_SCHEMAS['deployment_config']) deployment_config_schema["properties"]["important_extra_field"] = { "type": "string" } deployment_config_schema["required"] = [ "method", "delete", "set_sbit", "target_host", "important_extra_field" ] deployment_config_schema["maxProperties"] = 5 with self.assertRaises(ValidationError): cr_invalid = ConfReader( [os.path.join('tests', 'examples', 'deployment_config.yaml')], [deployment_config_schema]) print(cr_invalid)
def test_conf_reader_invalid_nested_field_missing(self): """This function tests behaviour of ConfReader when a required nested field is missing from the configuration (here: "author": "year_of_birth" is missing).""" book_schema = { "type": "object", "properties": { "title": { "type": "string" }, "author": { "type": "object", "properties": { "first_name": { "type": "string" }, "last_name": { "type": "string" }, "year_of_birth": { "type": "number" } }, "required": ["first_name", "last_name", "year_of_birth"], "maxProperties": 3 }, "isbn": { "type": "string" } }, "required": ["title", "author", "isbn"], "maxProperties": 3 } with self.assertRaises(ValidationError): cr_invalid = ConfReader( [os.path.join('tests', 'examples', 'book.yaml')], [book_schema]) print(cr_invalid)
class Builder: """This superclass will create a build based on buildrules. Builder is initialized based on given configuration files and schemas of said configurations. It checks the validity of the configurations using ConfReader. Deployment strategy is based on 'deployment_config' and created using deployer_factory. 'buildrules' are created using _get_rules-function. Subclasses of Builder should overwrite this function. An overview of the whole build can be obtained with describe-function. Build is initialized by running the Builder. By specifying dry_run no changes are made, but the output is presented. Args: conf_folder (str): Configuration folder that contains configuration files. """ BUILDER_NAME = 'None' CONF_FILES = [] SCHEMAS = [] @log_error_and_quit def __init__(self, conf_folder): self._logger = logging.getLogger(self.__class__.__name__) self._conf_files = list( map(lambda x: os.path.join(conf_folder, x), self.CONF_FILES + ['deployment_config.yaml'])) self._schemas = self.SCHEMAS + [DEPLOYMENTCONFIG_SCHEMA] self._confreader = ConfReader(self._conf_files, self._schemas) self._deployers = deployer_factory(self._confreader) def _skip_rule(self, step): return step in self._confreader.get('build_config', {}).get('skip_rules', []) def __call__(self, dry_run=False): """This function will execute all _build_rules.""" rules = self._get_rules() for deployer in self._deployers: rules = rules + deployer.get_rules() for rule in rules: try: rule(dry_run=dry_run) except RuleError as e: self._logger.error( 'Encountered an error while executing BuildRule: {0}: {1}'. format(rule, e)) sys.exit(1) def _get_rules(self): """""" return [] @log_error_and_quit def describe(self): """""" self._logger.info('Builder: {0}'.format(self.BUILDER_NAME)) self._logger.info('Configuration files: {0}'.format( ' '.join(self.CONF_FILES + ['deployment_config.yaml']))) self._logger.debug(str(self._confreader)) rules = self._get_rules() self._logger.info('Build rule descriptions:') for rule in rules: self._logger.info(rule) deployer_rules = [] deployers = deployer_factory(self._confreader) for deployer in deployers: deployer_rules = deployer_rules + deployer.get_rules() self._logger.info('Deployment descriptions:') for rule in deployer_rules: self._logger.info(rule)
class Builder: """This superclass will create a build based on buildrules. Builder is initialized based on given configuration files and schemas of said configurations. It checks the validity of the configurations using ConfReader. Deployment strategy is based on 'deployment_config' and created using deployer_factory. 'buildrules' are created using _get_rules-function. Subclasses of Builder should overwrite this function. An overview of the whole build can be obtained with describe-function. Build is initialized by running the Builder. By specifying dry_run no changes are made, but the output is presented. Args: conf_folder (str): Configuration folder that contains configuration files. """ BUILDER_NAME = 'None' CONF_FILES = [] SCHEMAS = [] @log_error_and_quit def __init__(self, conf_folder): self._logger = logging.getLogger(self.__class__.__name__) self._conf_files = list( map(lambda x: os.path.join(conf_folder, x), self.CONF_FILES + ['deployment_config.yaml'])) self._schemas = self.SCHEMAS + [DEPLOYMENTCONFIG_SCHEMA] self._confreader = ConfReader(self._conf_files, self._schemas) self._deployers = deployer_factory(self._confreader) def _skip_rule(self, step): return step in self._confreader.get('build_config', {}).get('skip_rules', []) def __call__(self, dry_run=False): """This function will execute all _build_rules.""" rules = self._get_rules() for deployer in self._deployers: rules = rules + deployer.get_rules() for rule in rules: try: rule(dry_run=dry_run) except RuleError as e: self._logger.error( 'Encountered an error while executing BuildRule: {0}: {1}'. format(rule, e)) sys.exit(1) def _get_rules(self): """""" return [] @log_error_and_quit def describe(self): """""" self._logger.info('Builder: {0}'.format(self.BUILDER_NAME)) self._logger.info('Configuration files: {0}'.format( ' '.join(self.CONF_FILES + ['deployment_config.yaml']))) self._logger.debug(str(self._confreader)) rules = self._get_rules() self._logger.info('Build rule descriptions:') for rule in rules: self._logger.info(rule) deployer_rules = [] deployers = deployer_factory(self._confreader) for deployer in deployers: deployer_rules = deployer_rules + deployer.get_rules() self._logger.info('Deployment descriptions:') for rule in deployer_rules: self._logger.info(rule) @classmethod def _makedirs(cls, path, chmod=None): """ This function creates a folder with requested permissions Args: path (str): Folder to create. chmod (str): Chmod permissions. Default is None. """ try: os.makedirs(path) if chmod: os.chmod(path, chmod) except FileExistsError: pass @classmethod def _copy_file(cls, src, target, chmod=None): """ This function copies a file from src to target with required permissions. Args: src (str): File that will be copied. target (str): Target folder / file. chmod (str): Chmod permissions. Default is None. """ copy2(src, target) if chmod: os.chmod(target, chmod) @classmethod def _copy_dir(cls, src, target, chmod=None): """ This function copies a folder from src to target with required permissions. Args: src (str): Folder that will be copied. target (str): Target folder. chmod (str): Chmod permissions. Default is None. """ copytree(src, target, symlinks=True) if chmod: os.chmod(target, chmod) def _write_template(self, target_path, template_path=None, template=None): """Writes a file based on jinja2-template. Args: target_path (str): Target path to fill. template_path (str): Template file to use. Default None. template (str): jinja2-template as a string. Default None. """ if not template: with open(template_path, 'r') as template_file: template = ''.join(template_file.readlines()) filled_template = self._fill_template(template) with open(target_path, 'w') as target_file: target_file.write(filled_template) def _fill_template(self, template): """Fills a jinja2-template based on build_config. Args: template (str): jinja2-template as a string. Returns: str: Filled template. """ return Template(template).render(self._confreader['build_config']) def _calculate_file_checksum(self, filename, hash_function='sha256'): hash_functions = {'sha256': hashlib.sha256} hash_function = hash_functions[hash_function]() with open(filename, "rb") as input_file: for byte_block in iter(lambda: input_file.read(4096), b""): hash_function.update(byte_block) return hash_function.hexdigest() def _calculate_dict_checksum(self, dict_object, hash_function='sha256'): hash_functions = {'sha256': hashlib.sha256} hash_function = hash_functions[hash_function]() json_dump = json.dumps(dict_object, ensure_ascii=False, sort_keys=True) hash_function.update(json_dump.encode('utf-8')) return hash_function.hexdigest()