class ValidationConfiguration(JSONSerialisableClass): """ Represents the validation configuration. Attributes: output_file_directory (str): The directory to which validation results will be saved. output_file_name (str): The name of the file to save containing the validation results. validation_analyses (List[AssetClass]): A list of asset class objects containing all validations to perform for the asset class. """ _serialisable_lists = { 'asset_classes': AssetClass, } _validation_schema = Schema({ Required("output_file_directory"): IsDir(), Required("output_file_name"): str, Required("asset_classes"): [AssetClass._validation_schema], }) def __init__(self, **kwargs): self.output_file_directory = None # type: str self.output_file_name = None # type: str self.asset_classes = [] # type: List[AssetClass] super().__init__(**kwargs) @classmethod def load_from_file(cls, file_path: str) -> 'ValidationConfiguration': """ Loads settings from a validation configuration file. Args: file_path: The file path of the validation configuration file. Returns: A ValidationConfiguration object containing the settings from the specified configuration file. """ with open(file_path) as config_file: config = json.load(config_file) config = cls._decode_json(config) # type: 'ValidationConfiguration' return config def save_to_file(self, file_path: str): """ Saves the configuration settings specified to a file. Args: file_path: The file path to which to save the configuration file. """ with open(file_path, 'w') as validation_configuration_file: json.dump(self._encode_json(), validation_configuration_file, indent=4) def validate(self): """ Validates the configuration. """ json_obj = self._encode_json() self._validation_schema(json_obj)
def validate_paths(options): # Refs Vagrant code: # https://github.com/mitchellh/vagrant/blob/9c299a2a357fcf87f356bb9d56e18a037a53d138/ # plugins/provisioners/puppet/config/puppet.rb#L112 if options.get('manifests_path') is not None: host_manifest_file = str( Path(options['manifests_path']) / options['manifest_file']) IsFile(msg="File {} does not exist".format(host_manifest_file))(host_manifest_file) elif options.get('environment_path') is not None: host_selected_environment_path = str( Path(options['environment_path']) / options['environment']) IsDir(msg="Directory {} does not exist".format(host_selected_environment_path))( host_selected_environment_path) return options
class PuppetProvisioner(Provisioner): """ Allows to perform provisioning operations using Puppet. """ name = 'puppet' guest_required_packages_arch = ['puppet'] guest_required_packages_debian = ['puppet'] guest_required_packages_fedora = ['puppet'] guest_required_packages_ubuntu = ['puppet'] # Refs Vagrant docs: # https://www.vagrantup.com/docs/provisioning/puppet_apply.html#options schema = All( { 'binary_path': str, 'facter': dict, 'hiera_config_path': IsFile(), 'manifest_file': str, 'manifests_path': IsDir(), 'module_path': IsDir(), 'environment': str, 'environment_path': IsDir(), 'environment_variables': dict, 'options': str, }, finalize_options, validate_paths) _guest_manifests_path = '/.lxdock.d/puppet/manifests' _guest_module_path = '/.lxdock.d/puppet/modules' _guest_default_module_path = '/etc/puppet/modules' _guest_environment_path = '/.lxdock.d/puppet/environments' _guest_hiera_file = '/.lxdock.d/puppet/hiera.yaml' def provision_single(self, guest): """ Performs the provisioning operations using puppet. """ # Verify if `puppet` has been installed. binary_path = self.options.get('binary_path') if binary_path is not None: puppet_bin = str(PurePosixPath(binary_path) / 'puppet') retcode = guest.run(['test', '-x', puppet_bin]) fail_msg = ( "puppet executable is not found in the specified path {} in the " "guest container. ".format(binary_path)) else: retcode = guest.run(['which', 'puppet']) fail_msg = ( "puppet was not automatically installed for this guest. " "Please specify the command to install it in LXDock file using " "a shell provisioner and try `lxdock provision` again. You may " "also specify `binary_path` that contains the puppet executable " "in LXDock file.") if retcode != 0: raise ProvisionFailed(fail_msg) # Copy manifests dir manifests_path = self.options.get('manifests_path') if manifests_path is not None: guest.copy_directory(Path(manifests_path), PurePosixPath(self._guest_manifests_path)) # Copy module dir module_path = self.options.get('module_path') if module_path is not None: guest.copy_directory(Path(module_path), PurePosixPath(self._guest_module_path)) # Copy environment dir environment_path = self.options.get('environment_path') if environment_path is not None: guest.copy_directory(Path(environment_path), PurePosixPath(self._guest_environment_path)) # Copy hiera file hiera_file = self.options.get('hiera_config_path') if hiera_file is not None: guest.copy_file(Path(hiera_file), PurePosixPath(self._guest_hiera_file)) # Run puppet. command = self._build_puppet_command() if environment_path: logger.info("Running Puppet with environment {}...".format( self.options['environment'])) else: logger.info("Running Puppet with {}...".format( self.options['manifest_file'])) guest.run(['sh', '-c', ' '.join(command)]) ################################## # PRIVATE METHODS AND PROPERTIES # ################################## def _build_puppet_command(self): """ Refs: https://github.com/mitchellh/vagrant/blob/9c299a2a357fcf87f356bb9d56e18a037a53d138/ plugins/provisioners/puppet/provisioner/puppet.rb#L173 """ options = self.options.get('options', '') options = list(map(shlex.quote, shlex.split(options))) module_path = self.options.get('module_path') if module_path is not None: options += [ '--modulepath', '{}:{}'.format(self._guest_module_path, self._guest_default_module_path) ] hiera_path = self.options.get('hiera_config_path') if hiera_path is not None: options += ['--hiera_config={}'.format(self._guest_hiera_file)] # TODO: we are not detecting console color support now, but please contribute if you need! options += ['--detailed-exitcodes'] environment_path = self.options.get('environment_path') if environment_path is not None: options += [ '--environmentpath', str(self._guest_environment_path), '--environment', self.options['environment'] ] else: options += ['--manifestdir', str(self._guest_manifests_path)] manifest_file = self.options.get('manifest_file') if manifest_file is not None: options += [ str(PurePosixPath(self._guest_manifests_path) / manifest_file) ] # Build up the custom facts if we have any facter = [] facter_dict = self.options.get('facter') if facter_dict is not None: for key, value in sorted(facter_dict.items()): facter.append("FACTER_{}={}".format(key, shlex.quote(value))) binary_path = self.options.get('binary_path') if binary_path is not None: puppet_bin = str(PurePosixPath(binary_path) / 'puppet') else: puppet_bin = 'puppet' # TODO: working_directory for hiera. Please contribute! env = [] env_variables = self.options.get('environment_variables') if env_variables is not None: for key, value in sorted(env_variables.items()): env.append("{}={}".format(key, shlex.quote(value))) command = env + facter + [puppet_bin, 'apply'] + options return command
'privileged': bool, 'profiles': [ str, ], 'protocol': In([ 'lxd', 'simplestreams', ]), 'provisioning': [], # will be set dynamically using provisioner classes... 'server': Url(), 'shares': [{ # The existence of the source directory will be checked! 'source': IsDir(), 'dest': str, 'set_host_acl': bool, }], 'shell': { 'user': str, 'home': str, }, 'users': [{ # Usernames max length is set 32 characters according to useradd's man page. Required('name'): All(str, Length(max=32)), 'home': str, 'password': str,
def test_IsDir(): schema = Schema(IsDir()) assert_raises(MultipleInvalid, schema, 3) schema(os.path.dirname(os.path.abspath(__file__)))
def get_schema(): _top_level_and_containers_common_options = { 'environment': { Extra: Coerce(str) }, 'hostnames': [ Hostname(), ], 'image': str, 'lxc_config': { Extra: str }, 'mode': In([ 'local', 'pull', ]), 'privileged': bool, 'profiles': [ str, ], 'protocol': In([ 'lxd', 'simplestreams', ]), 'provisioning': [], # will be set dynamically using provisioner classes... 'server': Url(), 'shares': [{ # The existence of the source directory will be checked! 'source': IsDir(), 'dest': str, 'set_host_acl': bool, }], 'shell': { 'user': str, 'home': str, }, 'users': [{ # Usernames max length is set 32 characters according to useradd's man page. Required('name'): All(str, Length(max=32)), 'home': str, 'password': str, }], } def _check_provisioner_config(config): provisioners = Provisioner.provisioners.values() # Check if 'type' is correctly defined Schema( { Required('type'): Any(*[provisioner.name for provisioner in provisioners]) }, extra=ALLOW_EXTRA)(config) # Check if the detected provisioner's schema is fully satisfied c = config.copy() name = c.pop('type') detected_provisioner = next(provisioner for provisioner in provisioners if provisioner.name == name) validated = Schema(detected_provisioner.schema)(c) validated['type'] = name return validated # Inserts provisioner specific schema rules in the global schema dict. _top_level_and_containers_common_options['provisioning'] = [ All(_check_provisioner_config) ] _container_options = { Required('name'): LXDIdentifier(), } _container_options.update(_top_level_and_containers_common_options) _lxdock_options = { Required('name'): LXDIdentifier(), 'containers': [ _container_options, ], } _lxdock_options.update(_top_level_and_containers_common_options) return Schema(_lxdock_options)
'author': str, 'locales': All(list, _locales_configuration), Required('base_url'): str, 'root_path': str, 'clean_urls': bool, 'content_negotiation': bool, 'mode': Any('development', 'production'), 'assets_directory_path': All(str, IsDir(), Path()), 'plugins': All( dict, lambda x: PluginsConfiguration({ Importable()(plugin_type_name): plugin_configuration for plugin_type_name, plugin_configuration in x.items() })), Required('theme', default=dict): All({ 'background_image_id': str, }, _theme_configuration), 'lifetime_threshold': All(int, Range(min=1)), }, _configuration))
class PyESGConfiguration(JSONSerialisableClass): """ Represents the full pyESG configuration. Attributes: number_of_simulations (int): The number of simulations to produce. number_of_projection_steps (int): The number of time steps to project in the simulations. projection_frequency (str): The frequency of projections. (e.g. 'annually', 'monthly', 'weekly') number_of_batches (int): The number of batches into which the simulations are split during generation. random_seed (int): The random seed to use when generating random samples. economies (list[Economy]): A list of the economies being modelled. correlations (Correlations): The correlations between the random drivers for the asset class models. """ _serialisable_lists = { 'economies': Economy, } _serialisable_attrs = { 'correlations': Correlations, } _validation_schema = Schema({ Required('number_of_simulations'): All(int, Range(min=1)), Required('number_of_projection_steps'): All(int, Range(min=1)), Required('output_file_directory'): IsDir(), Required('output_file_name'): str, Required('projection_frequency'): In(PROJECTION_FREQUENCIES), Required('number_of_batches'): All(int, Range(min=1)), Required('random_seed'): int, Required('start_date'): Date(), Required('economies'): [Economy._validation_schema], Required('correlations'): Correlations._validation_schema, }) def __init__(self, **kwargs): self.number_of_simulations = None # type: int self.number_of_projection_steps = None # type: int self.output_file_directory = None # type: str self.output_file_name = None # type: str self.projection_frequency = None # type: str self.number_of_batches = None # type: int self.random_seed = None # type: int self.start_date = None # type: str self.economies = [] # type: List[Economy] self.correlations = Correlations() # type: Correlations super().__init__(**kwargs) @classmethod def load_from_file(cls, file_path: str) -> 'PyESGConfiguration': """ Loads settings from a pyESG configuration file. Args: file_path: The file path of the pyESG configuration file. Returns: A PyESGConfiguration object containing the settings from the specified configuration file. """ with open(file_path) as config_file: config = json.load(config_file) pyesg_config = cls._decode_json(config) # type: 'PyESGConfiguration' return pyesg_config def save_to_file(self, file_path: str): """ Saves the configuration settings specified to a file. Args: file_path: The file path to which to save the configuration file. """ with open(file_path, 'w') as engine_settings_file: json.dump(self._encode_json(), engine_settings_file, indent=4) def validate(self): """ Validates the configuration. """ json_obj = self._encode_json() self._validation_schema(json_obj)
setattr(configuration, key, value) return configuration _ConfigurationSchema = Schema(All({ Required('output'): All(str, VoluptuousPath()), 'title': str, 'author': str, 'locales': All(list, _locales_configuration), Required('base_url'): str, 'root_path': str, 'clean_urls': bool, 'content_negotiation': bool, 'debug': bool, 'assets': All(str, IsDir(), VoluptuousPath()), 'extensions': All(dict, _extensions_configuration), Required('theme', default=dict): All({ 'background_image_id': str, }, _theme_configuration), 'lifetime_threshold': All(int, Range(min=0)), }, _configuration)) def _from_dict(configuration_dict: Dict) -> Configuration: try: return _ConfigurationSchema(configuration_dict) except Invalid as e: raise ConfigurationError(e)
ConfigurationSchema = Schema({ Required('output'): All(str), 'title': All(str), 'author': str, 'locales': All(list, [{ Required('locale'): validate_locale, Required('alias', default=None): Any(str, None), }]), Required('base_url'): All(str), 'root_path': All(str), 'clean_urls': All(bool), 'content_negotiation': bool, 'mode': Any('development', 'production'), 'resources': All(str, IsDir()), 'plugins': MapDict(str, dict), }) class ConfigurationError(ExternalContextError): pass def validate_configuration(schema: Schema, configuration: Any) -> Any: try: return schema(configuration) except Invalid as e: raise ConfigurationError(e)
ALLOW_EXTRA, All, Coerce, Invalid, IsDir, Required, Schema, ) class ConfigError(Exception): pass Dir = Coerce(Path, "expected a directory") ExistingDir = All(IsDir(None), Dir) Highlight = Coerce(get_style_by_name, "expected a pygments style") schema = Schema({ Required("build", default={}): { Required("build-dir", default="build"): Dir, Required("collection-dir", default="collections"): ExistingDir, Required("template-dir", default="templates"): ExistingDir, Required("highlight", default="default"): Highlight, Required("plugins", default=[]): [str], }, Required("collections", default=[]): [{ # yapf: ignore Required("name"): str, Required("template", default=""): str, Required("paths"): [str], Required("ignore-paths", default=[]): [str],
setattr(configuration, key, value) return configuration _ConfigurationSchema = Schema(All({ Required('output'): All(str, Path()), 'title': str, 'author': str, 'locales': All(list, _locales_configuration), Required('base_url'): str, 'root_path': str, 'clean_urls': bool, 'content_negotiation': bool, 'mode': Any('development', 'production'), 'assets_directory_path': All(str, IsDir(), Path()), 'plugins': All(dict, lambda x: PluginsConfiguration({Importable()(plugin_type_name): plugin_configuration for plugin_type_name, plugin_configuration in x.items()})), Required('theme', default=dict): All({ 'background_image_id': str, }, _theme_configuration), 'lifetime_threshold': All(int, Range(min=1)), }, _configuration)) class ConfigurationValueError(ContextError, UserFacingError, ValueError): pass # pragma: no cover def _from_voluptuous(config_builtin: Any) -> Configuration: try: return _ConfigurationSchema(config_builtin)
class AnsibleLocalProvisioner(Provisioner): """ Allows to perform provisioning operations using Ansible on the guest. """ PROVISIONING_DIR = "/provisioning" PLAYBOOOK_PATH = "/provisioning/playbook.yml" name = "ansible_local" schema = { Exclusive("playbook", "playbook"): IsFile(), Exclusive("dir", "playbook"): IsDir(), "ansible_version": str, } # guest_required_packages_alpine = ['python', ] # guest_required_packages_arch = ['python', ] # guest_required_packages_centos = ['python', ] # guest_required_packages_fedora = ['python2', ] # guest_required_packages_gentoo = ['dev-lang/python', ] # guest_required_packages_opensuse = ['python3-base', ] # guest_required_packages_ol = ['python', ] guest_required_packages_debian = [ 'apt-utils', 'aptitude', 'python', 'python3-pip', 'libssl-dev', ] guest_required_packages_ubuntu = [ 'apt-utils', 'aptitude', 'python', 'python3-pip', 'libssl-dev', ] def setup_single(self, guest): super().setup_single(guest) ansible_version = self.options.get("ansible_version") if ansible_version: ansible = "ansible=={}".format() else: ansible = "ansible" guest.run(["/usr/bin/pip3", "install", ansible]) def provision_single(self, guest): super().provision_single(guest) guest.run(["rm", "-rf", self.PROVISIONING_DIR]) guest.run(["mkdir", self.PROVISIONING_DIR]) if self.options.get("playbook"): with open(self.homedir_expanded_path( self.options["playbook"])) as fd: guest.lxd_container.files.put(self.PLAYBOOOK_PATH, fd.read()) if self.options.get("dir"): guest.lxd_container.files.recursive_put( self.homedir_expanded_path(self.options["dir"]), self.PROVISIONING_DIR, ) command = [ "ansible-playbook", "--connection=local", "--inventory=127.0.0.1,", self.PLAYBOOOK_PATH, ] guest.run(command, quiet=False)