class ValidationConfiguration(JSONSerialisableClass):
"""
Represents the validation configuration.
Attributes:
output_file_directory (str): The directory to which validation results will be saved.
output_file_name (str): The name of the file to save containing the validation results.
validation_analyses (List[AssetClass]): A list of asset class objects containing all validations to perform
for the asset class.
"""
_serialisable_lists = {
'asset_classes': AssetClass,
}
_validation_schema = Schema({
Required("output_file_directory"):
IsDir(),
Required("output_file_name"):
str,
Required("asset_classes"): [AssetClass._validation_schema],
})
def __init__(self, **kwargs):
self.output_file_directory = None # type: str
self.output_file_name = None # type: str
self.asset_classes = [] # type: List[AssetClass]
super().__init__(**kwargs)
@classmethod
def load_from_file(cls, file_path: str) -> 'ValidationConfiguration':
"""
Loads settings from a validation configuration file.
Args:
file_path: The file path of the validation configuration file.
Returns:
A ValidationConfiguration object containing the settings from the specified configuration file.
"""
with open(file_path) as config_file:
config = json.load(config_file)
config = cls._decode_json(config) # type: 'ValidationConfiguration'
return config
def save_to_file(self, file_path: str):
"""
Saves the configuration settings specified to a file.
Args:
file_path: The file path to which to save the configuration file.
"""
with open(file_path, 'w') as validation_configuration_file:
json.dump(self._encode_json(),
validation_configuration_file,
indent=4)
def validate(self):
"""
Validates the configuration.
"""
json_obj = self._encode_json()
self._validation_schema(json_obj)
def validate_paths(options):
# Refs Vagrant code:
# https://github.com/mitchellh/vagrant/blob/9c299a2a357fcf87f356bb9d56e18a037a53d138/
# plugins/provisioners/puppet/config/puppet.rb#L112
if options.get('manifests_path') is not None:
host_manifest_file = str(
Path(options['manifests_path']) / options['manifest_file'])
IsFile(msg="File {} does not exist".format(host_manifest_file))(host_manifest_file)
elif options.get('environment_path') is not None:
host_selected_environment_path = str(
Path(options['environment_path']) / options['environment'])
IsDir(msg="Directory {} does not exist".format(host_selected_environment_path))(
host_selected_environment_path)
return options
class PuppetProvisioner(Provisioner):
""" Allows to perform provisioning operations using Puppet. """
name = 'puppet'
guest_required_packages_arch = ['puppet']
guest_required_packages_debian = ['puppet']
guest_required_packages_fedora = ['puppet']
guest_required_packages_ubuntu = ['puppet']
# Refs Vagrant docs:
# https://www.vagrantup.com/docs/provisioning/puppet_apply.html#options
schema = All(
{
'binary_path': str,
'facter': dict,
'hiera_config_path': IsFile(),
'manifest_file': str,
'manifests_path': IsDir(),
'module_path': IsDir(),
'environment': str,
'environment_path': IsDir(),
'environment_variables': dict,
'options': str,
}, finalize_options, validate_paths)
_guest_manifests_path = '/.lxdock.d/puppet/manifests'
_guest_module_path = '/.lxdock.d/puppet/modules'
_guest_default_module_path = '/etc/puppet/modules'
_guest_environment_path = '/.lxdock.d/puppet/environments'
_guest_hiera_file = '/.lxdock.d/puppet/hiera.yaml'
def provision_single(self, guest):
""" Performs the provisioning operations using puppet. """
# Verify if `puppet` has been installed.
binary_path = self.options.get('binary_path')
if binary_path is not None:
puppet_bin = str(PurePosixPath(binary_path) / 'puppet')
retcode = guest.run(['test', '-x', puppet_bin])
fail_msg = (
"puppet executable is not found in the specified path {} in the "
"guest container. ".format(binary_path))
else:
retcode = guest.run(['which', 'puppet'])
fail_msg = (
"puppet was not automatically installed for this guest. "
"Please specify the command to install it in LXDock file using "
"a shell provisioner and try `lxdock provision` again. You may "
"also specify `binary_path` that contains the puppet executable "
"in LXDock file.")
if retcode != 0:
raise ProvisionFailed(fail_msg)
# Copy manifests dir
manifests_path = self.options.get('manifests_path')
if manifests_path is not None:
guest.copy_directory(Path(manifests_path),
PurePosixPath(self._guest_manifests_path))
# Copy module dir
module_path = self.options.get('module_path')
if module_path is not None:
guest.copy_directory(Path(module_path),
PurePosixPath(self._guest_module_path))
# Copy environment dir
environment_path = self.options.get('environment_path')
if environment_path is not None:
guest.copy_directory(Path(environment_path),
PurePosixPath(self._guest_environment_path))
# Copy hiera file
hiera_file = self.options.get('hiera_config_path')
if hiera_file is not None:
guest.copy_file(Path(hiera_file),
PurePosixPath(self._guest_hiera_file))
# Run puppet.
command = self._build_puppet_command()
if environment_path:
logger.info("Running Puppet with environment {}...".format(
self.options['environment']))
else:
logger.info("Running Puppet with {}...".format(
self.options['manifest_file']))
guest.run(['sh', '-c', ' '.join(command)])
##################################
# PRIVATE METHODS AND PROPERTIES #
##################################
def _build_puppet_command(self):
"""
Refs:
https://github.com/mitchellh/vagrant/blob/9c299a2a357fcf87f356bb9d56e18a037a53d138/
plugins/provisioners/puppet/provisioner/puppet.rb#L173
"""
options = self.options.get('options', '')
options = list(map(shlex.quote, shlex.split(options)))
module_path = self.options.get('module_path')
if module_path is not None:
options += [
'--modulepath', '{}:{}'.format(self._guest_module_path,
self._guest_default_module_path)
]
hiera_path = self.options.get('hiera_config_path')
if hiera_path is not None:
options += ['--hiera_config={}'.format(self._guest_hiera_file)]
# TODO: we are not detecting console color support now, but please contribute if you need!
options += ['--detailed-exitcodes']
environment_path = self.options.get('environment_path')
if environment_path is not None:
options += [
'--environmentpath',
str(self._guest_environment_path), '--environment',
self.options['environment']
]
else:
options += ['--manifestdir', str(self._guest_manifests_path)]
manifest_file = self.options.get('manifest_file')
if manifest_file is not None:
options += [
str(PurePosixPath(self._guest_manifests_path) / manifest_file)
]
# Build up the custom facts if we have any
facter = []
facter_dict = self.options.get('facter')
if facter_dict is not None:
for key, value in sorted(facter_dict.items()):
facter.append("FACTER_{}={}".format(key, shlex.quote(value)))
binary_path = self.options.get('binary_path')
if binary_path is not None:
puppet_bin = str(PurePosixPath(binary_path) / 'puppet')
else:
puppet_bin = 'puppet'
# TODO: working_directory for hiera. Please contribute!
env = []
env_variables = self.options.get('environment_variables')
if env_variables is not None:
for key, value in sorted(env_variables.items()):
env.append("{}={}".format(key, shlex.quote(value)))
command = env + facter + [puppet_bin, 'apply'] + options
return command
'privileged':
bool,
'profiles': [
str,
],
'protocol':
In([
'lxd',
'simplestreams',
]),
'provisioning': [], # will be set dynamically using provisioner classes...
'server':
Url(),
'shares': [{
# The existence of the source directory will be checked!
'source': IsDir(),
'dest': str,
'set_host_acl': bool,
}],
'shell': {
'user': str,
'home': str,
},
'users': [{
# Usernames max length is set 32 characters according to useradd's man page.
Required('name'):
All(str, Length(max=32)),
'home':
str,
'password':
str,
def test_IsDir():
schema = Schema(IsDir())
assert_raises(MultipleInvalid, schema, 3)
schema(os.path.dirname(os.path.abspath(__file__)))
def get_schema():
_top_level_and_containers_common_options = {
'environment': {
Extra: Coerce(str)
},
'hostnames': [
Hostname(),
],
'image':
str,
'lxc_config': {
Extra: str
},
'mode':
In([
'local',
'pull',
]),
'privileged':
bool,
'profiles': [
str,
],
'protocol':
In([
'lxd',
'simplestreams',
]),
'provisioning':
[], # will be set dynamically using provisioner classes...
'server':
Url(),
'shares': [{
# The existence of the source directory will be checked!
'source': IsDir(),
'dest': str,
'set_host_acl': bool,
}],
'shell': {
'user': str,
'home': str,
},
'users': [{
# Usernames max length is set 32 characters according to useradd's man page.
Required('name'):
All(str, Length(max=32)),
'home':
str,
'password':
str,
}],
}
def _check_provisioner_config(config):
provisioners = Provisioner.provisioners.values()
# Check if 'type' is correctly defined
Schema(
{
Required('type'):
Any(*[provisioner.name for provisioner in provisioners])
},
extra=ALLOW_EXTRA)(config)
# Check if the detected provisioner's schema is fully satisfied
c = config.copy()
name = c.pop('type')
detected_provisioner = next(provisioner for provisioner in provisioners
if provisioner.name == name)
validated = Schema(detected_provisioner.schema)(c)
validated['type'] = name
return validated
# Inserts provisioner specific schema rules in the global schema dict.
_top_level_and_containers_common_options['provisioning'] = [
All(_check_provisioner_config)
]
_container_options = {
Required('name'): LXDIdentifier(),
}
_container_options.update(_top_level_and_containers_common_options)
_lxdock_options = {
Required('name'): LXDIdentifier(),
'containers': [
_container_options,
],
}
_lxdock_options.update(_top_level_and_containers_common_options)
return Schema(_lxdock_options)
'author':
str,
'locales':
All(list, _locales_configuration),
Required('base_url'):
str,
'root_path':
str,
'clean_urls':
bool,
'content_negotiation':
bool,
'mode':
Any('development', 'production'),
'assets_directory_path':
All(str, IsDir(), Path()),
'plugins':
All(
dict, lambda x: PluginsConfiguration({
Importable()(plugin_type_name): plugin_configuration
for plugin_type_name, plugin_configuration in x.items()
})),
Required('theme', default=dict):
All({
'background_image_id': str,
}, _theme_configuration),
'lifetime_threshold':
All(int, Range(min=1)),
}, _configuration))
class PyESGConfiguration(JSONSerialisableClass):
"""
Represents the full pyESG configuration.
Attributes:
number_of_simulations (int): The number of simulations to produce.
number_of_projection_steps (int): The number of time steps to project in the simulations.
projection_frequency (str): The frequency of projections. (e.g. 'annually', 'monthly', 'weekly')
number_of_batches (int): The number of batches into which the simulations are split during generation.
random_seed (int): The random seed to use when generating random samples.
economies (list[Economy]): A list of the economies being modelled.
correlations (Correlations): The correlations between the random drivers for the asset class models.
"""
_serialisable_lists = {
'economies': Economy,
}
_serialisable_attrs = {
'correlations': Correlations,
}
_validation_schema = Schema({
Required('number_of_simulations'):
All(int, Range(min=1)),
Required('number_of_projection_steps'):
All(int, Range(min=1)),
Required('output_file_directory'):
IsDir(),
Required('output_file_name'):
str,
Required('projection_frequency'):
In(PROJECTION_FREQUENCIES),
Required('number_of_batches'):
All(int, Range(min=1)),
Required('random_seed'):
int,
Required('start_date'):
Date(),
Required('economies'): [Economy._validation_schema],
Required('correlations'):
Correlations._validation_schema,
})
def __init__(self, **kwargs):
self.number_of_simulations = None # type: int
self.number_of_projection_steps = None # type: int
self.output_file_directory = None # type: str
self.output_file_name = None # type: str
self.projection_frequency = None # type: str
self.number_of_batches = None # type: int
self.random_seed = None # type: int
self.start_date = None # type: str
self.economies = [] # type: List[Economy]
self.correlations = Correlations() # type: Correlations
super().__init__(**kwargs)
@classmethod
def load_from_file(cls, file_path: str) -> 'PyESGConfiguration':
"""
Loads settings from a pyESG configuration file.
Args:
file_path: The file path of the pyESG configuration file.
Returns:
A PyESGConfiguration object containing the settings from the specified configuration file.
"""
with open(file_path) as config_file:
config = json.load(config_file)
pyesg_config = cls._decode_json(config) # type: 'PyESGConfiguration'
return pyesg_config
def save_to_file(self, file_path: str):
"""
Saves the configuration settings specified to a file.
Args:
file_path: The file path to which to save the configuration file.
"""
with open(file_path, 'w') as engine_settings_file:
json.dump(self._encode_json(), engine_settings_file, indent=4)
def validate(self):
"""
Validates the configuration.
"""
json_obj = self._encode_json()
self._validation_schema(json_obj)
setattr(configuration, key, value)
return configuration
_ConfigurationSchema = Schema(All({
Required('output'): All(str, VoluptuousPath()),
'title': str,
'author': str,
'locales': All(list, _locales_configuration),
Required('base_url'): str,
'root_path': str,
'clean_urls': bool,
'content_negotiation': bool,
'debug': bool,
'assets': All(str, IsDir(), VoluptuousPath()),
'extensions': All(dict, _extensions_configuration),
Required('theme', default=dict): All({
'background_image_id': str,
}, _theme_configuration),
'lifetime_threshold': All(int, Range(min=0)),
}, _configuration))
def _from_dict(configuration_dict: Dict) -> Configuration:
try:
return _ConfigurationSchema(configuration_dict)
except Invalid as e:
raise ConfigurationError(e)
ConfigurationSchema = Schema({
Required('output'): All(str),
'title': All(str),
'author': str,
'locales': All(list, [{
Required('locale'): validate_locale,
Required('alias', default=None): Any(str, None),
}]),
Required('base_url'): All(str),
'root_path': All(str),
'clean_urls': All(bool),
'content_negotiation': bool,
'mode': Any('development', 'production'),
'resources': All(str, IsDir()),
'plugins': MapDict(str, dict),
})
class ConfigurationError(ExternalContextError):
pass
def validate_configuration(schema: Schema, configuration: Any) -> Any:
try:
return schema(configuration)
except Invalid as e:
raise ConfigurationError(e)
ALLOW_EXTRA,
All,
Coerce,
Invalid,
IsDir,
Required,
Schema,
)
class ConfigError(Exception):
pass
Dir = Coerce(Path, "expected a directory")
ExistingDir = All(IsDir(None), Dir)
Highlight = Coerce(get_style_by_name, "expected a pygments style")
schema = Schema({
Required("build", default={}): {
Required("build-dir", default="build"): Dir,
Required("collection-dir", default="collections"): ExistingDir,
Required("template-dir", default="templates"): ExistingDir,
Required("highlight", default="default"): Highlight,
Required("plugins", default=[]): [str],
},
Required("collections", default=[]): [{ # yapf: ignore
Required("name"): str,
Required("template", default=""): str,
Required("paths"): [str],
Required("ignore-paths", default=[]): [str],
setattr(configuration, key, value)
return configuration
_ConfigurationSchema = Schema(All({
Required('output'): All(str, Path()),
'title': str,
'author': str,
'locales': All(list, _locales_configuration),
Required('base_url'): str,
'root_path': str,
'clean_urls': bool,
'content_negotiation': bool,
'mode': Any('development', 'production'),
'assets_directory_path': All(str, IsDir(), Path()),
'plugins': All(dict, lambda x: PluginsConfiguration({Importable()(plugin_type_name): plugin_configuration for plugin_type_name, plugin_configuration in x.items()})),
Required('theme', default=dict): All({
'background_image_id': str,
}, _theme_configuration),
'lifetime_threshold': All(int, Range(min=1)),
}, _configuration))
class ConfigurationValueError(ContextError, UserFacingError, ValueError):
pass # pragma: no cover
def _from_voluptuous(config_builtin: Any) -> Configuration:
try:
return _ConfigurationSchema(config_builtin)
class AnsibleLocalProvisioner(Provisioner):
""" Allows to perform provisioning operations using Ansible on the guest. """
PROVISIONING_DIR = "/provisioning"
PLAYBOOOK_PATH = "/provisioning/playbook.yml"
name = "ansible_local"
schema = {
Exclusive("playbook", "playbook"): IsFile(),
Exclusive("dir", "playbook"): IsDir(),
"ansible_version": str,
}
# guest_required_packages_alpine = ['python', ]
# guest_required_packages_arch = ['python', ]
# guest_required_packages_centos = ['python', ]
# guest_required_packages_fedora = ['python2', ]
# guest_required_packages_gentoo = ['dev-lang/python', ]
# guest_required_packages_opensuse = ['python3-base', ]
# guest_required_packages_ol = ['python', ]
guest_required_packages_debian = [
'apt-utils',
'aptitude',
'python',
'python3-pip',
'libssl-dev',
]
guest_required_packages_ubuntu = [
'apt-utils',
'aptitude',
'python',
'python3-pip',
'libssl-dev',
]
def setup_single(self, guest):
super().setup_single(guest)
ansible_version = self.options.get("ansible_version")
if ansible_version:
ansible = "ansible=={}".format()
else:
ansible = "ansible"
guest.run(["/usr/bin/pip3", "install", ansible])
def provision_single(self, guest):
super().provision_single(guest)
guest.run(["rm", "-rf", self.PROVISIONING_DIR])
guest.run(["mkdir", self.PROVISIONING_DIR])
if self.options.get("playbook"):
with open(self.homedir_expanded_path(
self.options["playbook"])) as fd:
guest.lxd_container.files.put(self.PLAYBOOOK_PATH, fd.read())
if self.options.get("dir"):
guest.lxd_container.files.recursive_put(
self.homedir_expanded_path(self.options["dir"]),
self.PROVISIONING_DIR,
)
command = [
"ansible-playbook",
"--connection=local",
"--inventory=127.0.0.1,",
self.PLAYBOOOK_PATH,
]
guest.run(command, quiet=False)
