def set_data(self, data: Dict, validator: str = ""):
"""Re-set the data for the output."""
assert isinstance(data, dict), f"Expected Dict of data, got {data}"
if validator:
self._environment.config().validate(data, validator)
mock_instance_id = f"dict-output-{self._instance_id}"
self.loaded = Loaded(data=data,
parent=self._environment.config(),
instance_id=mock_instance_id)
def set_data(self, data: Dict = {}, validator: str = ''):
""" Re-set the data for the output """
assert isinstance(
data, dict), "Expected Dict of data, got {}".format(data)
if validator:
self.environment.config.validate(data, validator)
mock_instance_id = 'dict-output-{}'.format(self.instance_id)
self.loaded = Loaded(
data=data,
parent=self.environment.config,
instance_id=mock_instance_id)
def add_fixture_from_dict(self,
plugin_dict: Dict[str, Any],
type: Type = None,
instance_id: str = '',
validator: str = '',
arguments: Dict[str, Any] = {}) -> Fixture:
""" Create a single plugin from a Dict of information for it
Create a new plugin from a map/dict of settings for the needed parameters.
@see add_fixture_from_loadedconfig
Parameters:
-----------
config (Config) : configerus.Config object passed to each generated plugins.
type (.plugin.Type) : plugin type to create, pulled from the config/dict if
omitted
client_dict (Dict[str,Any]) : Dict from which all needed information will
be pulled. Optionally additional config sources can be included as well
as arguments which could be passed to the plugin.
@see add_fixture_from_dict for more details.
instance_id (str) : optionally pass an instance_id for the item.
validator (str) : optionally use a configerus validator on the entire .get()
for the instance config.
Return:
-------
A Fixture object with the new plugin added
The Fixtures has already been added to the environment, but is returned
so that the consumer can act on it separately without haveing to
search for it.
"""
# Create a mock configerus.loaded.Loaded object, not attached to anything
# and use it for config retrieval. This gives us formatting, validation
# etc.
mock_config_loaded = Loaded(data=plugin_dict,
parent=self.config,
instance_id='mock-plugin-construct')
""" Mock configerus loaded object for config retrieval """
base = LOADED_KEY_ROOT
""" to keep this function similar to add_fixture_from_config we use an empty .get() base """
return self.add_fixture_from_loadedconfig(loaded=mock_config_loaded,
base=base,
type=type,
instance_id=instance_id,
validator=validator,
arguments=arguments)
def make_fixtures(self) -> Fixtures:
"""Build fixtures for all of the clients.
Returns:
--------
Fixtures collection of fixtures that have been created
"""
# get fresh values for the launchpad config (in case it has changed) and
# treat this as a configerus Loaded object which allows us to use the
# searching and validation syntax.
client_config = self.describe_config()
launchpad_config = Loaded(data=client_config,
parent=None,
instance_id="launchpad_client")
# Retrieve a list of hosts, and use that to decide what clients to
# make. If we find a host for a client, then we retrieve needed
# config and use it to generate the related client.
hosts = launchpad_config.get(METTA_LAUNCHPAD_CONFIG_HOSTS_KEY,
default=[],
format=False)
# MKE Client
#
if METTA_MIRANTIS_CLIENT_MKE_PLUGIN_ID in self.systems:
mke_hosts: List[Any] = list(
host for host in hosts
if host[METTA_LAUNCHPAD_CONFIG_HOST_ROLE_KEY] in ["manager"])
if len(mke_hosts) > 0:
instance_id = f"{self._instance_id}-{METTA_MIRANTIS_CLIENT_MKE_PLUGIN_ID}"
mke_arguments: Dict[
str,
Any] = self.systems[METTA_MIRANTIS_CLIENT_MKE_PLUGIN_ID]
mke_arguments["hosts"] = mke_hosts
if "accesspoint" in mke_arguments and mke_arguments[
"accesspoint"]:
mke_arguments["accesspoint"] = clean_accesspoint(
mke_arguments["accesspoint"])
logger.debug(
"Launchpad client is creating an MKE client plugin: %s",
instance_id)
fixture = self._environment.new_fixture(
plugin_id=METTA_MIRANTIS_CLIENT_MKE_PLUGIN_ID,
instance_id=instance_id,
priority=70,
arguments=mke_arguments,
labels={
"parent_plugin_id": METTA_LAUNCHPAD_CLIENT_PLUGIN_ID,
"parent_instance_id": self._instance_id,
},
replace_existing=True,
)
self._fixtures.add(fixture, replace_existing=True)
# MSR Client
#
if METTA_MIRANTIS_CLIENT_MSR_PLUGIN_ID in self.systems:
msr_hosts: List[Any] = list(
host for host in hosts
if host[METTA_LAUNCHPAD_CONFIG_HOST_ROLE_KEY] in ["msr"])
if len(msr_hosts) > 0:
instance_id = f"{self._instance_id}-{METTA_MIRANTIS_CLIENT_MSR_PLUGIN_ID}"
msr_arguments: Dict[
str,
Any] = self.systems[METTA_MIRANTIS_CLIENT_MSR_PLUGIN_ID]
msr_arguments["hosts"] = msr_hosts
if "accesspoint" in msr_arguments and msr_arguments[
"accesspoint"]:
msr_arguments["accesspoint"] = clean_accesspoint(
msr_arguments["accesspoint"])
logger.debug(
"Launchpad client is creating an MSR client plugin: %s",
instance_id)
fixture = self._environment.new_fixture(
plugin_id=METTA_MIRANTIS_CLIENT_MSR_PLUGIN_ID,
instance_id=instance_id,
priority=70,
arguments=msr_arguments,
labels={
"parent_plugin_id": METTA_LAUNCHPAD_CLIENT_PLUGIN_ID,
"parent_instance_id": self._instance_id,
},
replace_existing=True,
)
self._fixtures.add(fixture, replace_existing=True)
class DictOutputPlugin(OutputBase):
""" MTT Output plugin a Dict output type
This output plugin leverages configurators features, treating the dict as
a config source, in order to get validation and navigations.
"""
def __init__(self, environment, instance_id,
data: Dict = {}, validator: str = ''):
""" Run the super constructor but also set class properties
Here we treat the data dict as a configerus.loaded.Loaded instance,
with out config as a parent, so that we can leverage the configerus
tools for searching, formating and validation.
a configerus.loaded.Loaded object is kept for the config.
Parameters:
-----------
data (Dict) : any dict data to be stored
validator (str) : a configerus validator target if you want valdiation
applied to the data before it is added.
Raises:
-------
A configerus.validate.ValidationError is raised if you passed in a
validator target and validation failed.
An AssertionError is raised if you didn't pass in a Dict.
"""
super(OutputBase, self).__init__(environment, instance_id)
self.set_data(data, validator)
def set_data(self, data: Dict = {}, validator: str = ''):
""" Re-set the data for the output """
assert isinstance(
data, dict), "Expected Dict of data, got {}".format(data)
if validator:
self.environment.config.validate(data, validator)
mock_instance_id = 'dict-output-{}'.format(self.instance_id)
self.loaded = Loaded(
data=data,
parent=self.environment.config,
instance_id=mock_instance_id)
def get_output(self, key: str = LOADED_KEY_ROOT, validator: str = ''):
""" retrieve an output
Because we treated that data as a high-priority configerus source with
a custom label, we can retrieve data from that source easily and also
leverage other configerus options such as templating and validation
If you don't pass a key, you will get the entire data value
Parameters:
-----------
key (str) : you can optionally pass a key to retrieve only a part of the
data structure. This uses the configerus .get() command which uses
dot "." notation to descend a tree.
validator (str) : you can tell configerus to apply a validator to the
return value
Returns:
--------
Any of retreived data in the assigned data, as though you were making
a configerus.loaded.Loaded.get()
Raises:
-------
AttributeError if you are trying to get output before you have
assigned data using .arguments()
configerus.validate.ValidationError if you passed in a validator and
validation failed.
"""
return self.loaded.get(key, validator=validator)
def info(self):
""" Return dict data about this plugin for introspection """
return {
'output': {
'data': self.loaded.data
}
}
class DictOutputPlugin:
"""Metta Output plugin for a Dict output type.
This output plugin leverages configurators features, treating the dict as
a config source, in order to get validation and navigations.
"""
def __init__(self,
environment: Environment,
instance_id: str,
data: Dict = None,
validator: str = ""):
"""Run the super constructor but also set class properties.
Here we treat the data dict as a configerus.loaded.Loaded instance,
with out config as a parent, so that we can leverage the configerus
tools for searching, formating and validation.
a configerus.loaded.Loaded object is kept for the config.
Parameters:
-----------
data (Dict) : any dict data to be stored
validator (str) : a configerus validator target if you want valdiation
applied to the data before it is added.
Raises:
-------
A configerus.validate.ValidationError is raised if you passed in a
validator target and validation failed.
An AssertionError is raised if you didn't pass in a Dict.
"""
self._environment: Environment = environment
""" Environemnt in which this plugin exists """
self._instance_id: str = instance_id
""" Unique id for this plugin instance """
if data is None:
data = {}
self.set_data(data, validator)
def set_data(self, data: Dict, validator: str = ""):
"""Re-set the data for the output."""
assert isinstance(data, dict), f"Expected Dict of data, got {data}"
if validator:
self._environment.config().validate(data, validator)
mock_instance_id = f"dict-output-{self._instance_id}"
self.loaded = Loaded(data=data,
parent=self._environment.config(),
instance_id=mock_instance_id)
def get_output(self, key: str = LOADED_KEY_ROOT, validator: str = ""):
"""Retrieve an output.
Because we treated that data as a high-priority configerus source with
a custom label, we can retrieve data from that source easily and also
leverage other configerus options such as templating and validation
If you don't pass a key, you will get the entire data value
Parameters:
-----------
key (str) : you can optionally pass a key to retrieve only a part of the
data structure. This uses the configerus .get() command which uses
dot "." notation to descend a tree.
validator (str) : you can tell configerus to apply a validator to the
return value
Returns:
--------
Any of retreived data in the assigned data, as though you were making
a configerus.loaded.Loaded.get()
Raises:
-------
AttributeError if you are trying to get output before you have
assigned data using .arguments()
configerus.validate.ValidationError if you passed in a validator and
validation failed.
"""
return self.loaded.get(key, validator=validator)
# deep argument is an info() standard across plugins
# pylint: disable=unused-argument
def info(self, deep: bool = False):
"""Return dict data about this plugin for introspection."""
return {"output": {"data": self.loaded.data}}
def add_fixture_from_loadedconfig(
self,
loaded: Loaded,
base: Any = LOADED_KEY_ROOT,
type: Type = None,
instance_id: str = '',
priority: int = -1,
validator: str = '',
arguments: Dict[str, Any] = {}) -> Fixture:
""" Create a plugin from loaded config
This method will interpret some config values as being usable to build plugin.
This function starts with a loaded config object because we can leverage
that from more starting points.
Using a configerus config object allows us to leverage advanced configerus
features such as tree searching, formatting and validation.
What is looked for:
1. valdiators if we need to validate the entire label/key before using it
2. type if we did not receive a type
3. plugin_id : which will tell us what plugin to load
4. optional instance_id if none was passed
5. config if you want config added - ONLY if fixtures is None
(plugins in Fixtures cannot override config objects)
6. arguments that will be executed on an argument() method if the
plugin has it.
Parameters:
-----------
config (Config) : Used to load and get the plugin configuration
type (.plugin.Type) : plugin type to create, pulled from the config/dict if
omitted. An exception will be thrown if Type is found from neither
this argument nor the passed config.
label (str) : config label to load to pull plugin configuration. That
label is loaded and config is pulled to produce a list of plugins.
base (str|List) : config key used as a .get() base for all gets. With this
you can instruct to pull config from a section of loaded config.
A list of strings is valid because configerus.loaded.get() can take that
as an argument. We will be using the list syntax anyway.
We call this base instead of key as we will be searching for sub-paths
to pull individual elements.
instance_id (str) : optionally pass an instance_id for the item.
validator (str) : optionally use a configerus validator on the entire .get()
for the instance config.
Returns:
--------
A Fixture object with the new plugin added
The Fixtures has already been added to the environment, but is returned
so that the consumer can act on it separately without haveing to
search for it.
Raises:
-------
If you ask for a plugin which has not been registered, then you're going
to get a NotImplementedError exception. To make sure that your desired
plugin is registered, make sure to import the module that contains the
factory method with a decorator.
A ValueError is thrown if the plugin cannot be created due to missing
plugin configuration/argument values. This means that we could not
determine how to create the plugin.
A configerus.validate.ValidationError will be raised if a validation
target was passed and validation failed.
"""
logger.debug('Construct config plugin [{}][{}]'.format(
type, instance_id))
# it might be expensive to retrieve all this but we do it to catch early
# faults in config.
plugin_base = loaded.get(base)
if plugin_base is None:
raise ValueError(
"Cannot build plugin as provided config was empty.")
validators = [
FIXTURE_VALIDATION_TARGET_FORMAT_STRING.format(
key=UCTT_FIXTURES_CONFIG_FIXTURE_KEY)
]
config_validators = loaded.get(
[base, UCTT_PLUGIN_CONFIG_KEY_VALIDATORS])
if config_validators:
validators = config_validators
if validator:
validators.append(validator)
if len(validators):
# Run configerus validation on the config base once per validator
try:
for validator in validators:
loaded.validate(plugin_base, validate_target=validator)
except ValidationError as e:
raise e
if type is None:
type = loaded.get([base, UCTT_PLUGIN_CONFIG_KEY_TYPE])
if isinstance(type, str):
# If a string type was passed in, ask the Type enum to convert it
type = Type.from_string(type)
if not type:
raise ValueError(
"Could not find a plugin type when trying to create a plugin : {}"
.format(loaded.get(base)))
plugin_id = loaded.get([base, UCTT_PLUGIN_CONFIG_KEY_PLUGINID])
if not plugin_id:
raise ValueError(
"Could not find a plugin_id when trying to create a '{}' plugin from config: {}"
.format(type, loaded.get(base)))
# if no instance_id was passed, try to load one or just make one up
if not instance_id:
instance_id = loaded.get([base, UCTT_PLUGIN_CONFIG_KEY_INSTANCEID])
if not instance_id:
instance_id = '{}-{}-{}'.format(
type.value, plugin_id, ''.join(
random.choice(string.ascii_lowercase)
for i in range(10)))
if priority < 0:
priority = loaded.get([base, UCTT_PLUGIN_CONFIG_KEY_PRIORITY])
if not priority:
priority = self.plugin_priority()
""" instance priority - this is actually a stupid way to get it """
# If arguments were given then pass them on
config_arguments = loaded.get([base, UCTT_PLUGIN_CONFIG_KEY_ARGUMENTS])
if config_arguments is not None:
arguments = arguments.copy()
arguments.update(config_arguments)
# Use the factory to make the .fixtures.Fixture
fixture = self.add_fixture(type=type,
plugin_id=plugin_id,
instance_id=instance_id,
priority=priority,
arguments=arguments)
plugin = fixture.plugin
return fixture
def add_fixture_from_loadedconfig(
self,
loaded: Loaded,
base: Union[str, List[Any]] = LOADED_KEY_ROOT,
instance_id: str = "",
priority: int = -1,
labels: Dict[str, str] = None,
validator: str = "",
arguments: Dict[str, Any] = None,
) -> Fixture:
"""Create a plugin from a Configerus loaded config object.
This method will interpret some config values as being usable to build
plugin. This function starts with a loaded config object because we can
leverage that from more starting points.
Using a configerus config object allows us to leverage advanced
configerus features such as tree searching, formatting and validation.
What is looked for:
1. valdiators if we need to validate the entire label/key before using
2. plugin_id : which will tell us what plugin to load
3. optional instance_id if none was passed
4. config if you want config added - ONLY if fixtures is None
(plugins in Fixtures cannot override config objects)
5. arguments that will be executed on an argument() method if the
plugin has it.
@TODO we should probably allow a setting to allow replacing existing
fixtures in the environment.
Parameters:
-----------
config (Config) : Used to load and get the plugin configuration
label (str) : config label to load to pull plugin configuration. That
label is loaded and config is pulled to produce a list of plugins.
base (str|List) : config key used as a .get() base for all gets. With
this you can instruct to pull config from a section of loaded
config.
A list of strings is valid because configerus.loaded.get() can take
that as an argument. We will be using the list syntax anyway.
We call this base instead of key as we will be searching for
sub-paths to pull individual elements.
instance_id (str) : optionally pass an instance_id for the item.
labels (Dict[str, str]) : Dictionary of labels which should be added to
the created fixture.
arguments (Dict[str, str]) : Dictionary of args which should be passed
to the plugin constructor.
validator (str) : optionally use a configerus validator on the entire
.get() for the instance config.
Returns:
--------
A Fixture object with the new plugin added
The Fixtures has already been added to the environment, but is returned
so that the consumer can act on it separately without haveing to
search for it.
Raises:
-------
If you ask for a plugin which has not been registered, then you're going
to get a NotImplementedError exception. To make sure that your desired
plugin is registered, make sure to import the module that contains the
factory method with a decorator.
A ValueError is thrown if the plugin cannot be created due to missing
plugin configuration/argument values. This means that we could not
determine how to create the plugin.
A configerus.validate.ValidationError will be raised if a validation
target was passed and validation failed.
"""
# Retrieve all of the plugin config, to test that it exists
# it might be expensive to retrieve all this but we do it to catch early
# faults in config.
if not loaded.has(base):
raise ValueError(f"Cannot build plugin as provided config was empty: {base}")
# get any validators from config, defaulting to just the jsonschema
# validator for a fixture
validators = loaded.get(
[base, METTA_FIXTURE_VALIDATION_JSONSCHEMA],
default=[{PLUGIN_ID_VALIDATE_JSONSCHEMA: METTA_FIXTURE_VALIDATION_JSONSCHEMA}],
)
if validator:
# if a validator arg was passed in, then add it
validators.append(validator)
if len(validators):
# Run configerus validation on the config base once per validator
try:
for val in validators:
loaded.get(base, validator=val)
except ValidationError as err:
raise err
try:
plugin_id = str(loaded.get([base, METTA_PLUGIN_CONFIG_KEY_PLUGINID]))
except KeyError as err:
full_config = loaded.get(base)
raise ValueError(
"Could not find a plugin_id when trying to create a "
f"plugin from config: {full_config}"
) from err
# if no instance_id was passed, try to load one or just make one up
if not instance_id:
instance_id = str(loaded.get([base, METTA_PLUGIN_CONFIG_KEY_INSTANCEID], default=""))
if not instance_id:
instance_rand = "".join(random.choice(string.ascii_lowercase) for i in range(10))
instance_id = f"{plugin_id}-{instance_rand}"
if priority < 0:
# instance priority - this is actually a stupid way to get it
priority = int(
loaded.get(
[base, METTA_FIXTURE_CONFIG_KEY_PRIORITY],
default=self.plugin_priority(),
)
)
config_arguments = loaded.get([base, METTA_PLUGIN_CONFIG_KEY_ARGUMENTS], default={})
if len(config_arguments) > 0:
if arguments is None:
arguments = {}
else:
# if we have config arguments from two different sources, then
# add the config args to a copy of the function parameter args.
# We use a copy so that we make no context mistakes by altering
# a passed Dict that may get used for more than one plugin.
arguments = arguments.copy()
arguments.update(config_arguments)
config_labels = loaded.get([base, METTA_PLUGIN_CONFIG_KEY_PLUGINLABELS], default={})
if len(config_labels) > 0:
if labels is None:
labels = {}
else:
labels = labels.copy()
labels.update(config_labels)
# Use the factory to make the .fixture.Fixture
return self.builder_callback(
plugin_id=plugin_id,
instance_id=instance_id,
priority=priority,
arguments=arguments,
labels=labels,
)