def attach_to_environment(cls):
"""
This API is called on the HardwarePlayerPoolMixin so it can process configuration information. The
:class:`HardwarePlayerPoolMixin` can then verify that it has a valid environment and configuration
to run in.
:raises :class:`akit.exceptions.AKitMissingConfigError`, :class:`akit.exceptions.AKitInvalidConfigError`:
"""
super(HardwarePlayerPoolMixin, cls).attach_to_environment()
# IMPORTANT: This check is important to prevent noisy test runs and failures due to configuration errors.
if "pod" not in cls.landscape._landscape_info:
errmsg = "ERROR: In order to attach to an environment when a Hardware Player Pool" \
" is involved, there must be an 'pod' declaration in the landscape.json config file."
raise AKitConfigurationError(errmsg)
pod_info = cls.landscape._landscape_info["pod"]
# IMPORTANT: This check is important to prevent noisy test runs and failures due to configuration errors.
if "reference" not in pod_info or 'ip' not in pod_info[
"reference"] or 'port' not in pod_info["reference"]:
errmsg = "ERROR: In order to ensure we can communicate on the correct network with the 'pod' devices, " \
"the 'pod' must contain 'reference' item that contains both an 'ip' and 'port'."
raise AKitConfigurationError(errmsg)
ref_info = pod_info["reference"]
cls.reference_ip = ref_info["ip"]
cls.reference_port = int(ref_info["port"])
cls.correspondance_interface, _ = get_correspondance_interface(
cls.reference_ip, cls.reference_port)
return
def lookup_agent(self, apname: str) -> WirelessApAgent:
"""
Looks up a serial agent by serial mapping.
"""
wireless_agent = None
lscape = self.landscape
if apname in self.self._wireless_ap_config:
wireless_mapping = self._wireless_ap_config[apname]
if apname not in self._wireless_agents:
host = wireless_mapping["host"]
credential_name = wireless_mapping["credential"]
credential = lscape.lookup_credential(credential_name)
if credential is not None:
wireless_agent = WirelessApAgent(host, credential)
self._wireless_agents[apname] = wireless_agent
else:
errmsg = "Failure to find credential '{}' specified for wireless apname={}".format(
credential_name, apname
)
raise AKitConfigurationError(errmsg)
else:
wireless_agent = self._wireless_agents[apname]
else:
errmsg = "Failure to lookup wireless configuration for apname={}.".format(apname)
raise AKitConfigurationError(errmsg) from None
return wireless_agent
def _validate_landscape(self):
"""
This method is overriden in order to validate the info found
in the landscape file.
:param linfo: A python dictionary with the information contained in
the landscape.json file.
:type linfo: dict
"""
if "pod" not in self.landscape_info:
raise AKitConfigurationError(
"The 'testlandscape.py' file requires an 'pod' member.")
pod_info = self.landscape_info["pod"]
if "reference" not in pod_info:
raise AKitConfigurationError(
"The 'testlandscape.py' file 'pod' data requires an 'reference' member."
)
ref_info = pod_info["reference"]
if "ip" not in ref_info:
raise AKitConfigurationError(
"The 'testlandscape.py' file 'pod->reference' data requires an 'ip' member."
)
if "port" not in ref_info:
raise AKitConfigurationError(
"The 'testlandscape.py' file 'pod->reference' data requires an 'port' member."
)
return
def lookup_agent(self, serial_mapping: dict) -> TcpSerialAgent:
"""
Looks up a serial agent by serial mapping.
"""
serial_agent = None
interface_name = serial_mapping["name"]
attachment_point = serial_mapping["port"]
lscape = self.landscape
if interface_name in self._serial_config:
serial_config = self._serial_config[interface_name]
serialType = serial_config["serialType"]
if serialType == "network/tcp":
host = serial_config["host"]
ports_table = serial_config["ports"]
port = ports_table[attachment_point]
serial_agent = TcpSerialAgent(host, port)
self._serial_agent[serial_mapping] = serial_agent
else:
errmsg = "Invalid serialType=%s for serial interface %r." % (
serialType, interface_name)
raise AKitConfigurationError(errmsg) from None
else:
errmsg = "Failure to lookup serial interface %r." % interface_name
raise AKitConfigurationError(errmsg) from None
return serial_agent
def _initialize_credentials(self):
"""
"""
credential_file = get_filename_for_credentials()
if os.path.exists(credential_file):
credential_info = None
with open(credential_file, 'r') as lf:
lfcontent = lf.read()
credential_info = yaml.safe_load(lfcontent)
try:
credentials_list = credential_info["credentials"]
errors, warnings = self._validate_credentials(credentials_list)
if len(errors) == 0:
for credential in credentials_list:
if "identifier" not in credential:
raise AKitConfigurationError("Credential items in 'environment/credentials' must have an 'identifier' member.") from None
ident = credential["identifier"]
if "category" not in credential:
raise AKitConfigurationError("Credential items in 'environment/credentials' must have an 'category' member.") from None
category = credential["category"]
if category == "basic":
BasicCredential.validate(credential)
credobj = BasicCredential(**credential)
self._credentials[ident] = credobj
elif category == "ssh":
SshCredential.validate(credential)
credobj = SshCredential(**credential)
self._credentials[ident] = credobj
else:
warnmsg = "Unknown category '{}' found in credential '{}'".format(category, ident)
logger.warn(warnmsg)
else:
errmsg_lines = [
"Errors found in credential file={}".format(credential_file),
"ERRORS:"
]
for err in errors:
errmsg_lines.append(" {}".format(err))
errmsg_lines.append("WARNINGS:")
for warn in warnings:
errmsg_lines.append(" {}".format(warn))
errmsg = os.linesep.join(errmsg_lines)
raise AKitConfigurationError(errmsg)
except KeyError:
errmsg = "No 'credentials' field found in file={}".format(credential_file)
raise AKitConfigurationError(errmsg)
else:
warnmsg = "Credential file not found. expected={}".format(credential_file)
logger.warn(warnmsg)
return
def lookup_database_connection_factory(conn_profile: str):
global database_connection_factories
conn_factory = None
if conn_profile in database_connection_factories:
conn_factory = database_connection_factories[conn_profile]
else:
from akit.environment.context import Context
ctx = Context()
conn_info = None
rcdatabases = ctx.lookup(ContextPaths.DATABASES)
if rcdatabases is not None and conn_profile in rcdatabases:
conn_info = rcdatabases[conn_profile].value
else:
from akit.interop.landscaping.landscape import Landscape
lscape = Landscape()
lsdatabases = lscape.databases
if lsdatabases is not None and conn_profile in lsdatabases:
conn_info = lsdatabases[conn_profile]
if conn_info is not None:
if "conntype" in conn_info:
conntype = conn_info["conntype"].lower()
if conntype == "basic":
conn_factory = BasicDatabaseConnectionFactory(
conn_profile, **conn_info)
elif conntype == "basic-tcp":
conn_factory = BasicTcpDatabaseConnectionFactory(
conn_profile, **conn_info)
else:
errmsg = "Unknown database connection type. connection={} conntype={}".format(
conn_profile, conntype)
raise AKitConfigurationError(errmsg)
database_connection_factories[conn_profile] = conn_factory
else:
errmsg = "Database connection entries must have a 'conntype' entry. connection={}".format(
conn_profile)
raise AKitConfigurationError(errmsg)
else:
errmsg = "Database connection not found. connection={}".format(
conn_profile)
raise AKitConfigurationError(errmsg)
return conn_factory
def _lookup_power_interface(self,
interface_name: str) -> Union[dict, None]:
"""
Looks up a power interface by power interface name.
"""
power_iface = None
if interface_name in self._power_interfaces:
power_iface = self._power_interfaces[interface_name]
else:
lscape = self.landscape
interface_config = self._power_config[interface_name]
powerType = interface_config["powerType"]
if powerType == "DliPowerSwitch":
model = interface_config["model"]
host = interface_config["host"]
credential_name = interface_config["credential"]
credobj = lscape.lookup_credential(credential_name)
power_iface = dlipower.PowerSwitch(userid=credobj.username,
password=credobj.password,
hostname=host)
self._power_interfaces[interface_name] = power_iface
else:
errmsg = "Un-Support power interface type={}.".format(
powerType)
raise AKitConfigurationError(errmsg) from None
return power_iface
def validate(cls, cred_info):
errmsg_lines = []
allow_agent = False
if "allow_agent" in cred_info:
allow_agent = cred_info["allow_agent"]
if "identifier" not in cred_info:
errmsg_lines.append(" * missing 'identifier' parameter")
if "username" not in cred_info:
errmsg_lines.append(" * missing 'username' parameter")
if "password" not in cred_info and "keyfile" not in cred_info and not allow_agent:
errmsg_lines.append(" * missing 'password' or 'keyfile' when allow_agent is 'False'")
if "keyfile" in cred_info:
keyfile = os.path.abspath(os.path.expandvars(os.path.expanduser(cred_info["keyfile"])))
if not os.path.exists(keyfile):
errmsg_lines.append(" * specified 'keyfile={}' not found.".format(keyfile))
if len(errmsg_lines) > 0:
identifier = "????"
if "identifier" in cred_info:
identifier = cred_info["identifier"]
errmsg = "Errors found while validating the '{}' SSH credential:".format(identifier)
errmsg_lines.insert(0, errmsg)
errmsg = os.linesep.join(errmsg_lines)
raise AKitConfigurationError(errmsg) from None
return
def _load_landscape(self, log_to_directory: Optional[str] = None):
self._landscape_file = get_filename_for_landscape()
landscape_desc = self.landscape_description()
self._landscape_info = landscape_desc.load(
self._landscape_file, log_to_directory=log_to_directory)
if "environment" not in self._landscape_info:
err_msg = "The landscape file must have an 'environment' decription. (%s)" % self._landscape_file
raise AKitConfigurationError(err_msg) from None
self._environment_info = self._landscape_info["environment"]
if "label" not in self._environment_info:
err_msg = "The landscape 'environment' decription must have a 'label' member (development, production, test). (%s)" % self._landscape_file
raise AKitConfigurationError(err_msg) from None
return
def open_apod_database(db_profile_name: str):
"""
Opens the 'apod' postgresql database.
"""
dbname = 'apod'
conn_factory = lookup_database_connection_factory(db_profile_name)
if conn_factory is None:
errmsg = "'open_apod_database' could not get a connection factory for profile={}".format(
db_profile_name
)
raise AKitConfigurationError(errmsg)
engine = conn_factory.create_engine(dbname, echo=True)
return engine
def execute_workflow(logger, *, environment: dict, parameters: dict,
tasklist: list, **kwargs):
# Publish the environment variables so they will take effect in the current
# process and any sub-processes lauched from this process
for key, val in environment.items():
os.environ[key] = val
result_code = 0
task_ordinal = 1
for task_info in tasklist:
task_label = task_info["label"]
tasktype = task_info["tasktype"]
task_module_name, task_module_class = tasktype.split("@")
task_module = import_by_name(task_module_name)
failure_section = None
if hasattr(task_module, task_module_class):
task_class = getattr(task_module, task_module_class)
task_instance = task_class(task_ordinal, task_label, task_info,
logger)
if failure_section is not None:
section = task_instance.section
if failure_section == section:
failure_section = None
else:
# Skip ahead until we file the failure section
continue
task_result = task_instance.execute(parameters=parameters,
**kwargs)
if task_result != 0:
result_code = 1
if task_instance.onfailure is not None:
failure_section = task_instance.onfailure
else:
error_msg = "The specified task module %r does not contain a class %r" % (
task_module_name, task_module_class)
raise AKitConfigurationError(error_msg) from None
return result_code
def lookup_agent(self, power_mapping: dict) -> Union[DliPowerAgent, None]:
"""
Looks up a power agent by power mapping.
"""
power_agent = None
pname = power_mapping["name"]
pswitch = power_mapping["switch"]
power_iface = self._lookup_power_interface(pname)
if power_iface is not None:
power_agent = DliPowerAgent(power_iface, pswitch)
else:
errmsg = "Failure to find power interface %r." % pname
raise AKitConfigurationError(errmsg) from None
return power_agent
def _log_device_activation_results(self):
landscape_first_contact_result_file = os.path.join(get_path_for_output(), "landscape-first-contact-results.json")
with open(landscape_first_contact_result_file, 'w') as fcrf:
json.dump(self._first_contact_results, fcrf, indent=4)
if len(self._activation_errors) > 0:
errmsg_lines = [
"Encountered device activation errors.",
"ACTIVATION ERROR LIST:"
]
for aerror in self._activation_errors:
errmsg_lines.append(" %s" % aerror)
errmsg = os.linesep.join(errmsg_lines)
raise AKitConfigurationError(errmsg) from None
return
def load(self, topology_file: str, log_to_directory: Optional[str]=None):
"""
Loads and validates the topology description file.
"""
logger = getAutomatonKitLogger()
topology_info = None
with open(topology_file, 'r') as tf:
tfcontent = tf.read()
topology_info = yaml.safe_load(tfcontent)
if log_to_directory is not None:
try:
topology_file_basename = os.path.basename(topology_file)
topology_file_basename, topology_file_ext = os.path.splitext(topology_file_basename)
topology_file_copy = os.path.join(log_to_directory, "topology-declared{}".format(topology_file_ext))
shutil.copy2(topology_file, topology_file_copy)
except Exception as xcpt:
err_msg = "Error while logging the topology file (%s)%s%s" % (
topology_file, os.linesep, traceback.format_exc())
raise AKitRuntimeError(err_msg) from xcpt
errors, warnings = self.validate_topology(topology_info)
if len(errors) > 0:
errmsg_lines = [
"ERROR Topology validation failures:"
]
for err in errors:
errmsg_lines.append(" %s" % err)
errmsg = os.linesep.join(errmsg_lines)
raise AKitConfigurationError(errmsg) from None
if len(warnings) > 0:
for wrn in warnings:
logger.warn("Topology Configuration Warning: (%s)" % wrn)
return topology_info
def attach_to_environment(cls, constraints: Dict = {}):
"""
This API is called so that the IntegrationCoupling can process configuration information. The :class:`IntegrationCoupling`
will verify that it has a valid environment and configuration to run in.
:raises :class:`akit.exceptions.AKitMissingConfigError`, :class:`akit.exceptions.AKitInvalidConfigError`:
"""
resources_acquired = False
upnp_device_hints = cls.landscape.get_upnp_device_config_lookup_table()
if len(upnp_device_hints) > 0:
resources_acquired = True
if resources_acquired:
cls.landscape.activate_integration_point("coordinator/upnp",
cls.create_coordinator)
else:
errmsg = "The required UPNP resources were not configured."
raise AKitConfigurationError(errmsg) from None
return
def filter_credentials(device_info, credential_lookup, category):
"""
Looks up the credentials associated with a device and returns the credentials found
that match a given category.
:param device_info: Device information dictionary with credential names to reference.
:param credential_lookup: A credential lookup dictionary that is used to convert credential
names into credential objects loaded from the landscape.
:param category: The category of credentials to return when filtering credentials.
"""
cred_found_list = []
cred_name_list = device_info["credentials"]
for cred_name in cred_name_list:
if cred_name in credential_lookup:
credential = credential_lookup[cred_name]
if credential.category == category:
cred_found_list.append(credential)
else:
error_lines = [
"The credential '{}' was not found in the credentials list.",
"DEVICE:"
]
dev_repr_lines = pprint.pformat(device_info,
indent=4).splitlines(False)
for dline in dev_repr_lines:
error_lines.append(" " + dline)
error_lines.append("CREDENTIALS:")
cred_available_list = [cname for cname in credential_lookup.keys()]
cred_available_list.sort()
for cred_avail in cred_available_list:
error_lines.append(" " + cred_avail)
errmsg = os.linesep.join(error_lines)
raise AKitConfigurationError(errmsg) from None
return cred_found_list
def _initialize_device_of_type(self, dev_type: str, dev_config_info: dict):
if dev_type == "network/upnp":
upnp_info = dev_config_info["upnp"]
keyid = upnp_info["USN"]
self._create_landscape_device(keyid, dev_type, dev_config_info)
elif dev_type == "network/ssh":
keyid = dev_config_info["host"]
self._create_landscape_device(keyid, dev_type, dev_config_info)
else:
errmsg_lines = [
"Unknown device type %r in configuration file." % dev_type,
"DEVICE INFO:"
]
errmsg_lines.extend(
split_and_indent_lines(
pprint.pformat(dev_config_info, indent=4), 1))
errmsg = os.linesep.join(errmsg_lines)
raise AKitConfigurationError(errmsg) from None
return
def validate(cls, cred_info):
errmsg_lines = []
if "password" not in cred_info:
errmsg_lines.append(" * missing 'password' in basic credential.")
if "username" not in cred_info:
errmsg_lines.append(" * missing 'username' in basic credential.")
if len(errmsg_lines) > 0:
identifier = "????"
if "identifier" in cred_info:
identifier = cred_info["identifier"]
errmsg = "Errors found while validating the '{}' basic credential:".format(identifier)
errmsg_lines.insert(0, errmsg)
errmsg = os.linesep.join(errmsg_lines)
raise AKitConfigurationError(errmsg) from None
return
from akit.environment.variables import ActivationProfile, AKIT_VARIABLES
__activation_profile__ = ActivationProfile.Service
# Guard against attemps to activate more than one, activation profile.
if AKIT_VARIABLES.AKIT_ACTIVATION_PROFILE is not None:
errmsg = "An attempt was made to activate multiple environment activation profiles. profile={}".format(
AKIT_VARIABLES.AKIT_ACTIVATION_PROFILE)
raise AKitSemanticError(errmsg)
AKIT_VARIABLES.AKIT_ACTIVATION_PROFILE = ActivationProfile.Service
if "AKIT_SERVICE_NAME" not in os.environ:
errmsg = "To use the AutomationKit to provide a service, you must " \
"set the AKIT_SERVICE_NAME environment variable."
raise AKitConfigurationError(errmsg)
service_name = os.environ["AKIT_SERVICE_NAME"]
AKIT_VARIABLES.AKIT_LOG_LEVEL_CONSOLE = "INFO"
AKIT_VARIABLES.AKIT_SERVICE_NAME = service_name
AKIT_VARIABLES.AKIT_JOBTYPE = "service"
AKIT_VARIABLES.AKIT_OUTPUT_DIRECTORY = "~/akit/services/{}".format(
service_name)
# For console activation we don't want to log to the console and we want
# to point the logs to a different output folder
os.environ["AKIT_LOG_LEVEL_CONSOLE"] = AKIT_VARIABLES.AKIT_LOG_LEVEL_CONSOLE
os.environ["AKIT_JOBTYPE"] = AKIT_VARIABLES.AKIT_JOBTYPE
os.environ["AKIT_OUTPUT_DIRECTORY"] = AKIT_VARIABLES.AKIT_OUTPUT_DIRECTORY
def load(self, landscape_file: str, log_to_directory: Optional[str]=None):
"""
Loads and validates the landscape description file.
"""
logger = getAutomatonKitLogger()
landscape_info = None
with open(landscape_file, 'r') as lf:
lfcontent = lf.read()
landscape_info = yaml.safe_load(lfcontent)
if log_to_directory is not None:
try:
landscape_file_basename = os.path.basename(landscape_file)
landscape_file_basename, landscape_file_ext = os.path.splitext(landscape_file_basename)
landscape_file_copy = os.path.join(log_to_directory, "landscape-declared{}".format(landscape_file_ext))
shutil.copy2(landscape_file, landscape_file_copy)
# Create a json copy of the landscape file until the time when we can
# parse yaml in the test summary javascript.
landscape_info_copy = copy.deepcopy(landscape_info)
landscape_file_copy = os.path.join(log_to_directory, "landscape-declared.json")
with open(landscape_file_copy, 'w') as lsf:
json.dump(landscape_info_copy, lsf, indent=4)
except Exception as xcpt:
err_msg = "Error while logging the landscape file (%s)%s%s" % (
landscape_file, os.linesep, traceback.format_exc())
raise AKitRuntimeError(err_msg) from xcpt
errors, warnings = self.validate_landscape(landscape_info)
if len(errors) > 0:
errmsg_lines = [
"ERROR Landscape validation failures:"
]
for err in errors:
errmsg_lines.append(" %s" % err)
errmsg = os.linesep.join(errmsg_lines)
raise AKitConfigurationError(errmsg) from None
if len(warnings) > 0:
for wrn in warnings:
logger.warn("Landscape Configuration Warning: (%s)" % wrn)
if "devices" in landscape_info["pod"]:
devices = landscape_info["pod"]["devices"]
device_lookup_table = {}
for dev in devices:
dev_type = dev["deviceType"]
if dev_type == "network/upnp":
dkey = "UPNP:{}".format(dev["upnp"]["USN"]).upper()
device_lookup_table[dkey] = dev
elif dev_type == "network/ssh":
dkey = "SSH:{}".format(dev["host"]).upper()
device_lookup_table[dkey] = dev
ctx = Context()
skipped_devices = ctx.lookup(ContextPaths.SKIPPED_DEVICES, default=[])
for dev_key in skipped_devices:
dev_key = dev_key.upper()
if dev_key in device_lookup_table:
device = device_lookup_table[dev_key]
device["skip"] = True
return landscape_info
def attach_to_devices(self,
sshdevices: List[LandscapeDevice],
upnp_coord: Optional[UpnpCoordinator] = None):
"""
Processes a list of device configs and creates and registers devices and SSH device extensions
attached with the landscape for the devices not already registered. If a device has already
been registered by the UPNP coordinator then a device extension is created and attached to the
existing device.
:param sshdevices: A list of ssh device configuration dictionaries.
:param upnp_coord: The UpnpCoordinator singleton instance.
"""
lscape = self.landscape
ssh_config_errors = []
ssh_devices_available = []
ssh_devices_unavailable = []
for sshdev in sshdevices:
sshdev_config = sshdev.device_config
ssh_credential_list = sshdev.ssh_credentials
if len(ssh_credential_list) == 0:
errmsg = format_ssh_device_configuration_error(
"All SSH devices must have at least one valid credential.",
sshdev_config)
raise AKitConfigurationError(errmsg) from None
ssh_cred_by_role = {}
for ssh_cred in ssh_credential_list:
cred_role = ssh_cred.role
if cred_role not in ssh_cred_by_role:
ssh_cred_by_role[cred_role] = ssh_cred
else:
errmsg = format_ssh_device_configuration_error(
"The SSH device had more than one credentials with the role '{}'"
.format(cred_role), sshdev_config)
raise AKitConfigurationError(errmsg) from None
if "priv" not in ssh_cred_by_role:
errmsg = format_ssh_device_configuration_error(
"All SSH devices must have a 'priv' credential.",
sshdev_config)
raise AKitConfigurationError(errmsg) from None
priv_cred = ssh_cred_by_role["priv"]
dev_type = sshdev_config["deviceType"]
host = None
usn = None
if "host" in sshdev_config:
host = sshdev_config["host"]
elif dev_type == "network/upnp":
usn = sshdev_config["upnp"]["USN"]
if upnp_coord is not None:
dev = upnp_coord.lookup_device_by_usn(usn)
if dev is not None:
ipaddr = dev.upnp.IPAddress
host = ipaddr
sshdev_config["host"] = host
self._cl_usn_to_ip_lookup[usn] = ipaddr
else:
ssh_config_errors.append(sshdev_config)
if host is not None:
called_id = dev.identity
ip = socket.gethostbyname(host)
self._cl_ip_to_host_lookup[ip] = host
agent = SshAgent(host,
priv_cred,
users=ssh_cred_by_role,
called_id=called_id)
sshdev_config["ipaddr"] = agent.ipaddr
try:
status, stdout, stderr = agent.run_cmd("echo Hello")
if status == 0 and stdout.strip() == "Hello":
ssh_devices_available.append(sshdev_config)
else:
ssh_devices_unavailable.append(sshdev_config)
except:
ssh_devices_unavailable.append(sshdev_config)
self._cl_children[host] = agent
coord_ref = weakref.ref(self)
basedevice = None
if usn is not None:
basedevice = lscape._internal_lookup_device_by_keyid(usn) # pylint: disable=protected-access
else:
basedevice = lscape._create_landscape_device(
host, dev_type, sshdev_config)
basedevice = lscape._enhance_landscape_device(
basedevice, agent)
basedevice.initialize_features()
basedevice.attach_extension("ssh", agent)
basedevice_ref = weakref.ref(basedevice)
agent.initialize(coord_ref, basedevice_ref, host, ip,
sshdev_config)
# If this device does not have a USN then it is not a UPNP device so the
# SshPoolCoordinator is repsonsible for activating it
if usn is None:
lscape._internal_activate_device(host)
else:
ssh_config_errors.append(sshdev_config)
self._available_devices = ssh_devices_available
self._unavailable_devices = ssh_devices_unavailable
return ssh_config_errors, ssh_devices_available, ssh_devices_unavailable