def clone(self, name: str) -> str:
"""
Clone a remote directory into the store.
:param name: Name of repository
:return: (string): Path to cloned repository
"""
if not name:
raise ValueError("Missing or bad name passed to Clone command.")
self.name = name
path = self.store + os.sep + name
self.path = path
if os.path.exists(path):
self.Repo = Repo(path)
# FIX for #56
if self.repo_url not in self.Repo.remotes.origin.urls:
logger.info('Found new remote URL for this named repo')
try:
# only recourse is to remove the .git directory
if os.path.exists(os.path.join(path, '.git')):
shutil.rmtree(path)
else:
raise SkilletLoaderException(
'Refusing to remove non-git directory')
except OSError:
raise SkilletLoaderException('Repo directory exists!')
logger.debug("Cloning into {}".format(path))
try:
self.Repo = Repo.clone_from(self.repo_url, path)
except GitCommandError as gce:
raise SkilletLoaderException(
f'Could not clone repository {gce}')
else:
logger.debug("Updating repository...")
try:
self.Repo.remotes.origin.pull()
except GitCommandError as gce:
logger.error('Could not clone repository!')
raise SkilletLoaderException(
f'Error Cloning repository {gce}')
return path
else:
logger.debug("Cloning into {}".format(path))
self.Repo = Repo.clone_from(self.repo_url, path)
self.path = path
return path
def cherry_pick_element(self, element: str, cherry_pick_path: str) -> str:
"""
Cherry picking allows the skillet builder to pull out specific bits of a larger configuration
and load only the smaller chunks. This is especially useful when combined with 'when' conditionals
:param element: string containing the jinja templated xml fragment
:param cherry_pick_path: string describing the relative xpath to use to cherry pick an xml node from the
element given as a parameter
:return: rendered and cherry_picked element
"""
# first, we need to render the entire element so we can parse it with xpath
rendered_element = self.render(element, self.context).strip()
# convert this string into an xml doc we can search for the cherry_pick path
try:
element_doc = elementTree.fromstring(
f'<xml>{rendered_element}</xml>')
cherry_picked_element = element_doc.find(cherry_pick_path)
if cherry_picked_element is None:
raise SkilletLoaderException(
'Could not locate cherry_pick path in source xml! '
'Check the cherry_pick xpath!')
new_element = elementTree.tostring(cherry_picked_element).strip()
return new_element
except ParseError:
raise SkilletLoaderException(
f'Could not parse element for cherry picking for snippet: {self.name}'
)
def _parse_skillet(self, path: (str, Path)) -> dict:
if type(path) is str:
path_str = path
path_obj = Path(path)
elif isinstance(path, Path):
path_str = str(path)
path_obj = path
else:
raise SkilletLoaderException(
f'Invalid path type found in _parse_skillet!')
if 'meta-cnc' in path_str:
meta_cnc_file = path_obj
if not path_obj.exists():
raise SkilletNotFoundException(
f'Could not find .meta-cnc file as this location: {path}')
else:
# we were only passed a directory like '.' or something, try to find a .meta-cnc.yaml or .meta-cnc.yml
directory = path_obj
logger.debug(f'using directory {directory}')
found_meta = False
for filename in [
'.meta-cnc.yaml', '.meta-cnc.yml', 'meta-cnc.yaml',
'meta-cnc.yml'
]:
meta_cnc_file = directory.joinpath(filename)
logger.debug(f'checking now {meta_cnc_file}')
if meta_cnc_file.exists():
found_meta = True
break
if not found_meta:
raise SkilletNotFoundException(
'Could not find .meta-cnc file at this location')
snippet_path = str(meta_cnc_file.parent.absolute())
try:
with meta_cnc_file.open(mode='r') as sc:
raw_service_config = oyaml.safe_load(sc.read())
skillet = self._normalize_skillet_structure(raw_service_config)
skillet['snippet_path'] = snippet_path
return skillet
except IOError as ioe:
logger.error('Could not open metadata file in dir %s' %
meta_cnc_file.parent)
raise SkilletLoaderException(
'IOError: Could not parse metadata file in dir %s' %
meta_cnc_file.parent)
except YAMLError as ye:
logger.error(ye)
raise SkilletLoaderException(
'YAMLError: Could not parse metadata file in dir %s' %
meta_cnc_file.parent)
except Exception as ex:
logger.error(ex)
raise SkilletLoaderException(
'Exception: Could not parse metadata file in dir %s' %
meta_cnc_file.parent)
def get_skillet_with_name(
self,
skillet_name: str,
include_resolved_skillets=False) -> (Skillet, None):
"""
Returns a single skillet from the loaded skillets list that has the matching 'name' attribute
:param skillet_name: Name of the skillet to return
:param include_resolved_skillets: boolean of whether to also check the resolved skillet list
:return: Skillet or None
"""
if not self.skillets and not self.resolved_skillets:
raise SkilletLoaderException("No Skillets have been loaded!")
for skillet in self.skillets:
if skillet.name == skillet_name:
return skillet
# also check the resolved skillet list, which are skillets that are included from snippet includes
if include_resolved_skillets:
for skillet in self.resolved_skillets:
if skillet.name == skillet_name:
return skillet
return None
def _handle_base64_outputs(self, results: str) -> dict:
"""
Parses results and returns a dict containing base64 encoded values
:param results: string as returned from some action, to be encoded as base64
:return: dict containing all outputs found from the capture pattern in each output
"""
outputs = dict()
snippet_name = 'unknown'
if 'name' in self.metadata:
snippet_name = self.metadata['name']
try:
if 'outputs' not in self.metadata:
logger.info(
f'No output defined in this snippet {snippet_name}')
return outputs
for output in self.metadata['outputs']:
if 'name' not in output:
continue
results_as_bytes = bytes(results, 'utf-8')
encoded_results = urlsafe_b64encode(results_as_bytes)
var_name = output['name']
outputs[var_name] = encoded_results.decode('utf-8')
except TypeError:
raise SkilletLoaderException(
f'Could not base64 encode results {snippet_name}')
return outputs
def __get_value_from_path(self, obj: dict, config_path: str) -> Any:
if type(obj) is not dict and type(obj) is not OrderedDict:
logger.error("Supplied object is not an Object")
logger.error(
'Ensure you are passing an object here and not a string as from capture_pattern'
)
raise SkilletLoaderException(
'Incorrect object format for get_value_from_path')
if '.' in config_path or '/' in config_path:
if '.' in config_path:
separator = '.'
else:
separator = '/'
path_elements = config_path.split(separator)
first_path_element = path_elements[0]
p0 = self.__check_inner_object(obj, first_path_element)
for p in path_elements:
if self.__has_child_node(p0, p):
new_p0 = p0[p]
p0 = new_p0
else:
raise NodeNotFoundException(f'{config_path} not found!')
return p0
p0 = self.__check_inner_object(obj, config_path)
if self.__has_child_node(p0, config_path):
return p0[config_path]
else:
raise NodeNotFoundException(f'{config_path} not found!')
def get_output(self) -> Tuple[str, str]:
if not self.detach:
return '', 'success'
try:
container = self.get_container()
if self.last_logs_time is None:
return_data = container.logs()
else:
return_data = container.logs(since=self.last_logs_time)
self.last_logs_time = int(time.time())
return_str = self.__clean_output(return_data)
if container.status == 'running':
return return_str, 'running'
else:
logger.info(container.status)
return return_str, self.__get_container_status()
except APIError as ae:
raise SkilletLoaderException(
f'Could not get logs for {self.name}: {ae}')
def load_template(self, template_path: str) -> str:
"""
Utility method to load a template file and return the contents as str
:param template_path: relative path to the template to load
:return: str contents
"""
if template_path == '' or template_path is None:
logger.error('Refusing to load empty template path')
return ''
skillet_path = Path(self.path)
template_file = skillet_path.joinpath(template_path).resolve()
if template_file.exists():
with template_file.open(encoding='utf-8') as sf:
return html.unescape(sf.read())
else:
# Add the snippet name here as well to allow for more context
# fix for https://gitlab.com/panw-gse/as/panhandler/-/issues/19
logger.error(
f'Snippet: {self.name} has file attribute that does not exist')
logger.error(f'Snippet file path is: {template_path}')
raise SkilletLoaderException(
f'Snippet: {self.name} - Could not resolve template path!')
def create_skillet(self, skillet_dict: dict) -> Skillet:
skillet_type = skillet_dict['type']
if skillet_type == 'panos' or skillet_type == 'panorama' or skillet_type == 'panorama-gpcs':
from skilletlib.skillet.panos import PanosSkillet
return PanosSkillet(skillet_dict)
elif skillet_type == 'pan_validation':
from skilletlib.skillet.pan_validation import PanValidationSkillet
return PanValidationSkillet(skillet_dict)
elif skillet_type == 'python3':
from skilletlib.skillet.python3 import Python3Skillet
return Python3Skillet(skillet_dict)
elif skillet_type == 'template':
from skilletlib.skillet.template import TemplateSkillet
return TemplateSkillet(skillet_dict)
elif skillet_type == 'docker':
from skilletlib.skillet.docker import DockerSkillet
return DockerSkillet(skillet_dict)
elif skillet_type == 'rest':
from skilletlib.skillet.rest import RestSkillet
return RestSkillet(skillet_dict)
elif skillet_type == 'workflow':
from skilletlib.skillet.workflow import WorkflowSkillet
return WorkflowSkillet(skillet_dict, self)
else:
raise SkilletLoaderException('Unknown Skillet Type!')
def sanitize_metadata(self, metadata: dict) -> dict:
"""
Ensure all required keys are present in the snippet definition
:param metadata: dict
:return: dict
"""
metadata = super().sanitize_metadata(metadata)
err = f'Unknown cmd {self.cmd}'
if self.cmd in ('set', 'edit', 'override'):
if {'xpath', 'element'}.issubset(metadata):
return metadata
elif {'xpath', 'file'}.issubset(metadata):
return metadata
err = 'xpath and either file or element attributes are required for set, edit, or override cmds'
elif self.cmd in ('show', 'get', 'delete'):
if {'xpath'}.issubset(metadata):
return metadata
err = 'xpath attribute is required for show, get, or delete cmds'
elif self.cmd == 'move':
if 'where' in metadata:
return metadata
err = 'where attribute is required for move cmd'
elif self.cmd in ('rename', 'clone'):
if 'new_name' in metadata or 'newname' in metadata:
return metadata
err = 'new_name attribute is required for rename or move cmd'
elif self.cmd == 'clone':
if 'xpath_from' in metadata:
return metadata
err = 'xpath_from attribute is required for clone cmd'
elif self.cmd == 'op' or self.cmd == 'cli':
if 'cmd_str' in metadata:
return metadata
err = 'cmd_str attribute is required for op or cli cmd'
elif self.cmd == 'validate':
if {'test', 'label', 'documentation_link'}.issubset(metadata):
# configure validation outputs manually if necessary
# for validation we only need the output_type set to 'validation'
metadata['output_type'] = 'validation'
return metadata
err = 'test, label, and documentation_link are required attributes for validate cmd'
elif self.cmd == 'parse':
if {'variable', 'outputs'}.issubset(metadata):
return metadata
err = 'variable and outputs are required attributes for parse cmd'
elif self.cmd == 'validate_xml':
if {'xpath'}.issubset(metadata):
if 'file' in metadata or 'element' in metadata:
metadata['output_type'] = 'validation'
return metadata
err = 'xpath and file or element are required attributes for validate_xml cmd'
elif self.cmd == 'noop':
if 'output_type' not in metadata:
metadata['output_type'] = 'manual'
return metadata
raise SkilletLoaderException(f'Invalid metadata configuration: {err}')
def create_skillet(self, skillet_dict: dict) -> Skillet:
"""
Creates a Skillet object from the given skillet definition
:param skillet_dict: Dictionary loaded from the skillet.yaml definition file
:return: Skillet Object
"""
skillet_type = skillet_dict["type"]
if skillet_type == "panos" or skillet_type == "panorama" or skillet_type == "panorama-gpcs":
from skilletlib.skillet.panos import PanosSkillet
return PanosSkillet(skillet_dict)
elif skillet_type == "pan_validation":
from skilletlib.skillet.pan_validation import PanValidationSkillet
return PanValidationSkillet(skillet_dict)
elif skillet_type == "python3":
from skilletlib.skillet.python3 import Python3Skillet
return Python3Skillet(skillet_dict)
elif skillet_type == "template":
from skilletlib.skillet.template import TemplateSkillet
return TemplateSkillet(skillet_dict)
elif skillet_type == "docker":
from skilletlib.skillet.docker import DockerSkillet
return DockerSkillet(skillet_dict)
elif skillet_type == "rest":
from skilletlib.skillet.rest import RestSkillet
return RestSkillet(skillet_dict)
elif skillet_type == "workflow":
from skilletlib.skillet.workflow import WorkflowSkillet
return WorkflowSkillet(skillet_dict, self)
elif skillet_type == "terraform":
from skilletlib.skillet.terraform import TerraformSkillet
return TerraformSkillet(skillet_dict)
elif skillet_type == "app":
from skilletlib.skillet.app import AppSkillet
return AppSkillet(skillet_dict)
else:
raise SkilletLoaderException(
f"Unknown Skillet Type: {skillet_type}!")
def get_skillet_with_name(self, skillet_name: str) -> (Skillet, None):
if not self.all_skillets:
raise SkilletLoaderException('No Skillets have been loaded!')
for skillet in self.all_skillets:
if skillet.name == skillet_name:
return skillet
return None
def __difference(list1: list, list2: list) -> list:
"""
Returns a list of items from list1 that do not exist in list2
:param list1: list of items expected to be in list2
:param list2: list of items list1 will be evaluated against
"""
if not isinstance(list1, list) or not isinstance(list2, list):
raise SkilletLoaderException(
'difference filter takes only type list for both arguments.')
return list(set([x for x in list1 if x not in list2]))
def __json_query(obj: dict, query: str) -> Any:
"""
JMESPath query, jmespath.org for examples
:param query: JMESPath query string
:param obj: object to be queried
"""
if not isinstance(query, str):
raise SkilletLoaderException(
'json_query requires an argument of type str')
path = jmespath.search(query, obj)
return path
def sanitize_metadata(self, metadata):
"""
Ensure the configured metadata is valid for this snippet type
:param metadata: dict
:return: validated metadata dict
"""
name = metadata.get('name', '')
if not self.required_metadata.issubset(metadata):
for attr_name in metadata:
if attr_name not in self.required_metadata:
raise SkilletLoaderException(
f'Invalid snippet metadata configuration: attribute: {attr_name} '
f'is required for snippet: {name}')
return metadata
def create_skillet(self, skillet_dict: dict) -> Skillet:
"""
Creates a Skillet object from the given skillet definition
:param skillet_dict: Dictionary loaded from the skillet.yaml definition file
:return: Skillet Object
"""
skillet_type = skillet_dict['type']
if skillet_type == 'panos' or skillet_type == 'panorama' or skillet_type == 'panorama-gpcs':
from skilletlib.skillet.panos import PanosSkillet
return PanosSkillet(skillet_dict)
elif skillet_type == 'pan_validation':
from skilletlib.skillet.pan_validation import PanValidationSkillet
return PanValidationSkillet(skillet_dict)
elif skillet_type == 'python3':
from skilletlib.skillet.python3 import Python3Skillet
return Python3Skillet(skillet_dict)
elif skillet_type == 'template':
from skilletlib.skillet.template import TemplateSkillet
return TemplateSkillet(skillet_dict)
elif skillet_type == 'docker':
from skilletlib.skillet.docker import DockerSkillet
return DockerSkillet(skillet_dict)
elif skillet_type == 'rest':
from skilletlib.skillet.rest import RestSkillet
return RestSkillet(skillet_dict)
elif skillet_type == 'workflow':
from skilletlib.skillet.workflow import WorkflowSkillet
return WorkflowSkillet(skillet_dict, self)
elif skillet_type == 'terraform':
from skilletlib.skillet.terraform import TerraformSkillet
return TerraformSkillet(skillet_dict)
elif skillet_type == 'app':
from skilletlib.skillet.app import AppSkillet
return AppSkillet(skillet_dict)
else:
raise SkilletLoaderException(
f'Unknown Skillet Type: {skillet_type}!')
def get_skillet_with_name(self, skillet_name: str) -> (Skillet, None):
"""
Returns a single skillet from the loaded skillets list that has the matching 'name' attribute
:param skillet_name: Name of the skillet to return
:return: Skillet
"""
if not self.skillets:
raise SkilletLoaderException('No Skillets have been loaded!')
for skillet in self.skillets:
if skillet.name == skillet_name:
return skillet
return None
def __init__(self, repo_url, store=os.getcwd()):
"""
Initialize a new Git repo object
:param repo_url: URL path to repository.
:param store: Directory to store repository in. Defaults to the current directory.
"""
if not self.check_git_exists():
raise SkilletLoaderException(
'A git client must be installed to use this remote!')
self.repo_url = repo_url
self.store = store
self.Repo = None
self.name = ''
self.path = ''
self.update = ''
def execute(self, initial_context: dict) -> dict:
context = dict()
try:
context = self.initialize_context(initial_context)
for snippet in self.get_snippets():
# render anything that looks like a jinja template in the snippet metadata
# mostly useful for xpaths in the panos case
metadata = snippet.render_metadata(context)
# check the 'when' conditional against variables currently held in the context
if snippet.should_execute(context):
(output, status) = snippet.execute(context)
running_counter = 0
while status == 'running':
logger.info('Snippet still running...')
time.sleep(5)
(output, status) = snippet.get_output()
running_counter += 1
if running_counter > 60:
raise SkilletLoaderException('Snippet took too long to execute!')
returned_output = snippet.capture_outputs(output)
context.update(returned_output)
else:
fail_action = metadata.get('fail_action', 'skip')
fail_message = metadata.get('fail_message', 'Aborted due to failed conditional!')
if fail_action == 'skip':
logger.debug(f' Skipping Snippet: {snippet.name}')
else:
logger.debug('Conditional failed and found a fail_action')
logger.error(fail_message)
context['fail_message'] = fail_message
return context
except SkilletLoaderException as sle:
logger.error(f'Caught Exception during execution: {sle}')
except Exception as e:
logger.error(f'Exception caught: {e}')
finally:
self.cleanup()
return context
def __listify(obj: Any) -> list:
"""
Attempt to convert a string input into a list
:param obj: raw input text
"""
if isinstance(obj, list):
return obj
if isinstance(obj, str):
if '\n' in obj:
return [x.strip() for x in obj.split('\n') if x]
elif ',' in obj:
return [x.strip() for x in obj.split(',') if x]
return [obj.strip()]
raise SkilletLoaderException(
'listify filter requires input of type str.')
def __validate_snippet_metadata(self) -> None:
"""
Perform snippet metadata validation before we attempt to instantiate the snippet
This will throw a SkilletLoaderException if a required attribute is not present in the metadata
Will also set all optional metadata attributes with their default values
:raises: SkilletLoaderException if a required field is not present
:return: None
"""
for s in self.snippet_stack:
name = s.get('name', '')
for r in self.snippet_required_metadata:
if r not in s:
raise SkilletLoaderException(
f'Invalid snippet metadata configuration: attribute: {r} '
f'is required for snippet: {name}')
for k, v in self.snippet_optional_metadata.items():
if k not in s:
s[k] = v
def cleanup(self) -> None:
"""
Clean up action is the docker container was started with 'async', no-op is async is False
:return: None
"""
if not self.detach:
return
try:
container = self.get_container()
if container:
if container.status != 'running':
container.remove()
else:
logger.warning(
f'Docker container {self.container_id} may need to be manually removed!'
)
except APIError as ae:
raise SkilletLoaderException(
f'Could not clean up {self.name}: {ae}')
def execute(self, context) -> Tuple[str, str]:
"""
Execute this cmd in the specified docker container
:param context: context containing all the user-supplied input variables. Also contains output from previous
steps. Raises SkilletLoaderException on error
:return: Tuple(dict, str) output and string representing 'success' or 'failure'
"""
try:
self.client = DockerClient()
except DockerException:
logger.error(traceback.format_exc())
raise SkilletLoaderException(
f'Could not contact Docker API in {self.name}')
try:
logger.info(f'Pulling image: {self.image} with tag: {self.tag}')
self.client.images.pull(self.image, self.tag)
vols = self.volumes
image = self.image + ":" + self.tag
logger.info('Creating container...')
return_data = self.client.containers.run(
image,
self.metadata['cmd'],
volumes=vols,
stderr=True,
detach=self.detach,
working_dir=self.working_dir,
user=self.metadata.get('user', 'root'),
auto_remove=self.auto_remove,
environment=context)
if self.detach:
# return_data will be a Container object if self.detach is True
self.container_id = return_data.id
output = self.container_id
print('container id is ' + self.container_id)
return output, 'running'
else:
# return_data will be the bytes returned from the command
if type(return_data) is bytes:
return_str = return_data.decode('UTF-8')
return return_str, self.__get_container_status()
else:
return return_data, self.__get_container_status()
except ImageNotFound:
logger.error(traceback.format_exc())
raise SkilletLoaderException(
f'Could not locate image {self.image} in {self.name}')
except APIError as ae:
logger.error(traceback.format_exc())
raise SkilletLoaderException(
f'Error communicating with Docker API: {ae}')
except ContainerError as ce:
logger.error(traceback.format_exc())
raise SkilletLoaderException(f'Container command failed: {ce}')
except DockerException as de:
logger.error(traceback.format_exc())
raise SkilletLoaderException(
f'Could not execute docker container {self.name}: {de}')
except ValueError as ve:
# added or GL #77 - add diagnostics for failed docker container creation
logger.error(traceback.format_exc())
raise SkilletLoaderException(
f'Could not execute docker container ValueError in {self.name}: {ve}'
)
def execute(self, initial_context: dict) -> dict:
"""
The heart of the Skillet class. This method executes the skillet by iterating over all the skillets returned
from the 'get_skillets' method. Each one is checked if it should be executed if a 'when' conditional attribute
is found, and if so, is executed using the snippet execute method.
:param initial_context: context of key values pairs to use for the execution. By default this is all the
variables defined in the skillet file with their default values. Updates from user input, the environment, etc
will override these default values via the 'update_context' method.
:return: a dict containing the updated context containing the output of each of the snippets
"""
try:
context = self.initialize_context(initial_context)
logger.debug(f'Executing Skillet: {self.name}')
for snippet in self.get_snippets():
try:
# render anything that looks like a jinja template in the snippet metadata
# mostly useful for xpaths in the panos case
snippet.render_metadata(context)
# check the 'when' conditional against variables currently held in the context
if snippet.should_execute(context):
(output, status) = snippet.execute(context)
logger.debug(f'{snippet.name} - status: {status}')
if output:
logger.debug(f'{snippet.name} - output: {output}')
running_counter = 0
while status == 'running':
logger.info('Snippet still running...')
time.sleep(5)
(output, status) = snippet.get_output()
running_counter += 1
if running_counter > 60:
raise SkilletLoaderException(
'Snippet took too long to execute!')
# capture all outputs
snippet_outputs = snippet.get_default_output(
output, status)
captured_outputs = snippet.capture_outputs(
output, status)
if captured_outputs:
logger.debug(
f'{snippet.name} - captured_outputs: {captured_outputs}'
)
self.snippet_outputs.update(snippet_outputs)
self.captured_outputs.update(captured_outputs)
context.update(snippet_outputs)
context.update(captured_outputs)
except SkilletLoaderException as sle:
logger.error(f'Caught Exception during execution: {sle}')
snippet_outputs = snippet.get_default_output(
str(sle), 'error')
logger.error(snippet_outputs)
self.snippet_outputs.update(snippet_outputs)
except Exception as e:
logger.error(f'Exception caught: {e}')
snippet_outputs = snippet.get_default_output(
str(e), 'error')
self.snippet_outputs.update(snippet_outputs)
finally:
self.cleanup()
return self.get_results()
def compile_skillet_dict(self, skillet: dict) -> dict:
"""
Compile the skillet dictionary including any included snippets from other skillets. Included snippets and
variables will be inserted into the skillet dictionary and any replacements / updates to those snippets /
variables will be made before hand.
:param skillet: skillet definition dictionary
:return: full compiled skillet definition dictionary
"""
snippets = list()
variables: list = skillet['variables']
for snippet in skillet.get('snippets', []):
if 'include' not in snippet:
snippets.append(snippet)
continue
included_skillet: Skillet = self.get_skillet_with_name(
snippet['include'], include_resolved_skillets=True)
if included_skillet is None:
raise SkilletLoaderException(
f'Could not find included Skillet with name: {snippet["include"]}'
)
if 'include_snippets' not in snippet and 'include_variables' not in snippet:
# include all snippets by default
for included_snippet in included_skillet.snippet_stack:
propagated_snippet = self.__propagate_snippet_metadata(
snippet, included_snippet)
snippets.append(propagated_snippet)
for v in included_skillet.variables:
found_variable = False
for tv in skillet['variables']:
if tv['name'] == v['name']:
# do not add variable if one with the same name already exists
found_variable = True
if not found_variable:
# this variable does not exist in the skillet_dict variables, so add it here
variables.append(v)
elif 'include_snippets' not in snippet:
# include all snippets by default
for included_snippet in included_skillet.snippet_stack:
included_snippet = self.__propagate_snippet_metadata(
snippet, included_snippet)
snippets.append(included_snippet)
else:
for include_snippet in snippet['include_snippets']:
include_snippet_name = include_snippet['name']
include_snippet_object = included_skillet.get_snippet_by_name(
include_snippet_name)
include_meta = include_snippet_object.metadata
# the meta attribute in the metadata is a dict that we do not want to completely overwrite
if 'meta' in include_snippet:
include_snippet_object_meta = include_meta.get(
'meta', {})
if isinstance(include_snippet_object_meta, dict) and \
isinstance(include_snippet.get('meta', {}), dict):
new_meta = include_snippet_object_meta.copy()
new_meta.update(include_snippet.get('meta', {}))
include_snippet['meta'] = new_meta
# propagate everything form the parent if it's there
include_meta = self.__propagate_snippet_metadata(
snippet, include_meta)
# update with locally defined options as well, if anyu
include_meta.update(include_snippet)
# ensure the name is set properly
include_meta[
'name'] = f'{included_skillet.name}.{include_snippet_name}'
snippets.append(include_meta)
if 'include_variables' in snippet:
if isinstance(snippet['include_variables'],
str) and snippet['include_variables'] == 'all':
for v in included_skillet.variables:
found_variable = False
for tv in skillet['variables']:
if tv['name'] == v['name']:
# do not add variable if one with the same name already exists
found_variable = True
if not found_variable:
# this variable does not exist in the skillet_dict variables, so add it here
variables.append(v)
elif isinstance(snippet['include_variables'], list):
for v in snippet['include_variables']:
# we need to include only the variables listed here and possibly update them with any
# new / modified attributes
included_variable_orig = included_skillet.get_variable_by_name(
v['name'])
# #163 - always uses deepcopy when using includes / overrides
included_variable = copy.deepcopy(
included_variable_orig)
# update this variable definition accordingly if necessary
included_variable.update(v)
# now check to see if this skillet has this variable already defined
found_variable = False
for ev in variables:
if ev['name'] == v['name']:
found_variable = True
# it is nonsensical to update the variable definition here from the included skillet
# just use what is defined locally, otherwise the builder should not have defined it
# here!
logger.info(
'not updating existing variable definition from '
'the resolved skillet definition')
if not found_variable:
# this included variable was not defined locally, so go ahead and append the updated version
variables.append(included_variable)
skillet['snippets'] = snippets
skillet['variables'] = variables
return skillet
def __handle_xml_outputs(self, output_definition: dict,
results: str) -> dict:
"""
Parse the results string as an XML document
Example skillet.yaml snippets section:
snippets:
- name: system_info
path: /api/?type=op&cmd=<show><system><info></info></system></show>&key={{ api_key }}
output_type: xml
outputs:
- name: hostname
capture_value: result/system/hostname
- name: uptime
capture_value: result/system/uptime
- name: sw_version
capture_value: result/system/sw-version
:param results: string as returned from some action, to be parsed as XML document
:return: dict containing all outputs found from the capture pattern in each output
"""
captured_output = dict()
def unique_tag_list(elements: list) -> bool:
tag_list = list()
for el in elements:
# some xpath queries can return a list of str
if isinstance(el, str):
return False
if el.tag not in tag_list:
tag_list.append(el.tag)
if len(tag_list) == 1:
# all tags in this list are the same
return False
else:
# there are unique tags in this list
return True
def convert_entry(el: elementTree.Element):
# force_lists always returns a list even though in most cases, we really only want a single item
# due to an exact xpath match. However, we might still want force_list to apply further down in the
# the document.
tag_name = el.tag
res = xmltodict.parse(elementTree.tostring(el),
force_list=self.xml_force_list_keys)
if tag_name not in self.xml_force_list_keys:
return res
if tag_name in res and \
isinstance(res[tag_name], list) and \
len(res[tag_name]) == 1:
# unwind unnecessary list at the top level here
return {tag_name: res[tag_name][0]}
return res
try:
xml_doc = etree.XML(results)
# xml_doc = elementTree.fromstring(results)
# allow jinja syntax in capture_pattern, capture_value, capture_object etc
local_context = self.context.copy()
output = self.__render_output_metadata(output_definition,
local_context)
var_name = output['name']
if 'capture_pattern' in output or 'capture_value' in output:
if 'capture_value' in output:
capture_pattern = output['capture_value']
else:
capture_pattern = output['capture_pattern']
# by default we will attempt to return the text of the found element
return_type = 'text'
entries = xml_doc.xpath(capture_pattern)
logger.debug(f'found entries: {entries}')
if len(entries) == 0:
captured_output[var_name] = ''
elif len(entries) == 1:
entry = entries.pop()
if isinstance(entry, str):
captured_output[var_name] = str(entry)
else:
if len(entry) == 0:
# this tag has no children, so try to grab the text
if return_type == 'text':
captured_output[var_name] = str(
entry.text).strip()
else:
captured_output[var_name] = entry.tag
else:
# we have 1 Element returned, so the user has a fairly specific xpath
# however, this element has children itself, so we can't return a text value
# just return the tag name of this element only
captured_output[var_name] = entry.tag
else:
# we have a list of elements returned from the users xpath query
capture_list = list()
# are there unique tags in this list? or is this a list of the same tag names?
if unique_tag_list(entries):
return_type = 'tag'
for entry in entries:
if isinstance(entry, str):
capture_list.append(entry)
else:
if len(entry) == 0:
if return_type == 'text':
if entry.text is not None:
capture_list.append(entry.text.strip())
else:
# If there is no text, then try to grab a sensible attribute
# if you need more control than this, then you should first
# capture_object to convert to a python object then use a jinja filter
# to get what you need
if 'value' in entry.attrib:
capture_list.append(
entry.attrib.get('value', ''))
elif 'name' in entry.attrib:
capture_list.append(
entry.attrib.get('name', ''))
else:
capture_list.append(
json.dumps(dict(entry.attrib)))
else:
capture_list.append(entry.tag)
else:
capture_list.append(entry.tag)
captured_output[var_name] = capture_list
elif 'capture_object' in output:
capture_pattern = output['capture_object']
entries = xml_doc.xpath(capture_pattern)
if len(entries) == 0:
captured_output[var_name] = None
elif len(entries) == 1:
captured_output[var_name] = convert_entry(entries.pop())
else:
capture_list = list()
for entry in entries:
capture_list.append(convert_entry(entry))
# FIXME - isn't this duplicated below?
captured_output[var_name] = self.__filter_outputs(
output, capture_list, self.context)
elif 'capture_list' in output:
capture_pattern = output['capture_list']
entries = xml_doc.xpath(capture_pattern)
capture_list = list()
for entry in entries:
if isinstance(entry, str):
capture_list.append(entry)
else:
capture_list.append(convert_entry(entry))
captured_output[var_name] = capture_list
elif 'capture_xml' in output:
capture_pattern = output['capture_xml']
entries = xml_doc.xpath(capture_pattern)
if len(entries) == 0:
captured_output[var_name] = None
elif len(entries) == 1:
captured_output[var_name] = etree.tostring(
entries.pop(), encoding='unicode')
else:
outer_tag = etree.fromstring('<xml/>')
for e in entries:
outer_tag.append(e)
found_entries_str = etree.tostring(outer_tag,
encoding='unicode')
captured_output[var_name] = found_entries_str
# short circuit return here as it makes no sense to do the filtering on a plain string object
return captured_output
# filter selected items here
captured_output[var_name] = self.__filter_outputs(
output, captured_output[var_name], local_context)
except ParseError:
logger.error('Could not parse XML document in output_utils')
# just return blank captured_outputs here
raise SkilletLoaderException(
f'Could not parse output as XML in {self.name}')
return captured_output
def execute(self, initial_context: dict) -> dict:
"""
The heart of the Skillet class. This method executes the skillet by iterating over all the skillets returned
from the 'get_skillets' method. Each one is checked if it should be executed if a 'when' conditional attribute
is found, and if so, is executed using the snippet execute method.
:param initial_context: context of key values pairs to use for the execution. By default this is all the
variables defined in the skillet file with their default values. Updates from user input, the environment, etc
will override these default values via the 'update_context' method.
:return: a dict containing the updated context containing the output of each of the snippets
"""
try:
# reset success on execution
self.success = True
context = self.initialize_context(initial_context)
logger.debug(f'Executing Skillet: {self.name}')
for snippet in self.get_snippets():
try:
# allow subclasses to override this
snippet.update_context(context)
loop_vars = snippet.get_loop_parameter()
index = 0
for item in loop_vars:
context['loop'] = item
context['loop_index'] = index
# check the 'when' conditional against variables currently held in the context
if snippet.should_execute(context):
# fix for #136
snippet.render_metadata(context)
(output, status) = snippet.execute(context)
logger.debug(f'{snippet.name} - status: {status}')
if status != 'success':
self.success = False
if output:
logger.debug(
f'{snippet.name} - output: {output}')
running_counter = 0
while status == 'running':
logger.info('Snippet still running...')
time.sleep(5)
(output, status) = snippet.get_output()
running_counter += 1
if running_counter > 60:
raise SkilletLoaderException(
'Snippet took too long to execute!')
# capture all outputs
snippet_outputs = snippet.get_default_output(
output, status)
captured_outputs = snippet.capture_outputs(
output, status)
if snippet.name in self.snippet_outputs:
self.snippet_outputs[snippet.name].append(
snippet_outputs)
else:
# create a list of track progress here
self.snippet_outputs[snippet.name] = [
snippet_outputs
]
if captured_outputs:
logger.debug(
f'{snippet.name} - captured_outputs: {captured_outputs}'
)
# fixme - how does this interact with looping?
self.captured_outputs.update(captured_outputs)
# simple context addition here, does not count on iteration ?
# context.update(snippet_outputs)
context.update(captured_outputs)
index = index + 1
snippet.reset_metadata()
except SkilletLoaderException as sle:
self.success = False
logger.error(f'Caught Exception during execution: {sle}')
snippet_outputs = snippet.get_default_output(
str(sle), 'error')
logger.error(snippet_outputs)
if snippet.name in self.snippet_outputs:
self.snippet_outputs[snippet.name].append(
snippet_outputs)
else:
self.snippet_outputs[snippet.name] = [snippet_outputs]
except Exception as e:
self.success = False
logger.error(
f'Exception caught in snippet: {snippet.name}: {e}')
snippet_outputs = snippet.get_default_output(
str(e), 'error')
if snippet.name in self.snippet_outputs:
self.snippet_outputs[snippet.name].append(
snippet_outputs)
else:
self.snippet_outputs[snippet.name] = [snippet_outputs]
finally:
self.cleanup()
return self.get_results()
def compile_skillet_dict(self, skillet: dict) -> dict:
"""
Compile the skillet dictionary including any included snippets from other skillets. Included snippets and
variables will be inserted into the skillet dictionary and any replacements / updates to those snippets /
variables will be made before hand.
:param skillet: skillet definition dictionary
:return: full compiled skillet definition dictionary
"""
snippets = list()
variables: list = skillet["variables"]
parent_only_variables: list = variables.copy()
for snippet in skillet.get("snippets", []):
if "include" not in snippet:
snippets.append(snippet)
continue
included_skillet: Skillet = self.get_skillet_with_name(
snippet["include"], include_resolved_skillets=True)
if included_skillet is None:
raise SkilletLoaderException(
f'Could not find included Skillet with name: {snippet["include"]}'
)
if "include_snippets" not in snippet:
# include all snippets by default
for included_snippet in included_skillet.snippet_stack:
include_snippet_name = included_snippet["name"]
included_meta = self.__propagate_snippet_metadata(
snippet, included_snippet)
included_meta[
"name"] = f"{included_skillet.name}.{include_snippet_name}"
snippets.append(included_meta)
else:
for include_snippet in snippet["include_snippets"]:
include_snippet_name = include_snippet["name"]
include_snippet_object = included_skillet.get_snippet_by_name(
include_snippet_name)
include_meta = include_snippet_object.metadata
# the meta attribute in the metadata is a dict that we do not want to completely overwrite
if "meta" in include_snippet:
include_snippet_object_meta = include_meta.get(
"meta", {})
if isinstance(include_snippet_object_meta,
dict) and isinstance(
include_snippet.get("meta", {}),
dict):
new_meta = include_snippet_object_meta.copy()
new_meta.update(include_snippet.get("meta", {}))
include_snippet["meta"] = new_meta
# propagate everything form the parent if it's there
include_meta = self.__propagate_snippet_metadata(
snippet, include_meta)
# update with locally defined options as well, if anyu
include_meta.update(include_snippet)
# ensure the name is set properly
include_meta[
"name"] = f"{included_skillet.name}.{include_snippet_name}"
snippets.append(include_meta)
if "include_variables" not in snippet:
for v in included_skillet.variables:
variables = self.__update_variable_list(
variables, parent_only_variables, v, False)
else:
if isinstance(snippet["include_variables"],
str) and snippet["include_variables"] == "all":
# incorporate every variable into the compiled skillet's variable list
for v in included_skillet.variables:
variables = self.__update_variable_list(
variables, parent_only_variables, v, False)
elif isinstance(snippet["include_variables"], list):
# handle case where we have a single dict item with name == 'all' to override all snippet attributes
# see issue #182 for details
if len(snippet["include_variables"]) == 1 and snippet[
"include_variables"][0]["name"] == "all":
override_attribute = snippet["include_variables"][0]
for v in included_skillet.variables:
original_name = v["name"]
# #163 - always uses deepcopy when using includes / overrides
overridden_variable = copy.deepcopy(v)
# update this variable definition accordingly if necessary
overridden_variable.update(override_attribute)
# reformat name from 'all' back to original name
overridden_variable["name"] = original_name
# incorporate variable into compiled list but merge if var exists
variables = self.__update_variable_list(
variables, parent_only_variables,
overridden_variable, True)
# handle case where there's a list of variable to include and override
else:
for v in snippet["include_variables"]:
# we need to include only the variables listed here and possibly update them with any
# new / modified attributes
included_variable_orig = included_skillet.get_variable_by_name(
v["name"])
# #163 - always uses deepcopy when using includes / overrides
included_variable = copy.deepcopy(
included_variable_orig)
# update this variable definition accordingly if necessary
included_variable.update(v)
# incorporate variable into compiled list and no merge since in this case the
# child variable overrides anything that exists
variables = self.__update_variable_list(
variables, parent_only_variables,
included_variable, False)
skillet["snippets"] = snippets
skillet["variables"] = variables
return skillet
def _parse_skillet(self, path: (str, Path)) -> dict:
"""
Parse the skillet metadata file from the Path and return a valid skillet definition dictionary
:param path: relative PosixPath of a file to load and validate
:return: skillet definition dictionary
"""
if type(path) is str:
path_str = path
path_obj = Path(path)
elif isinstance(path, Path):
path_str = str(path)
path_obj = path
else:
raise SkilletLoaderException(
"Invalid path type found in _parse_skillet!")
if "meta-cnc" in path_str or "skillet.y" in path_str:
meta_cnc_file = path_obj
if not path_obj.exists():
raise SkilletNotFoundException(
f"Could not find skillet.yaml file as this location: {path}"
)
else:
# we were only passed a directory like '.' or something, try to find a skillet.yaml or .meta-cnc.yml
directory = path_obj
logger.debug(f"using directory {directory}")
found_files = list()
found_files.extend(directory.glob(".meta-cnc.y*"))
found_files.extend(directory.glob("*skillet.y*"))
if not found_files:
raise SkilletNotFoundException(
"Could not find skillet definition file at this location")
if len(found_files) > 1:
logger.warning(
"Found more than 1 skillet file at this location! Using first file found!"
)
meta_cnc_file = found_files[0]
if meta_cnc_file is None:
raise SkilletNotFoundException(
"Could not find skillet definition file at this location")
snippet_path = str(meta_cnc_file.parent.absolute())
skillet_file = str(meta_cnc_file.name)
try:
with meta_cnc_file.open(mode="r", encoding="utf-8") as sc:
raw_service_config = oyaml.safe_load(sc.read())
skillet = self.normalize_skillet_dict(raw_service_config)
skillet["snippet_path"] = snippet_path
skillet["skillet_path"] = snippet_path
skillet["skillet_filename"] = skillet_file
return skillet
except IOError:
logger.error("Could not open metadata file in dir %s" %
meta_cnc_file.parent)
raise SkilletLoaderException(
"IOError: Could not parse metadata file in dir %s" %
meta_cnc_file.parent)
except YAMLError as ye:
logger.error(ye)
raise SkilletLoaderException(
"YAMLError: Could not parse metadata file in dir %s" %
meta_cnc_file.parent)
except Exception as ex:
logger.error(ex)
raise SkilletLoaderException(
"Exception: Could not parse metadata file in dir %s" %
meta_cnc_file.parent)
def __handle_xml_outputs(self, output_definition: dict,
results: str) -> dict:
"""
Parse the results string as an XML document
Example .meta-cnc snippets section:
snippets:
- name: system_info
path: /api/?type=op&cmd=<show><system><info></info></system></show>&key={{ api_key }}
output_type: xml
outputs:
- name: hostname
capture_value: result/system/hostname
- name: uptime
capture_value: result/system/uptime
- name: sw_version
capture_value: result/system/sw-version
:param results: string as returned from some action, to be parsed as XML document
:return: dict containing all outputs found from the capture pattern in each output
"""
captured_output = dict()
def unique_tag_list(elements: list) -> bool:
tag_list = list()
for el in elements:
# some xpath queries can return a list of str
if isinstance(el, str):
return False
if el.tag not in tag_list:
tag_list.append(el.tag)
if len(tag_list) == 1:
# all tags in this list are the same
return False
else:
# there are unique tags in this list
return True
try:
xml_doc = etree.XML(results)
# xml_doc = elementTree.fromstring(results)
# allow jinja syntax in capture_pattern, capture_value, capture_object etc
local_context = self.context.copy()
output = self.__render_output_metadata(output_definition,
local_context)
var_name = output['name']
if 'capture_pattern' in output or 'capture_value' in output:
if 'capture_value' in output:
capture_pattern = output['capture_value']
else:
capture_pattern = output['capture_pattern']
# by default we will attempt to return the text of the found element
return_type = 'text'
entries = xml_doc.xpath(capture_pattern)
logger.debug(f'found entries: {entries}')
if len(entries) == 0:
captured_output[var_name] = ''
elif len(entries) == 1:
entry = entries.pop()
if isinstance(entry, str):
captured_output[var_name] = str(entry)
else:
if len(entry) == 0:
# this tag has no children, so try to grab the text
if return_type == 'text':
captured_output[var_name] = str(
entry.text).strip()
else:
captured_output[var_name] = entry.tag
else:
# we have 1 Element returned, so the user has a fairly specific xpath
# however, this element has children itself, so we can't return a text value
# just return the tag name of this element only
captured_output[var_name] = entry.tag
else:
# we have a list of elements returned from the users xpath query
capture_list = list()
# are there unique tags in this list? or is this a list of the same tag names?
if unique_tag_list(entries):
return_type = 'tag'
for entry in entries:
if isinstance(entry, str):
capture_list.append(entry)
else:
if len(entry) == 0:
if return_type == 'text':
if entry.text is not None:
capture_list.append(entry.text.strip())
else:
# If there is no text, then try to grab a sensible attribute
# if you need more control than this, then you should first
# capture_object to convert to a python object then use a jinja filter
# to get what you need
if 'value' in entry.attrib:
capture_list.append(
entry.attrib.get('value', ''))
elif 'name' in entry.attrib:
capture_list.append(
entry.attrib.get('name', ''))
else:
capture_list.append(
json.dumps(dict(entry.attrib)))
else:
capture_list.append(entry.tag)
else:
capture_list.append(entry.tag)
captured_output[var_name] = capture_list
elif 'capture_object' in output:
capture_pattern = output['capture_object']
entries = xml_doc.xpath(capture_pattern)
if len(entries) == 0:
captured_output[var_name] = None
elif len(entries) == 1:
captured_output[var_name] = xmltodict.parse(
elementTree.tostring(entries.pop()))
else:
capture_list = list()
for entry in entries:
capture_list.append(
xmltodict.parse(elementTree.tostring(entry)))
captured_output[var_name] = capture_list
elif 'capture_list' in output:
capture_pattern = output['capture_list']
entries = xml_doc.xpath(capture_pattern)
capture_list = list()
for entry in entries:
if isinstance(entry, str):
capture_list.append(entry)
else:
capture_list.append(
xmltodict.parse(elementTree.tostring(entry)))
captured_output[var_name] = capture_list
# filter selected items here
captured_output[var_name] = self.__filter_outputs(
output, captured_output[var_name], local_context)
except ParseError:
logger.error('Could not parse XML document in output_utils')
# just return blank captured_outputs here
raise SkilletLoaderException(
f'Could not parse output as XML in {self.name}')
return captured_output
