def _parse_token(self, token):
"""Concrete implementation of parent abstract method.
:Parameters:
according to parent :py:meth:`cumin.backends.BaseQueryAggregator._parse_token`.
"""
if not isinstance(token, pp.ParseResults): # pragma: no cover - this should never happen
raise InvalidQueryError('Expecting ParseResults object, got {type}: {token}'.format(
type=type(token), token=token))
token_dict = token.asDict()
self.logger.trace('Token is: %s | %s', token_dict, token)
if 'hosts' in token_dict:
element = self._get_stack_element()
element['hosts'] = nodeset_fromlist(token_dict['hosts'])
if 'bool' in token_dict:
element['bool'] = token_dict['bool']
self.stack_pointer['children'].append(element)
elif 'open_subgroup' in token_dict and 'close_subgroup' in token_dict:
self._open_subgroup()
if 'bool' in token_dict:
self.stack_pointer['bool'] = token_dict['bool']
for subtoken in token:
if isinstance(subtoken, str): # Grammar literals, boolean operators and parentheses
continue
self._parse_token(subtoken)
self._close_subgroup()
else: # pragma: no cover - this should never happen
raise InvalidQueryError('Got unexpected token: {token}'.format(token=token))
def _parse_token(self, token):
"""Concrete implementation of parent abstract method.
:Parameters:
according to parent :py:meth:`cumin.backends.BaseQuery._parse_token`.
Raises:
cumin.backends.InvalidQueryError: on internal parsing error.
"""
if not isinstance(token, pp.ParseResults
): # pragma: no cover - this should never happen
raise InvalidQueryError(
'Expecting ParseResults object, got {type}: {token}'.format(
type=type(token), token=token))
token_dict = token.asDict()
self.logger.trace('Token is: %s | %s', token_dict, token)
if 'key' in token_dict and 'value' in token_dict:
if token_dict['key'] == 'project':
self.search_project = token_dict['value']
else:
self.search_params[token_dict['key']] = token_dict['value']
elif 'all' in token_dict:
pass # nothing to do, search_project and search_params have the right defaults
else: # pragma: no cover - this should never happen
raise InvalidQueryError(
'Got unexpected token: {token}'.format(token=token))
def endpoint(self, value):
"""Setter for the `endpoint` property. The relative documentation is in the getter."""
if value not in self.endpoints.values():
raise InvalidQueryError(
"Invalid value '{endpoint}' for endpoint property".format(
endpoint=value))
if self._endpoint is not None and value != self._endpoint:
raise InvalidQueryError(
'Mixed endpoints are not supported, use the global grammar to mix them.'
)
self._endpoint = value
def _get_resource_query(self, key, value=None, operator='='): # pylint: disable=no-self-use
"""Build a resource query based on the parameters, resolving the special cases for ``%params`` and ``@field``.
Arguments:
key (str): the key of the resource.
value (str, optional): the value to match, if not specified the key itself will be matched.
operator (str, optional): the comparison operator to use, one of :py:const:`OPERATORS`.
Returns:
str: the resource query.
Raises:
cumin.backends.InvalidQueryError: on invalid combinations of parameters.
"""
if all(char in key for char in ('%', '@')):
raise InvalidQueryError((
"Resource key cannot contain both '%' (query a resource's parameter) and '@' "
"(query a resource's field)"))
if '%' in key:
# Querying a specific parameter of the resource
if operator == '~' and self.api_version == 3:
raise InvalidQueryError(
'Regex operations are not supported in PuppetDB API v3 for resource parameters'
)
key, param = key.split('%', 1)
query_part = ', ["{op}", ["parameter", "{param}"], {value}]'.format(
op=operator, param=param, value=value)
elif '@' in key:
# Querying a specific field of the resource
key, field = key.split('@', 1)
query_part = ', ["{op}", "{field}", {value}]'.format(op=operator,
field=field,
value=value)
elif value is None:
# Querying a specific resource type
query_part = ''
else:
# Querying a specific resource title
if key.lower() == 'class' and operator != '~':
value = value.capwords('::') # Auto ucfirst the class title
query_part = ', ["{op}", "title", {value}]'.format(op=operator,
value=value)
query = '["and", ["=", "type", "{type}"]{query_part}]'.format(
type=capwords(key, '::'), query_part=query_part)
return query
def _get_special_resource_query(self, category, key, value, operator):
"""Build a query for Roles and Profiles, resolving the special cases for ``%params`` and ``@field``.
Arguments:
category (str): the category of the token, one of :py:data:`category_prefixes` keys.
key (str): the key of the resource to use as a suffix for the Class title matching.
value (str, optional): the value to match in case ``%params`` or ``@field`` is specified.
operator (str, optional): the comparison operator to use if there is a value, one of :py:const:`OPERATORS`.
Returns:
str: the resource query.
Raises:
cumin.backends.InvalidQueryError: on invalid combinations of parameters.
"""
if all(char in key for char in ('%', '@')):
raise InvalidQueryError((
"Resource key cannot contain both '%' (query a resource's parameter) and '@' "
"(query a resource's field)"))
if '%' in key:
special = '%'
key, param = key.split('%')
elif '@' in key:
special = '@'
key, param = key.split('@')
else:
special = None
if value is not None:
raise InvalidQueryError((
"Invalid query of the form '{category}:key = value'. The matching of a value "
"is accepted only when using %param or @field.").format(
category=category))
if self.category_prefixes[category]:
title = ParsedString(
'{prefix}::{key}'.format(
prefix=self.category_prefixes[category], key=key), True)
else:
title = ParsedString(key, True)
query = self._get_resource_query('Class', title, '=')
if special is not None:
param_query = self._get_resource_query(
''.join(('Class', special, param)), value, operator)
query = '["and", {query}, {param_query}]'.format(
query=query, param_query=param_query)
return query
def __init__(self, config):
"""Query constructor for the PuppetDB backend.
:Parameters:
according to parent :py:meth:`cumin.backends.BaseQuery.__init__`.
"""
super().__init__(config)
self.grouped_tokens = None
self.current_group = self.grouped_tokens
self._endpoint = None
puppetdb_config = self.config.get('puppetdb', {})
base_url = self.base_url_template.format(
scheme=puppetdb_config.get('scheme', 'https'),
host=puppetdb_config.get('host', 'localhost'),
port=puppetdb_config.get('port', 443))
self.api_version = puppetdb_config.get('api_version', 4)
if self.api_version == 3:
self.url = base_url + '/v3/'
self.hosts_keys = {'nodes': 'name', 'resources': 'certname'}
elif self.api_version == 4:
self.url = base_url + '/pdb/query/v4/'
self.hosts_keys = {'nodes': 'certname', 'resources': 'certname'}
else:
raise InvalidQueryError(
'Unsupported PuppetDB API version {ver}'.format(
ver=self.api_version))
for exception in puppetdb_config.get('urllib3_disable_warnings', []):
urllib3.disable_warnings(
category=getattr(urllib3.exceptions, exception))
def _query_default_backend(self, query_string):
"""Execute the query with the default backend, according to the configuration.
Arguments:
query_string (str): the query string to be parsed and executed with the default backend.
Returns:
ClusterShell.NodeSet.NodeSet: with the FQDNs of the matching hosts.
Raises:
cumin.backends.InvalidQueryError: if unable to get the default backend from the registered backends.
"""
for registered_backend in self.registered_backends.values():
if registered_backend.name == self.config['default_backend']:
backend = registered_backend
break
else:
raise InvalidQueryError(
"Default backend '{name}' is not registered: {backends}".
format(name=self.config['default_backend'],
backends=self.registered_backends))
query = backend.cls(self.config)
return query.execute(query_string)
def _replace_alias(self, token_dict):
"""Replace any alias in the query in a recursive way, alias can reference other aliases.
Arguments:
token_dict (dict): the dictionary of the parsed token returned by the grammar parsing.
Returns:
bool: :py:data:`True` if a replacement was made, :py:data`False` otherwise.
Raises:
cumin.backends.InvalidQueryError: if unable to replace an alias.
"""
if 'alias' not in token_dict:
return False
alias_name = token_dict['alias']
if alias_name not in self.config.get('aliases', {}):
raise InvalidQueryError(
"Unable to find alias replacement for '{alias}' in the configuration"
.format(alias=alias_name))
self._open_subgroup()
if 'bool' in token_dict:
self.stack_pointer['bool'] = token_dict['bool']
# Calling BaseQuery._build() directly and not the parent's one to avoid resetting the stack
BaseQuery._build(self, self.config['aliases'][alias_name]) # pylint: disable=protected-access
self._close_subgroup()
return True
def _parse_token(self, token):
"""Concrete implementation of parent abstract method.
:Parameters:
according to parent :py:meth:`cumin.backends.BaseQueryAggregator._parse_token`.
Raises:
cumin.backends.InvalidQueryError: on internal parsing error.
"""
if not isinstance(
token,
ParseResults): # pragma: no cover - this should never happen
raise InvalidQueryError(
'Expecting ParseResults object, got {type}: {token}'.format(
type=type(token), token=token))
token_dict = token.asDict()
self.logger.trace('Token is: %s', token_dict)
if self._replace_alias(token_dict):
return # This token was an alias and got replaced
if 'backend' in token_dict and 'query' in token_dict:
element = self._get_stack_element()
query = self.registered_backends[token_dict['backend']].cls(
self.config)
element['hosts'] = query.execute(token_dict['query'])
if 'bool' in token_dict:
element['bool'] = token_dict['bool']
self.stack_pointer['children'].append(element)
elif 'open_subgroup' in token_dict and 'close_subgroup' in token_dict:
self._open_subgroup()
if 'bool' in token_dict:
self.stack_pointer['bool'] = token_dict['bool']
for subtoken in token:
if isinstance(subtoken, str):
continue
self._parse_token(subtoken)
self._close_subgroup()
else: # pragma: no cover - this should never happen
raise InvalidQueryError(
'Got unexpected token: {token}'.format(token=token))
def execute(self, query_string):
"""Override parent class execute method to implement the multi-query capability.
:Parameters:
according to parent :py:meth:`cumin.backends.BaseQueryAggregator.execute`.
Returns:
ClusterShell.NodeSet.NodeSet: with the FQDNs of the matching hosts.
Raises:
cumin.backends.InvalidQueryError: if unable to parse the query.
"""
if 'default_backend' not in self.config:
try: # No default backend set, using directly the global grammar
return super().execute(query_string)
except ParseException as e:
raise InvalidQueryError((
"Unable to parse the query '{query}' with the global grammar and no "
"default backend is set:\n{error}").format(
query=query_string, error=e))
try: # Default backend set, trying it first
hosts = self._query_default_backend(query_string)
except ParseException as e_default:
try: # Trying global grammar as a fallback
hosts = super().execute(query_string)
except ParseException as e_global:
raise InvalidQueryError((
"Unable to parse the query '{query}' neither with the default backend '{name}' nor with the "
"global grammar:\n{name}: {e_def}\nglobal: {e_glob}"
).format(query=query_string,
name=self.config['default_backend'],
e_def=e_default,
e_glob=e_global))
return hosts
def _parse_token(self, token):
"""Concrete implementation of parent abstract method.
:Parameters:
according to parent :py:meth:`cumin.backends.BaseQuery._parse_token`.
"""
if isinstance(token, str):
return
token_dict = token.asDict()
try:
with open(token_dict['filename'], 'r') as f:
self.hosts = NodeSet.fromlist(f.readlines(),
resolver=RESOLVER_NOGROUP)
except Exception as e:
raise InvalidQueryError(e)
def _add_bool(self, bool_op):
"""Add a boolean AND or OR query block to the query and validate logic.
Arguments:
bool_op (str): the boolean operator to add to the query: ``and``, ``or``.
Raises:
cumin.backends.InvalidQueryError: if an invalid boolean operator was found.
"""
if self.current_group['bool'] is None:
self.current_group['bool'] = bool_op
elif self.current_group['bool'] == bool_op:
return
else:
raise InvalidQueryError(
"Got unexpected '{bool}' boolean operator, current operator was '{current}'"
.format(bool=bool_op, current=self.current_group['bool']))
def _parse_token(self, token):
"""Concrete implementation of parent abstract method.
:Parameters:
according to parent :py:meth:`cumin.backends.BaseQuery._parse_token`.
Raises:
cumin.backends.InvalidQueryError: on internal parsing error.
"""
if isinstance(token, str):
return
token_dict = token.asDict()
# post-process types
if 'quoted' in token_dict:
token_dict['value'] = ParsedString(token_dict['quoted'], True)
del token_dict['quoted']
elif 'value' in token_dict:
token_dict['value'] = ParsedString(token_dict['value'], False)
# Based on the token type build the corresponding query object
if 'open_subgroup' in token_dict:
self._open_subgroup()
for subtoken in token:
self._parse_token(subtoken)
self._close_subgroup()
elif 'bool' in token_dict:
self._add_bool(token_dict['bool'])
elif 'hosts' in token_dict:
token_dict['hosts'] = nodeset(token_dict['hosts'])
self._add_hosts(**token_dict)
elif 'category' in token_dict:
self._add_category(**token_dict)
else: # pragma: no cover - this should never happen
raise InvalidQueryError(
"No valid key found in token, one of bool|hosts|category expected: {token}"
.format(token=token_dict))
def _add_category(self,
category,
key,
value=None,
operator='=',
neg=False):
"""Add a category token to the query 'F:key = value'.
Arguments:
category (str): the category of the token, one of :py:const:`CATEGORIES`.
key (str): the key for this category.
value (str, optional): the value to match, if not specified the key itself will be matched.
operator (str, optional): the comparison operator to use, one of :py:const:`OPERATORS`.
neg (bool, optional): whether the token must be negated.
Raises:
cumin.backends.InvalidQueryError: on internal parsing error.
"""
self.endpoint = self.endpoints[category]
if operator == '~':
# PuppetDB API requires to escape every backslash
# See: https://puppet.com/docs/puppetdb/4.4/api/query/v4/ast.html#regexp-match
value = value.replace('\\', '\\\\')
if category in ('C', 'O', 'P'):
query = self._get_special_resource_query(category, key, value,
operator)
elif category == 'R':
query = self._get_resource_query(key, value, operator)
elif category == 'F':
query = '["{op}", ["fact", "{key}"], {val}]'.format(op=operator,
key=key,
val=value)
else: # pragma: no cover - this should never happen
raise InvalidQueryError(
"Got invalid category '{category}', one of F|O|P|R expected".
format(category=category))
if neg:
query = '["not", {query}]'.format(query=query)
self.current_group['tokens'].append(query)