def inventory_ajax(env):
"""Backend endpoint for inventory table"""
draw = int(request.args.get('draw', 0))
envs = environments()
check_env(env, envs)
headers, fact_names = inventory_facts()
query = AndOperator()
fact_query = OrOperator()
fact_query.add([EqualsOperator("name", name) for name in fact_names])
query.add(fact_query)
if env != '*':
query.add(EqualsOperator("environment", env))
facts = puppetdb.facts(query=query)
fact_data = {}
for fact in facts:
if fact.node not in fact_data:
fact_data[fact.node] = {}
fact_data[fact.node][fact.name] = fact.value
total = len(fact_data)
return render_template('inventory.json.tpl',
draw=draw,
total=total,
total_filtered=total,
fact_data=fact_data,
columns=fact_names)
def test_fromoperator(self):
op = InOperator('certname')
ex = ExtractOperator()
ex.add_field(["certname", "facts"])
fr = FromOperator("facts")
fr.add_query(ex)
fr.add_offset(10)
op.add_query(fr)
assert repr(op) == 'Query: ["in", "certname", ' \
'["from", "facts", ["extract", ' \
'["certname", "facts"]], ["offset", 10]]]'
# last example on page
# https://puppet.com/docs/puppetdb/5.1/api/query/v4/ast.html
op = InOperator('certname')
ex = ExtractOperator()
ex.add_field('certname')
fr = FromOperator('fact_contents')
nd = AndOperator()
nd.add(
EqualsOperator("path", ["networking", "eth0", "macaddresses", 0]))
nd.add(EqualsOperator("value", "aa:bb:cc:dd:ee:00"))
ex.add_query(nd)
fr.add_query(ex)
op.add_query(fr)
assert str(op) == '["in", "certname", ' \
'["from", "fact_contents", ' \
'["extract", ["certname"], ["and", ["=", "path", ' \
'["networking", "eth0", "macaddresses", 0]], ' \
'["=", "value", "aa:bb:cc:dd:ee:00"]]]]]'
def nodes(env):
"""Fetch all (active) nodes from PuppetDB and stream a table displaying
those nodes.
Downside of the streaming aproach is that since we've already sent our
headers we can't abort the request if we detect an error. Because of this
we'll end up with an empty table instead because of how yield_or_stop
works. Once pagination is in place we can change this but we'll need to
provide a search feature instead.
:param env: Search for nodes in this (Catalog and Fact) environment
:type env: :obj:`string`
"""
envs = environments()
status_arg = request.args.get('status', '')
check_env(env, envs)
query = AndOperator()
if env != '*':
query.add(EqualsOperator("catalog_environment", env))
if status_arg in ['failed', 'changed', 'unchanged']:
query.add(EqualsOperator('latest_report_status', status_arg))
elif status_arg == 'unreported':
unreported = datetime.utcnow()
unreported = (unreported -
timedelta(hours=app.config['UNRESPONSIVE_HOURS']))
unreported = unreported.replace(microsecond=0).isoformat()
unrep_query = OrOperator()
unrep_query.add(NullOperator('report_timestamp', True))
unrep_query.add(LessEqualOperator('report_timestamp', unreported))
query.add(unrep_query)
if len(query.operations) == 0:
query = None
nodelist = puppetdb.nodes(
query=query,
unreported=app.config['UNRESPONSIVE_HOURS'],
with_status=True,
with_event_numbers=app.config['WITH_EVENT_NUMBERS'])
nodes = []
for node in yield_or_stop(nodelist):
if status_arg:
if node.status == status_arg:
nodes.append(node)
else:
nodes.append(node)
return Response(
stream_with_context(
stream_template('nodes.html',
nodes=nodes,
envs=envs,
current_env=env)))
def test_not_with_list(self):
op = NotOperator()
with pytest.raises(APIError):
str(
op.add([
EqualsOperator('clientversion', '4.5.1'),
EqualsOperator('facterversion', '3.2.0')
]))
def _puppetdb_nodes(rules, master_id, master_address):
if "facts" not in rules or not rules["facts"]:
return set()
if rules["match_type"] == "ALL":
operator = AndOperator()
else:
operator = OrOperator()
for fact_rule in rules["facts"]:
rule_op = fact_rule["operator"]
lhs = ["fact", fact_rule["fact"]]
rhs = fact_rule["value"]
if rule_op == "=":
operator.add(EqualsOperator(lhs, rhs))
elif rule_op == "!=":
notop = NotOperator()
notop.add(EqualsOperator(lhs, rhs))
operator.add(notop)
elif rule_op == "~":
operator.add(RegexOperator(lhs, rhs))
elif rule_op == "!~":
notop = NotOperator()
notop.add(RegexOperator(lhs, rhs))
operator.add(notop)
elif rule_op == ">":
operator.add(GreaterOperator(lhs, rhs))
elif rule_op == ">=":
operator.add(GreaterEqualOperator(lhs, rhs))
elif rule_op == "<":
operator.add(LessOperator(lhs, rhs))
elif rule_op == "<=":
operator.add(LessEqualOperator(lhs, rhs))
query_str = str(operator).replace("'", '"')
cert_instance = CertsClass(master_id)
cert_path = cert_instance.get_cert()
private_key_path = cert_instance.get_key()
req = requests.get(
f"https://{master_address}:8081/pdb/meta/v1/version",
verify=False,
cert=(cert_path, private_key_path),
)
if req.status_code == 404:
# PuppetDB <= 2.x
nodes_uri = f"https://{master_address}:8081/v4/nodes?query={query_str}"
else:
# PuppetDB >= 3.x
nodes_uri = (
f"https://{master_address}:8081/pdb/query/v4/nodes?query={query_str}"
)
# Fetch the matching nodes
req = requests.get(nodes_uri,
verify=False,
cert=(cert_path, private_key_path))
return {node["certname"] for node in req.json()}
def facts(self, query=None, **kwargs):
"""Get all facts of this node. Additional arguments may also be
specified that will be passed to the query function.
"""
q = EqualsOperator("certname", self.name)
if query:
q = AndOperator()
q.add(EqualsOperator("certname", self.name))
q.add(query)
return self.__api.facts(query=q, **kwargs)
def get_nodes(environment: str, configuration: Configuration) -> List[Node]:
database = _get_puppetdb_connexion(configuration)
result = []
query = AndOperator()
query.add(EqualsOperator('catalog_environment', environment))
query.add(EqualsOperator('facts_environment', environment))
nodes = database.nodes(with_status=True, query=query)
for node in nodes:
try:
result.append(Node(node.name, Status(node.status)))
except ValueError:
continue
return result
def test_not_operator(self):
op = NotOperator()
op.add(EqualsOperator("operatingsystem", "CentOS"))
assert str(op) == '["not", ["=", "operatingsystem", "CentOS"]]'
assert repr(op) == 'Query: ["not", ["=", "operatingsystem", "CentOS"]]'
assert str(op) == '["not", ["=", "operatingsystem", "CentOS"]]'
with pytest.raises(APIError):
op.add(GreaterOperator("operatingsystemmajrelease", 6))
with pytest.raises(APIError):
op.add([EqualsOperator("architecture", "x86_64"),
GreaterOperator("operatingsystemmajrelease", 6)])
def _build_query(env, start, end, certname=None):
"""Build a extract query with optional certname and environment."""
query = ExtractOperator()
query.add_field(FunctionOperator('count'))
query.add_field('status')
subquery = AndOperator()
subquery.add(GreaterEqualOperator('producer_timestamp', start))
subquery.add(LessOperator('producer_timestamp', end))
if certname is not None:
subquery.add(EqualsOperator('certname', certname))
if env != '*':
subquery.add(EqualsOperator('environment', env))
query.add_query(subquery)
query.add_group_by("status")
return query
def test_order_by(self):
fr = FromOperator("facts")
op = EqualsOperator("certname", "test01")
fr.add_query(op)
o1 = ["certname"]
o2 = ["certname", ["timestamp", "desc"], "facts"]
o3inv = ['certname', ['timestamp', 'desc', ['oops']]]
fr.add_order_by(o1)
assert str(fr) == \
'["from", "facts", ["=", "certname", ' \
'"test01"], ["order_by", ["certname"]]]'
fr.add_order_by(o2)
assert repr(fr) == \
'Query: ["from", "facts", ' \
'["=", "certname", "test01"], ' \
'["order_by", ["certname", ' \
'["timestamp", "desc"], "facts"]]]'
assert str(fr) == \
'["from", "facts", ' \
'["=", "certname", "test01"], ' \
'["order_by", ["certname", ' \
'["timestamp", "desc"], "facts"]]]'
assert str(fr) == \
'["from", "facts", ' \
'["=", "certname", "test01"], ' \
'["order_by", ["certname", ' \
'["timestamp", "desc"], "facts"]]]'
with pytest.raises(APIError):
fr.add_order_by(o3inv)
def test_limit_offset(self):
fr = FromOperator("facts")
op = EqualsOperator("certname", "test01")
fr.add_query(op)
fr.add_offset(10)
assert str(fr) == \
'["from", "facts", ["=", "certname", "test01"], ["offset", 10]]'
fr.add_limit(5)
assert str(fr) == \
'["from", "facts", ["=", "certname",' \
' "test01"], ["limit", 5], ["offset", 10]]'
fr.add_limit(15)
assert str(fr) == \
'["from", "facts", ["=", "certname",' \
' "test01"], ["limit", 15], ["offset", 10]]'
assert repr(fr) == \
'Query: ["from", "facts", ["=", "certname",' \
' "test01"], ["limit", 15], ["offset", 10]]'
with pytest.raises(APIError):
fr.add_offset("invalid")
with pytest.raises(APIError):
fr.add_limit(["invalid"])
def report(env, node_name, report_id):
"""Displays a single report including all the events associated with that
report and their status.
The report_id may be the puppetdb's report hash or the
configuration_version. This allows for better integration
into puppet-hipchat.
:param env: Search for reports in this environment
:type env: :obj:`string`
:param node_name: Find the reports whose certname match this value
:type node_name: :obj:`string`
:param report_id: The hash or the configuration_version of the desired
report
:type report_id: :obj:`string`
"""
envs = environments()
check_env(env, envs)
query = AndOperator()
report_id_query = OrOperator()
report_id_query.add(EqualsOperator("hash", report_id))
report_id_query.add(EqualsOperator("configuration_version", report_id))
if env != '*':
query.add(EqualsOperator("environment", env))
query.add(EqualsOperator("certname", node_name))
query.add(report_id_query)
reports = puppetdb.reports(query=query)
try:
report = next(reports)
except StopIteration:
abort(404)
report.version = commonmark.commonmark(report.version)
return render_template('report.html',
report=report,
events=yield_or_stop(report.events()),
logs=report.logs,
metrics=report.metrics,
envs=envs,
current_env=env)
def test_events_endpoint(self):
assert str(SubqueryOperator('events')) == \
'["select_events"]'
op = SubqueryOperator('events')
op.add_query(EqualsOperator('status', 'noop'))
assert repr(op) == 'Query: ["select_events", '\
'["=", "status", "noop"]]'
def test_or_operator(self):
op = OrOperator()
op.add(EqualsOperator("operatingsystem", "CentOS"))
op.add([EqualsOperator("architecture", "x86_64"),
GreaterOperator("operatingsystemmajrelease", 6)])
assert str(op) == '["or", ["=", "operatingsystem", "CentOS"], '\
'["=", "architecture", "x86_64"], '\
'[">", "operatingsystemmajrelease", 6]]'
assert repr(op) == 'Query: ["or", '\
'["=", "operatingsystem", "CentOS"], '\
'["=", "architecture", "x86_64"], '\
'[">", "operatingsystemmajrelease", 6]]'
assert str(op) == '["or", ["=", "operatingsystem", "CentOS"], ' \
'["=", "architecture", "x86_64"], '\
'[">", "operatingsystemmajrelease", 6]]'
with pytest.raises(APIError):
op.add({"query1": '["=", "catalog_environment", "production"]'})
def resource(self, type_, title, **kwargs):
"""Get a resource matching the supplied type and title. Additional
arguments may also be specified that will be passed to the query
function.
"""
resources = self.__api.resources(type_=type_,
title=title,
query=EqualsOperator(
"certname", self.name),
**kwargs)
return next(resource for resource in resources)
def resources(self, type_=None, title=None, **kwargs):
"""Get all resources of this node or all resources of the specified
type. Additional arguments may also be specified that will be passed
to the query function.
"""
if type_ is None:
resources = self.__api.resources(query=EqualsOperator(
"certname", self.name),
**kwargs)
elif type_ is not None and title is None:
resources = self.__api.resources(type_=type_,
query=EqualsOperator(
"certname", self.name),
**kwargs)
else:
resources = self.__api.resources(type_=type_,
title=title,
query=EqualsOperator(
"certname", self.name),
**kwargs)
return resources
def nodes(self,
unreported=2,
with_status=False,
with_event_numbers=True,
**kwargs):
"""Query for nodes by either name or query. If both aren't
provided this will return a list of all nodes. This method
also (optionally) fetches the nodes status and (optionally)
event counts of the latest report from puppetdb.
:param with_status: (optional) include the node status in the\
returned nodes
:type with_status: :bool:
:param unreported: (optional) amount of hours when a node gets
marked as unreported
:type unreported: :obj:`None` or integer
:param with_event_numbers: (optional) include the exact number of\
changed/unchanged/failed/noop events when\
with_status is set to True. If set to False
only "some" string is provided if there are
resources with such status in the last report.
This provides performance benefits as potentially
slow event-counts query is omitted completely.
:type with_event_numbers: :bool:
:param \*\*kwargs: The rest of the keyword arguments are passed
to the _query function
:returns: A generator yieling Nodes.
:rtype: :class:`pypuppetdb.types.Node`
"""
nodes = self._query('nodes', **kwargs)
now = datetime.utcnow()
# If we happen to only get one node back it
# won't be inside a list so iterating over it
# goes boom. Therefor we wrap a list around it.
if type(nodes) == dict:
nodes = [
nodes,
]
latest_events = None
if with_status and with_event_numbers:
latest_events = self._query(
'event-counts',
query=EqualsOperator("latest_report?", True),
summarize_by='certname',
)
for node in nodes:
yield Node.create_from_dict(self, node, with_status,
with_event_numbers, latest_events, now,
unreported)
def catalogs_ajax(env, compare):
"""Server data to catalogs as JSON to Jquery datatables
"""
draw = int(request.args.get('draw', 0))
start = int(request.args.get('start', 0))
length = int(request.args.get('length', app.config['NORMAL_TABLE_COUNT']))
paging_args = {'limit': length, 'offset': start}
search_arg = request.args.get('search[value]')
order_column = int(request.args.get('order[0][column]', 0))
order_filter = CATALOGS_COLUMNS[order_column].get(
'filter', CATALOGS_COLUMNS[order_column]['attr'])
order_dir = request.args.get('order[0][dir]', 'asc')
order_args = '[{"field": "%s", "order": "%s"}]' % (order_filter, order_dir)
envs = environments()
check_env(env, envs)
query = AndOperator()
if env != '*':
query.add(EqualsOperator("catalog_environment", env))
if search_arg:
query.add(RegexOperator("certname", r"%s" % search_arg))
query.add(NullOperator("catalog_timestamp", False))
nodes = get_or_abort(puppetdb.nodes,
query=query,
include_total=True,
order_by=order_args,
**paging_args)
catalog_list = []
total = None
for node in nodes:
if total is None:
total = puppetdb.total
catalog_list.append({
'certname': node.name,
'catalog_timestamp': node.catalog_timestamp,
'form': compare,
})
if total is None:
total = 0
return render_template('catalogs.json.tpl',
total=total,
total_filtered=total,
draw=draw,
columns=CATALOGS_COLUMNS,
catalogs=catalog_list,
envs=envs,
current_env=env)
def node(env, node_name):
"""Display a dashboard for a node showing as much data as we have on that
node. This includes facts and reports but not Resources as that is too
heavy to do within a single request.
:param env: Ensure that the node, facts and reports are in this environment
:type env: :obj:`string`
"""
envs = environments()
check_env(env, envs)
query = AndOperator()
if env != '*':
query.add(EqualsOperator("environment", env))
query.add(EqualsOperator("certname", node_name))
node = get_or_abort(puppetdb.node, node_name)
return render_template('node.html',
node=node,
envs=envs,
current_env=env,
columns=REPORTS_COLUMNS[:2])
def test_equal_operator(self):
op = EqualsOperator("certname", "test01")
assert str(op) == '["=", "certname", "test01"]'
assert repr(op) == 'Query: ["=", "certname", "test01"]'
assert str(op) == '["=", "certname", "test01"]'
assert str(EqualsOperator("clientversion", 91))\
== '["=", "clientversion", 91]'
assert str(EqualsOperator("start_time", "2016-05-11T23:22:48.709Z"))\
== '["=", "start_time", "2016-05-11T23:22:48.709Z"]'
assert str(EqualsOperator("is_virtual", True))\
== '["=", "is_virtual", true]'
assert str(EqualsOperator("bios_version", ["6.00", 5.00]))\
== '["=", "bios_version", ["6.00", 5.0]]'
assert str(EqualsOperator(['parameter', 'ensure'], "present"))\
== '["=", ["parameter", "ensure"], "present"]'
assert str(EqualsOperator(u"latest_report?", True))\
== '["=", "latest_report?", true]'
assert str(EqualsOperator("report_timestamp",
datetime.datetime(2016, 6, 11)))\
== '["=", "report_timestamp", "2016-06-11 00:00:00"]'
def test_with_add_group_by(self):
op = ExtractOperator()
op.add_field(['certname', 'fact_environment', 'catalog_environment'])
op.add_query(EqualsOperator('domain', 'example.com'))
op.add_group_by(["fact_environment", "catalog_environment"])
with pytest.raises(APIError):
op.add_group_by({"deactivated": False})
assert repr(op) == 'Query: ["extract", '\
'["certname", "fact_environment", "catalog_environment"], '\
'["=", "domain", "example.com"], '\
'["group_by", "fact_environment", "catalog_environment"]]'
assert str(op) == '["extract", '\
'["certname", "fact_environment", "catalog_environment"], '\
'["=", "domain", "example.com"], '\
'["group_by", "fact_environment", "catalog_environment"]]'
assert str(op) == '["extract", ' \
'["certname", "fact_environment", "catalog_environment"], '\
'["=", "domain", "example.com"], '\
'["group_by", "fact_environment", "catalog_environment"]]'
def test_with_add_query(self):
op = ExtractOperator()
op.add_field(['certname', 'fact_environment', 'catalog_environment'])
with pytest.raises(APIError):
op.add_query({'less': 42, 'greater': 50})
op.add_query(EqualsOperator('domain', 'example.com'))
assert repr(op) == 'Query: ["extract", '\
'["certname", "fact_environment", "catalog_environment"], '\
'["=", "domain", "example.com"]]'
assert str(op) == '["extract", '\
'["certname", "fact_environment", "catalog_environment"], '\
'["=", "domain", "example.com"]]'
assert str(op) == '["extract", ' \
'["certname", "fact_environment", "catalog_environment"], '\
'["=", "domain", "example.com"]]'
with pytest.raises(APIError):
op.add_query(GreaterOperator("processorcount", 1))
def test_add_query(self):
fr = FromOperator("facts")
op = EqualsOperator("certname", "test01")
fr.add_query(op)
assert str(fr) == '["from", "facts", ["=", "certname", "test01"]]'
fr2 = FromOperator("facts")
op2 = "test, test, test"
with pytest.raises(APIError):
fr2.add_query(op2)
fr2.add_query(op)
with pytest.raises(APIError):
fr2.add_query(op)
fr3 = FromOperator("facts")
op3 = ExtractOperator()
op3.add_field(['certname', 'fact_environment', 'catalog_environment'])
fr3.add_query(op3)
assert str(fr3) == \
'["from", "facts", ["extract", '\
'["certname", "fact_environment", "catalog_environment"]]]'
def report(env, node_name, report_id, show_error_as):
"""Displays a single report including all the events associated with that
report and their status.
The report_id may be the puppetdb's report hash or the
configuration_version. This allows for better integration
into puppet-hipchat.
:param env: Search for reports in this environment
:type env: :obj:`string`
:param node_name: Find the reports whose certname match this value
:type node_name: :obj:`string`
:param report_id: The hash or the configuration_version of the desired
report
:type report_id: :obj:`string`
:param show_error_as: 'friendly' or 'raw', the former means that messages
will be show in a mode transformed for human-readability, the latter
that the messages will be unchanged
:param show_error_as: :obj:`string`
"""
envs = environments()
check_env(env, envs)
query = AndOperator()
report_id_query = OrOperator()
report_id_query.add(EqualsOperator("hash", report_id))
report_id_query.add(EqualsOperator("configuration_version", report_id))
if env != '*':
query.add(EqualsOperator("environment", env))
query.add(EqualsOperator("certname", node_name))
query.add(report_id_query)
reports = puppetdb.reports(query=query)
try:
report = next(reports)
except StopIteration:
abort(404)
if show_error_as not in ['friendly', 'raw']:
abort(404)
report.version = commonmark.commonmark(report.version)
events = [{
'resource': f"{event.item['type']}[{event.item['title']}]",
'status': event.status,
'old': event.item['old'],
'new': event.item['new'],
'failed': event.failed,
} for event in report.events()]
logs = [
{
'timestamp': log['time'],
'level': log["level"],
'source': log['source'],
'tags': ', '.join(log['tags']),
'message': get_message(node_name, log, show_error_as),
'location': get_location(log),
# this could be also done with a different rendered in DataTables,
# - feel free to refactor it into that if you know how
'short_location': get_short_location(get_location(log)),
} for log in report.logs
]
return render_template('report.html',
report=report,
events=events,
logs=logs,
metrics=report.metrics,
envs=envs,
current_env=env,
current_show_error_as=show_error_as)
def reports_ajax(env, node_name):
"""Query and Return JSON data to reports Jquery datatable
:param env: Search for all reports in this environment
:type env: :obj:`string`
"""
draw = int(request.args.get('draw', 0))
start = int(request.args.get('start', 0))
length = int(request.args.get('length', app.config['NORMAL_TABLE_COUNT']))
paging_args = {'limit': length, 'offset': start}
search_arg = request.args.get('search[value]')
order_column = int(request.args.get('order[0][column]', 0))
order_filter = REPORTS_COLUMNS[order_column].get(
'filter', REPORTS_COLUMNS[order_column]['attr'])
order_dir = request.args.get('order[0][dir]', 'desc')
order_args = '[{"field": "%s", "order": "%s"}]' % (order_filter, order_dir)
status_args = request.args.get('columns[1][search][value]', '').split('|')
date_args = request.args.get('columns[0][search][value]', '')
max_col = len(REPORTS_COLUMNS)
for i in range(len(REPORTS_COLUMNS)):
if request.args.get("columns[%s][data]" % i, None):
max_col = i + 1
envs = environments()
check_env(env, envs)
reports_query = AndOperator()
if env != '*':
reports_query.add(EqualsOperator("environment", env))
if node_name:
reports_query.add(EqualsOperator("certname", node_name))
if search_arg:
search_query = OrOperator()
search_query.add(RegexOperator("certname", r"%s" % search_arg))
search_query.add(RegexOperator("puppet_version", r"%s" % search_arg))
search_query.add(
RegexOperator("configuration_version", r"%s" % search_arg))
reports_query.add(search_query)
if date_args:
dates = json.loads(date_args)
if len(dates) > 0:
date_query = AndOperator()
if 'min' in dates:
date_query.add(GreaterEqualOperator('end_time', dates['min']))
if 'max' in dates:
date_query.add(LessEqualOperator('end_time', dates['max']))
reports_query.add(date_query)
status_query = OrOperator()
for status_arg in status_args:
if status_arg in ['failed', 'changed', 'unchanged']:
arg_query = AndOperator()
arg_query.add(EqualsOperator('status', status_arg))
arg_query.add(EqualsOperator('noop', False))
status_query.add(arg_query)
if status_arg == 'unchanged':
arg_query = AndOperator()
arg_query.add(EqualsOperator('noop', True))
arg_query.add(EqualsOperator('noop_pending', False))
status_query.add(arg_query)
elif status_arg == 'noop':
arg_query = AndOperator()
arg_query.add(EqualsOperator('noop', True))
arg_query.add(EqualsOperator('noop_pending', True))
status_query.add(arg_query)
if len(status_query.operations) == 0:
if len(reports_query.operations) == 0:
reports_query = None
else:
reports_query.add(status_query)
if status_args[0] != 'none':
reports = get_or_abort(puppetdb.reports,
query=reports_query,
order_by=order_args,
include_total=True,
**paging_args)
reports, reports_events = tee(reports)
total = None
else:
reports = []
reports_events = []
total = 0
# Convert metrics to relational dict
metrics = {}
for report in reports_events:
if total is None:
total = puppetdb.total
metrics[report.hash_] = {}
for m in report.metrics:
if m['category'] not in metrics[report.hash_]:
metrics[report.hash_][m['category']] = {}
metrics[report.hash_][m['category']][m['name']] = m['value']
if total is None:
total = 0
return render_template('reports.json.tpl',
draw=draw,
total=total,
total_filtered=total,
reports=reports,
metrics=metrics,
envs=envs,
current_env=env,
columns=REPORTS_COLUMNS[:max_col])
def fact_ajax(env, node, fact, value):
"""Fetches the specific facts matching (node/fact/value) from PuppetDB and
return a JSON table
:param env: Searches for facts in this environment
:type env: :obj:`string`
:param node: Find all facts for this node
:type node: :obj:`string`
:param fact: Find all facts with this name
:type fact: :obj:`string`
:param value: Filter facts whose value is equal to this
:type value: :obj:`string`
"""
draw = int(request.args.get('draw', 0))
envs = environments()
check_env(env, envs)
render_graph = False
if fact in graph_facts and not value and not node:
render_graph = True
query = AndOperator()
if node:
query.add(EqualsOperator("certname", node))
if env != '*':
query.add(EqualsOperator("environment", env))
if len(query.operations) == 0:
query = None
# Generator needs to be converted (graph / total)
try:
value = int(value)
except ValueError:
if value is not None and query is not None:
if is_bool(value):
query.add(EqualsOperator('value', bool(strtobool(value))))
else:
query.add(EqualsOperator('value', unquote_plus(value)))
except TypeError:
pass
facts = [f for f in get_or_abort(puppetdb.facts, name=fact, query=query)]
total = len(facts)
counts = {}
json = {
'draw': draw,
'recordsTotal': total,
'recordsFiltered': total,
'data': []
}
for fact_h in facts:
line = []
if not fact:
line.append(fact_h.name)
if not node:
line.append('<a href="{0}">{1}</a>'.format(
url_for('node', env=env, node_name=fact_h.node), fact_h.node))
if not value:
fact_value = fact_h.value
if isinstance(fact_value, str):
fact_value = quote_plus(fact_h.value)
line.append('<a href="{0}">{1}</a>'.format(
url_for('fact', env=env, fact=fact_h.name, value=fact_value),
fact_h.value))
json['data'].append(line)
if render_graph:
if fact_h.value not in counts:
counts[fact_h.value] = 0
counts[fact_h.value] += 1
if render_graph:
json['chart'] = [{
"label": "{0}".format(k).replace('\n', ' '),
"value": counts[k]
} for k in sorted(counts, key=lambda k: counts[k], reverse=True)]
return jsonify(json)
def test_multiple_add_query(self):
with pytest.raises(APIError):
op = SubqueryOperator('events')
op.add_query(EqualsOperator('status', 'noop'))
op.add_query(EqualsOperator('status', 'changed'))
def nodes(self,
unreported=2,
with_status=False,
with_event_numbers=True,
**kwargs):
"""Query for nodes by either name or query. If both aren't
provided this will return a list of all nodes. This method
also (optionally) fetches the nodes status and (optionally)
event counts of the latest report from puppetdb.
:param with_status: (optional) include the node status in the\
returned nodes
:type with_status: :bool:
:param unreported: (optional) amount of hours when a node gets
marked as unreported
:type unreported: :obj:`None` or integer
:param with_event_numbers: (optional) include the exact number of\
changed/unchanged/failed/noop events when\
with_status is set to True. If set to False
only "some" string is provided if there are
resources with such status in the last report.
This provides performance benefits as potentially
slow event-counts query is omitted completely.
:type with_event_numbers: :bool:
:param \*\*kwargs: The rest of the keyword arguments are passed
to the _query function
:returns: A generator yieling Nodes.
:rtype: :class:`pypuppetdb.types.Node`
"""
nodes = self._query('nodes', **kwargs)
now = datetime.utcnow()
# If we happen to only get one node back it
# won't be inside a list so iterating over it
# goes boom. Therefor we wrap a list around it.
if type(nodes) == dict:
nodes = [
nodes,
]
if with_status and with_event_numbers:
latest_events = self.event_counts(query=EqualsOperator(
"latest_report?", True),
summarize_by='certname')
for node in nodes:
node['status_report'] = None
node['events'] = None
if with_status:
if with_event_numbers:
status = [
s for s in latest_events
if s['subject']['title'] == node['certname']
]
try:
node['status_report'] = node['latest_report_status']
if status:
node['events'] = status[0]
except KeyError:
if status:
node['events'] = status = status[0]
if status['successes'] > 0:
node['status_report'] = 'changed'
if status['noops'] > 0:
node['status_report'] = 'noop'
if status['failures'] > 0:
node['status_report'] = 'failed'
else:
node['status_report'] = 'unchanged'
else:
node['status_report'] = node['latest_report_status']
node['events'] = {
'successes': 0,
'failures': 0,
'noops': 0,
}
if node['status_report'] == 'changed':
node['events']['successes'] = 'some'
elif node['status_report'] == 'noop':
node['events']['noops'] = 'some'
elif node['status_report'] == 'failed':
node['events']['failures'] = 'some'
# node report age
if node['report_timestamp'] is not None:
try:
last_report = json_to_datetime(
node['report_timestamp'])
last_report = last_report.replace(tzinfo=None)
unreported_border = now - timedelta(hours=unreported)
if last_report < unreported_border:
delta = (now - last_report)
node['unreported'] = True
node['unreported_time'] = '{0}d {1}h {2}m'.format(
delta.days, int(delta.seconds / 3600),
int((delta.seconds % 3600) / 60))
except AttributeError:
node['unreported'] = True
if not node['report_timestamp']:
node['unreported'] = True
yield Node(self,
name=node['certname'],
deactivated=node['deactivated'],
expired=node['expired'],
report_timestamp=node['report_timestamp'],
catalog_timestamp=node['catalog_timestamp'],
facts_timestamp=node['facts_timestamp'],
status_report=node['status_report'],
noop=node.get('latest_report_noop'),
noop_pending=node.get('latest_report_noop_pending'),
events=node['events'],
unreported=node.get('unreported'),
unreported_time=node.get('unreported_time'),
report_environment=node['report_environment'],
catalog_environment=node['catalog_environment'],
facts_environment=node['facts_environment'],
latest_report_hash=node.get('latest_report_hash'),
cached_catalog_status=node.get('cached_catalog_status'))
def radiator(env):
"""This view generates a simplified monitoring page
akin to the radiator view in puppet dashboard
"""
envs = environments()
check_env(env, envs)
if env == '*':
db_version = get_db_version(puppetdb)
query_type, metric_version = metric_params(db_version)
query = None
metrics = get_or_abort(
puppetdb.metric,
'puppetlabs.puppetdb.population:%sname=num-nodes' % query_type,
version=metric_version)
num_nodes = metrics['Value']
else:
query = AndOperator()
metric_query = ExtractOperator()
query.add(EqualsOperator("catalog_environment", env))
metric_query.add_field(FunctionOperator('count'))
metric_query.add_query(query)
metrics = get_or_abort(puppetdb._query, 'nodes', query=metric_query)
num_nodes = metrics[0]['count']
nodes = puppetdb.nodes(query=query,
unreported=app.config['UNRESPONSIVE_HOURS'],
with_status=True)
stats = {
'changed_percent': 0,
'changed': 0,
'failed_percent': 0,
'failed': 0,
'noop_percent': 0,
'noop': 0,
'skipped_percent': 0,
'skipped': 0,
'unchanged_percent': 0,
'unchanged': 0,
'unreported_percent': 0,
'unreported': 0,
}
for node in nodes:
if node.status == 'unreported':
stats['unreported'] += 1
elif node.status == 'changed':
stats['changed'] += 1
elif node.status == 'failed':
stats['failed'] += 1
elif node.status == 'noop':
stats['noop'] += 1
elif node.status == 'skipped':
stats['skipped'] += 1
else:
stats['unchanged'] += 1
try:
stats['changed_percent'] = int(100 *
(stats['changed'] / float(num_nodes)))
stats['failed_percent'] = int(100 * stats['failed'] / float(num_nodes))
stats['noop_percent'] = int(100 * stats['noop'] / float(num_nodes))
stats['skipped_percent'] = int(100 *
(stats['skipped'] / float(num_nodes)))
stats['unchanged_percent'] = int(
100 * (stats['unchanged'] / float(num_nodes)))
stats['unreported_percent'] = int(
100 * (stats['unreported'] / float(num_nodes)))
except ZeroDivisionError:
stats['changed_percent'] = 0
stats['failed_percent'] = 0
stats['noop_percent'] = 0
stats['skipped_percent'] = 0
stats['unchanged_percent'] = 0
stats['unreported_percent'] = 0
if ('Accept' in request.headers
and request.headers["Accept"] == 'application/json'):
return jsonify(**stats)
return render_template('radiator.html', stats=stats, total=num_nodes)
def index(env):
"""This view generates the index page and displays a set of metrics and
latest reports on nodes fetched from PuppetDB.
:param env: Search for nodes in this (Catalog and Fact) environment
:type env: :obj:`string`
"""
envs = environments()
metrics = {'num_nodes': 0, 'num_resources': 0, 'avg_resources_node': 0}
check_env(env, envs)
if env == '*':
query = app.config['OVERVIEW_FILTER']
prefix = 'puppetlabs.puppetdb.population'
db_version = get_db_version(puppetdb)
query_type, metric_version = metric_params(db_version)
num_nodes = get_or_abort(puppetdb.metric,
"{0}{1}".format(
prefix, ':%sname=num-nodes' % query_type),
version=metric_version)
num_resources = get_or_abort(puppetdb.metric,
"{0}{1}".format(
prefix,
':%sname=num-resources' % query_type),
version=metric_version)
metrics['num_nodes'] = num_nodes['Value']
metrics['num_resources'] = num_resources['Value']
try:
# Compute our own average because avg_resources_node['Value']
# returns a string of the format "num_resources/num_nodes"
# example: "1234/9" instead of doing the division itself.
metrics['avg_resources_node'] = "{0:10.0f}".format(
(num_resources['Value'] / num_nodes['Value']))
except ZeroDivisionError:
metrics['avg_resources_node'] = 0
else:
query = AndOperator()
query.add(EqualsOperator('catalog_environment', env))
num_nodes_query = ExtractOperator()
num_nodes_query.add_field(FunctionOperator('count'))
num_nodes_query.add_query(query)
if app.config['OVERVIEW_FILTER'] is not None:
query.add(app.config['OVERVIEW_FILTER'])
num_resources_query = ExtractOperator()
num_resources_query.add_field(FunctionOperator('count'))
num_resources_query.add_query(EqualsOperator("environment", env))
num_nodes = get_or_abort(puppetdb._query,
'nodes',
query=num_nodes_query)
num_resources = get_or_abort(puppetdb._query,
'resources',
query=num_resources_query)
metrics['num_nodes'] = num_nodes[0]['count']
metrics['num_resources'] = num_resources[0]['count']
try:
metrics['avg_resources_node'] = "{0:10.0f}".format(
(num_resources[0]['count'] / num_nodes[0]['count']))
except ZeroDivisionError:
metrics['avg_resources_node'] = 0
nodes = get_or_abort(puppetdb.nodes,
query=query,
unreported=app.config['UNRESPONSIVE_HOURS'],
with_status=True,
with_event_numbers=app.config['WITH_EVENT_NUMBERS'])
nodes_overview = []
stats = {
'changed': 0,
'unchanged': 0,
'failed': 0,
'unreported': 0,
'noop': 0
}
for node in nodes:
if node.status == 'unreported':
stats['unreported'] += 1
elif node.status == 'changed':
stats['changed'] += 1
elif node.status == 'failed':
stats['failed'] += 1
elif node.status == 'noop':
stats['noop'] += 1
else:
stats['unchanged'] += 1
if node.status != 'unchanged':
nodes_overview.append(node)
return render_template('index.html',
metrics=metrics,
nodes=nodes_overview,
stats=stats,
envs=envs,
current_env=env)
