def node(env, node_name):
"""Display a dashboard for a node showing as much data as we have on that
node. This includes facts and reports but not Resources as that is too
heavy to do within a single request.
:param env: Ensure that the node, facts and reports are in this environment
:type env: :obj:`string`
"""
envs = environments()
check_env(env, envs)
query = AndOperator()
if env != '*':
query.add(EqualsOperator("environment", env))
query.add(EqualsOperator("certname", node_name))
node = get_or_abort(puppetdb.node, node_name)
facts = node.facts()
reports = get_or_abort(puppetdb.reports,
query=query,
limit=app.config['REPORTS_COUNT'],
order_by=DEFAULT_ORDER_BY)
reports, reports_events = tee(reports)
report_event_counts = {}
for report in reports_events:
report_event_counts[report.hash_] = {}
for event in report.events():
if event.status == 'success':
try:
report_event_counts[report.hash_]['successes'] += 1
except KeyError:
report_event_counts[report.hash_]['successes'] = 1
elif event.status == 'failure':
try:
report_event_counts[report.hash_]['failures'] += 1
except KeyError:
report_event_counts[report.hash_]['failures'] = 1
elif event.status == 'noop':
try:
report_event_counts[report.hash_]['noops'] += 1
except KeyError:
report_event_counts[report.hash_]['noops'] = 1
elif event.status == 'skipped':
try:
report_event_counts[report.hash_]['skips'] += 1
except KeyError:
report_event_counts[report.hash_]['skips'] = 1
return render_template(
'node.html',
node=node,
facts=yield_or_stop(facts),
reports=yield_or_stop(reports),
reports_count=app.config['REPORTS_COUNT'],
report_event_counts=report_event_counts,
envs=envs,
current_env=env)
def node(env, node_name):
"""Display a dashboard for a node showing as much data as we have on that
node. This includes facts and reports but not Resources as that is too
heavy to do within a single request.
:param env: Ensure that the node, facts and reports are in this environment
:type env: :obj:`string`
"""
envs = environments()
check_env(env, envs)
if env == '*':
query = '["=", "certname", "{0}"]]'.format(node_name)
else:
query='["and", ["=", "environment", "{0}"],' \
'["=", "certname", "{1}"]]'.format(env, node_name),
node = get_or_abort(puppetdb.node, node_name)
facts = node.facts()
reports = get_or_abort(puppetdb.reports,
query=query,
limit=app.config['REPORTS_COUNT'],
order_by='[{"field": "start_time", "order": "desc"}]')
reports, reports_events = tee(reports)
report_event_counts = {}
for report in reports_events:
report_event_counts[report.hash_] = {}
for event in report.events():
if event.status == 'success':
try:
report_event_counts[report.hash_]['successes'] += 1
except KeyError:
report_event_counts[report.hash_]['successes'] = 1
elif event.status == 'failure':
try:
report_event_counts[report.hash_]['failures'] += 1
except KeyError:
report_event_counts[report.hash_]['failures'] = 1
elif event.status == 'noop':
try:
report_event_counts[report.hash_]['noops'] += 1
except KeyError:
report_event_counts[report.hash_]['noops'] = 1
elif event.status == 'skipped':
try:
report_event_counts[report.hash_]['skips'] += 1
except KeyError:
report_event_counts[report.hash_]['skips'] = 1
return render_template(
'node.html',
node=node,
facts=yield_or_stop(facts),
reports=yield_or_stop(reports),
reports_count=app.config['REPORTS_COUNT'],
report_event_counts=report_event_counts,
envs=envs,
current_env=env)
def node(env, node_name):
"""Display a dashboard for a node showing as much data as we have on that
node. This includes facts and reports but not Resources as that is too
heavy to do within a single request.
:param env: Ensure that the node, facts and reports are in this environment
:type env: :obj:`string`
"""
envs = environments()
check_env(env, envs)
query = AndOperator()
if env != '*':
query.add(EqualsOperator("environment", env))
query.add(EqualsOperator("certname", node_name))
node = get_or_abort(puppetdb.node, node_name)
facts = node.facts()
reports = get_or_abort(puppetdb.reports,
query=query,
limit=app.config['REPORTS_COUNT'],
order_by=DEFAULT_ORDER_BY)
reports, reports_events = tee(reports)
report_event_counts = {}
for report in reports_events:
report_event_counts[report.hash_] = {}
for event in report.events():
if event.status == 'success':
try:
report_event_counts[report.hash_]['successes'] += 1
except KeyError:
report_event_counts[report.hash_]['successes'] = 1
elif event.status == 'failure':
try:
report_event_counts[report.hash_]['failures'] += 1
except KeyError:
report_event_counts[report.hash_]['failures'] = 1
elif event.status == 'noop':
try:
report_event_counts[report.hash_]['noops'] += 1
except KeyError:
report_event_counts[report.hash_]['noops'] = 1
elif event.status == 'skipped':
try:
report_event_counts[report.hash_]['skips'] += 1
except KeyError:
report_event_counts[report.hash_]['skips'] = 1
return render_template('node.html',
node=node,
facts=yield_or_stop(facts),
reports=yield_or_stop(reports),
reports_count=app.config['REPORTS_COUNT'],
report_event_counts=report_event_counts,
envs=envs,
current_env=env)
def node(node_name):
"""Display a dashboard for a node showing as much data as we have on that
node. This includes facts and reports but not Resources as that is too
heavy to do within a single request.
"""
node = get_or_abort(puppetdb.node, node_name)
facts = node.facts()
reports = ten_reports(node.reports())
return render_template('node.html',
node=node,
facts=yield_or_stop(facts),
reports=yield_or_stop(reports))
def node(node_name):
"""Display a dashboard for a node showing as much data as we have on that
node. This includes facts and reports but not Resources as that is too
heavy to do within a single request.
"""
node = get_or_abort(puppetdb.node, node_name)
facts = node.facts()
reports = ten_reports(node.reports())
return render_template(
'node.html',
node=node,
facts=yield_or_stop(facts),
reports=yield_or_stop(reports))
def node(node_name):
"""Display a dashboard for a node showing as much data as we have on that
node. This includes facts and reports but not Resources as that is too
heavy to do within a single request.
"""
node = get_or_abort(puppetdb.node, node_name)
facts = node.facts()
reports = limit_reports(node.reports(), app.config["REPORTS_COUNT"])
return render_template(
"node.html",
node=node,
facts=yield_or_stop(facts),
reports=yield_or_stop(reports),
reports_count=app.config["REPORTS_COUNT"],
)
def mwapps():
localfacts = [f for f in yield_or_stop(puppetdb.facts(name='mwapps'))]
funfacts = []
for fact in localfacts:
if fact.value == "false": continue
for appl in fact.value.split(";"):
if ":" in appl:
(appn, instnr) = appl.split(":")
node = puppetdb.node(fact.node)
mwapp_factname = "mwapp_" + appn + "_" + instnr + "_version"
try:
applver = node.fact(mwapp_factname).value
except:
applver = "Err"
pass
tmphash = { "node" : fact.node, "application" : appn, "instance" : instnr, "version" : applver }
funfacts.append(tmphash)
else:
tmphash = { "node" : fact.node, "application" : appl, "instance" : "NA", "version" : applver}
funfacts.append(tmphash)
return Response(stream_with_context(stream_template(
'mwapps.html',
name='Meltwater Apps',
facts=funfacts)))
def reports_node(node):
"""Fetches all reports for a node and processes them eventually rendering
a table displaying those reports."""
reports = ten_reports(
yield_or_stop(puppetdb.reports(
'["=", "certname", "{0}"]'.format(node))))
return render_template('reports_node.html', reports=reports, nodename=node)
def nodes(env):
"""Fetch all (active) nodes from PuppetDB and stream a table displaying
those nodes.
Downside of the streaming aproach is that since we've already sent our
headers we can't abort the request if we detect an error. Because of this
we'll end up with an empty table instead because of how yield_or_stop
works. Once pagination is in place we can change this but we'll need to
provide a search feature instead.
:param env: Search for nodes in this (Catalog and Fact) environment
:type env: :obj:`string`
"""
check_env(env)
status_arg = request.args.get('status', '')
nodelist = puppetdb.nodes(
query='["and", {0}]'.format(
", ".join('["=", "{0}", "{1}"]'.format(field, env)
for field in ['catalog_environment', 'facts_environment'])),
unreported=app.config['UNRESPONSIVE_HOURS'],
with_status=True)
nodes = []
for node in yield_or_stop(nodelist):
if status_arg:
if node.status == status_arg:
nodes.append(node)
else:
nodes.append(node)
return Response(stream_with_context(
stream_template('nodes.html',
nodes=nodes,
envs=envs,
current_env=env)))
def nodes(env):
"""Fetch all (active) nodes from PuppetDB and stream a table displaying
those nodes.
Downside of the streaming aproach is that since we've already sent our
headers we can't abort the request if we detect an error. Because of this
we'll end up with an empty table instead because of how yield_or_stop
works. Once pagination is in place we can change this but we'll need to
provide a search feature instead.
:param env: Search for nodes in this (Catalog and Fact) environment
:type env: :obj:`string`
"""
envs = environments()
check_env(env, envs)
if env == "*":
query = None
else:
query = AndOperator()
query.add(EqualsOperator("catalog_environment", env))
query.add(EqualsOperator("facts_environment", env))
status_arg = request.args.get("status", "")
nodelist = puppetdb.nodes(query=query, unreported=app.config["UNRESPONSIVE_HOURS"], with_status=True)
nodes = []
for node in yield_or_stop(nodelist):
if status_arg:
if node.status == status_arg:
nodes.append(node)
else:
nodes.append(node)
return Response(stream_with_context(stream_template("nodes.html", nodes=nodes, envs=envs, current_env=env)))
def nodes():
"""Fetch all (active) nodes from PuppetDB and stream a table displaying
those nodes.
Downside of the streaming aproach is that since we've already sent our
headers we can't abort the request if we detect an error. Because of this
we'll end up with an empty table instead because of how yield_or_stop
works. Once pagination is in place we can change this but we'll need to
provide a search feature instead.
"""
status_arg = request.args.get('status', '')
nodelist = puppetdb.nodes(
unreported=app.config['UNRESPONSIVE_HOURS'],
with_status=True)
node_facts = puppetdb.facts(name="operatingsystem")
osfacts = {}
for f in node_facts:
if not osfacts.has_key(f.node):
osfacts[f.node] = f.value.lower()
nodes = []
for node in yield_or_stop(nodelist):
if osfacts.has_key(node.name):
node.os = osfacts[node.name]
if status_arg:
if node.status == status_arg:
nodes.append(node)
else:
nodes.append(node)
return Response(stream_with_context(
stream_template('nodes.html', nodes=nodes)))
def node(env, node_name):
"""Display a dashboard for a node showing as much data as we have on that
node. This includes facts and reports but not Resources as that is too
heavy to do within a single request.
:param env: Ensure that the node, facts and reports are in this environment
:type env: :obj:`string`
"""
envs = environments()
check_env(env, envs)
query = AndOperator()
if env != '*':
query.add(EqualsOperator("environment", env))
query.add(EqualsOperator("certname", node_name))
node = get_or_abort(puppetdb.node, node_name)
facts = node.facts()
return render_template(
'node.html',
node=node,
facts=yield_or_stop(facts),
envs=envs,
current_env=env,
columns=REPORTS_COLUMNS[:2])
def fact(env, fact):
"""Fetches the specific fact from PuppetDB and displays its value per
node for which this fact is known.
:param env: Searches for facts in this environment
:type env: :obj:`string`
:param fact: Find all facts with this name
:type fact: :obj:`string`
"""
envs = environments()
check_env(env, envs)
# we can only consume the generator once, lists can be doubly consumed
# om nom nom
render_graph = False
if fact in graph_facts:
render_graph = True
if env == "*":
query = None
else:
query = EqualsOperator("environment", env)
localfacts = [f for f in yield_or_stop(puppetdb.facts(name=fact, query=query))]
return Response(
stream_with_context(
stream_template(
"fact.html", name=fact, render_graph=render_graph, facts=localfacts, envs=envs, current_env=env
)
)
)
def reports(env, page):
"""Displays a list of reports and status from all nodes, retreived using the
reports endpoint, sorted by start_time.
:param env: Search for all reports in this environment
:type env: :obj:`string`
:param page: Calculates the offset of the query based on the report count
and this value
:type page: :obj:`int`
"""
envs = environments()
check_env(env, envs)
if env == '*':
reports_query = None
total_query = '["extract", [["function", "count"]], ["~", "certname", ""]]'
else:
reports_query = '["=", "environment", "{0}"]'.format(env)
total_query = '["extract", [["function", "count"]],'\
'["and", ["=", "environment", "{0}"]]]'.format(env)
reports = get_or_abort(
puppetdb.reports,
query=reports_query,
limit=app.config['REPORTS_COUNT'],
offset=(page - 1) * app.config['REPORTS_COUNT'],
order_by='[{"field": "start_time", "order": "desc"}]')
total = get_or_abort(puppetdb._query, 'reports', query=total_query)
total = total[0]['count']
reports, reports_events = tee(reports)
report_event_counts = {}
if total == 0 and page != 1:
abort(404)
for report in reports_events:
counts = get_or_abort(puppetdb.event_counts,
query='["and",' \
'["=", "environment", "{0}"],' \
'["=", "certname", "{1}"],' \
'["=", "report", "{2}"]]'.format(
env,
report.node,
report.hash_),
summarize_by="certname")
try:
report_event_counts[report.hash_] = counts[0]
except IndexError:
report_event_counts[report.hash_] = {}
return Response(
stream_with_context(
stream_template('reports.html',
reports=yield_or_stop(reports),
reports_count=app.config['REPORTS_COUNT'],
report_event_counts=report_event_counts,
pagination=Pagination(page,
app.config['REPORTS_COUNT'],
total),
envs=envs,
current_env=env)))
def fact_value(env, fact, value):
"""On asking for fact/value get all nodes with that fact.
:param env: Searches for facts in this environment
:type env: :obj:`string`
:param fact: Find all facts with this name
:type fact: :obj:`string`
:param value: Filter facts whose value is equal to this
:type value: :obj:`string`
"""
check_env(env)
if env == '*':
query = None
else:
query = '["=", "environment", "{0}"]'.format(env)
facts = get_or_abort(puppetdb.facts,
name=fact,
value=value,
query=query)
localfacts = [f for f in yield_or_stop(facts)]
return render_template(
'fact.html',
name=fact,
value=value,
facts=localfacts,
envs=envs,
current_env=env)
def events_node_count(node, max_events):
"""Fetches all events for a node"""
try:
max_events = int(max_events)
except:
max_events = app.config['DEFAULT_EVENTS']
node = get_or_abort(puppetdb.node, node)
events = yield_or_stop(
puppetdb.events('["=", "certname", "{0}"]'.format(node),
'[{"field": "timestamp", "order": "desc"}]',
max_events))
events_list = []
for event in events:
events_list.append(event)
events_list.sort(key=lambda e: e.timestamp, reverse=True)
reports = collections.OrderedDict()
for event in events_list:
if reports.has_key(event.hash_) is False:
reports[event.hash_] = []
reports[event.hash_].append(event)
return render_template('events_node.html',
node=node,
reports=reports,
fromDate=events_list[-1].timestamp,
toDate=events_list[0].timestamp,
event_count=len(events_list))
def node(env, node_name):
"""Display a dashboard for a node showing as much data as we have on that
node. This includes facts and reports but not Resources as that is too
heavy to do within a single request.
:param env: Ensure that the node, facts and reports are in this environment
:type env: :obj:`string`
"""
envs = environments()
check_env(env, envs)
query = AndOperator()
if env != '*':
query.add(EqualsOperator("environment", env))
query.add(EqualsOperator("certname", node_name))
node = get_or_abort(puppetdb.node, node_name)
facts = node.facts()
return render_template('node.html',
node=node,
facts=yield_or_stop(facts),
envs=envs,
current_env=env,
columns=REPORTS_COLUMNS[:2])
def fact(env, fact):
"""Fetches the specific fact from PuppetDB and displays its value per
node for which this fact is known.
:param env: Searches for facts in this environment
:type env: :obj:`string`
:param fact: Find all facts with this name
:type fact: :obj:`string`
"""
envs = environments()
check_env(env, envs)
# we can only consume the generator once, lists can be doubly consumed
# om nom nom
render_graph = False
if fact in graph_facts:
render_graph = True
if env == '*':
query = None
else:
query = EqualsOperator("environment", env)
localfacts = [
f for f in yield_or_stop(puppetdb.facts(name=fact, query=query))
]
return Response(
stream_with_context(
stream_template('fact.html',
name=fact,
render_graph=render_graph,
facts=localfacts,
envs=envs,
current_env=env)))
def fact_value(env, fact, value):
"""On asking for fact/value get all nodes with that fact.
:param env: Searches for facts in this environment
:type env: :obj:`string`
:param fact: Find all facts with this name
:type fact: :obj:`string`
:param value: Filter facts whose value is equal to this
:type value: :obj:`string`
"""
envs = environments()
check_env(env, envs)
if env == '*':
query = None
else:
query = EqualsOperator("environment", env)
facts = get_or_abort(puppetdb.facts, name=fact, value=value, query=query)
localfacts = [f for f in yield_or_stop(facts)]
return render_template('fact.html',
name=fact,
value=value,
facts=localfacts,
envs=envs,
current_env=env)
def fact(fact):
"""Fetches the specific fact from PuppetDB and displays its value per
node for which this fact is known."""
# we can only consume the generator once, lists can be doubly consumed
# om nom nom
localfacts = [f for f in yield_or_stop(puppetdb.facts(name=fact))]
return Response(stream_with_context(stream_template("fact.html", name=fact, facts=localfacts)))
def node(env, node_name):
"""Display a dashboard for a node showing as much data as we have on that
node. This includes facts and reports but not Resources as that is too
heavy to do within a single request.
:param env: Ensure that the node, facts and reports are in this environment
:type env: :obj:`string`
"""
envs = environments()
check_env(env, envs)
if env == '*':
query = '["=", "certname", "{0}"]]'.format(node_name)
else:
query='["and", ["=", "environment", "{0}"],' \
'["=", "certname", "{1}"]]'.format(env, node_name),
node = get_or_abort(puppetdb.node, node_name)
facts = node.facts()
reports = get_or_abort(
puppetdb.reports,
query=query,
limit=app.config['REPORTS_COUNT'],
order_by='[{"field": "start_time", "order": "desc"}]')
reports, reports_events = tee(reports)
report_event_counts = {}
for report in reports_events:
counts = get_or_abort(puppetdb.event_counts,
query='["and", ["=", "environment", "{0}"],' \
'["=", "certname", "{1}"], ["=", "report", "{2}"]]'.format(
env,
node_name,
report.hash_),
summarize_by="certname")
try:
report_event_counts[report.hash_] = counts[0]
except IndexError:
report_event_counts[report.hash_] = {}
return render_template('node.html',
node=node,
facts=yield_or_stop(facts),
reports=yield_or_stop(reports),
reports_count=app.config['REPORTS_COUNT'],
report_event_counts=report_event_counts,
envs=envs,
current_env=env)
def fact(fact):
"""Fetches the specific fact from PuppetDB and displays its value per
node for which this fact is known."""
return Response(
stream_with_context(
stream_template('fact.html',
name=fact,
facts=yield_or_stop(puppetdb.facts(name=fact)))))
def test_stop_conn_error():
def my_generator():
yield 1
raise ConnectionError
yield 2
gen = utils.yield_or_stop(my_generator())
for val in gen:
assert 1 == val
def test_stop_empty():
def my_generator():
yield 1
raise EmptyResponseError
gen = utils.yield_or_stop(my_generator())
for val in gen:
assert 1 == val
def test_stop_conn_error():
def my_generator():
yield 1
raise ConnectionError
yield 2
gen = utils.yield_or_stop(my_generator())
for val in gen:
assert 1 == val
def test_stop_http_error():
def my_generator():
yield 1
raise HTTPError
yield 2
gen = utils.yield_or_stop(my_generator())
for val in gen:
assert 1 == val
def test_stop_http_error():
def my_generator():
yield 1
raise HTTPError
yield 2
gen = utils.yield_or_stop(my_generator())
for val in gen:
assert 1 == val
def fact_value(fact, value):
"""On asking for fact/value get all nodes with that fact."""
facts = get_or_abort(puppetdb.facts, fact, value)
localfacts = [f for f in yield_or_stop(facts)]
return render_template('fact.html',
name=fact,
value=value,
facts=localfacts)
def reports(env, page):
"""Displays a list of reports and status from all nodes, retreived using the
reports endpoint, sorted by start_time.
:param env: Search for all reports in this environment
:type env: :obj:`string`
:param page: Calculates the offset of the query based on the report count
and this value
:type page: :obj:`int`
"""
envs = environments()
check_env(env, envs)
if env == '*':
reports_query = None
total_query = '["extract", [["function", "count"]], ["~", "certname", ""]]'
else:
reports_query = '["=", "environment", "{0}"]'.format(env)
total_query = '["extract", [["function", "count"]],'\
'["and", ["=", "environment", "{0}"]]]'.format(env)
reports = get_or_abort(puppetdb.reports,
query=reports_query,
limit=app.config['REPORTS_COUNT'],
offset=(page-1) * app.config['REPORTS_COUNT'],
order_by='[{"field": "start_time", "order": "desc"}]')
total = get_or_abort(puppetdb._query,
'reports',
query=total_query)
total = total[0]['count']
reports, reports_events = tee(reports)
report_event_counts = {}
if total == 0 and page != 1:
abort(404)
for report in reports_events:
counts = get_or_abort(puppetdb.event_counts,
query='["and",' \
'["=", "environment", "{0}"],' \
'["=", "certname", "{1}"],' \
'["=", "report", "{2}"]]'.format(
env,
report.node,
report.hash_),
summarize_by="certname")
try:
report_event_counts[report.hash_] = counts[0]
except IndexError:
report_event_counts[report.hash_] = {}
return Response(stream_with_context(stream_template(
'reports.html',
reports=yield_or_stop(reports),
reports_count=app.config['REPORTS_COUNT'],
report_event_counts=report_event_counts,
pagination=Pagination(page, app.config['REPORTS_COUNT'], total),
envs=envs,
current_env=env)))
def nodes(env):
"""Fetch all (active) nodes from PuppetDB and stream a table displaying
those nodes.
Downside of the streaming aproach is that since we've already sent our
headers we can't abort the request if we detect an error. Because of this
we'll end up with an empty table instead because of how yield_or_stop
works. Once pagination is in place we can change this but we'll need to
provide a search feature instead.
:param env: Search for nodes in this (Catalog and Fact) environment
:type env: :obj:`string`
"""
envs = environments()
status_arg = request.args.get('status', '')
check_env(env, envs)
query = AndOperator()
if env != '*':
query.add(EqualsOperator("catalog_environment", env))
query.add(EqualsOperator("facts_environment", env))
if status_arg in ['failed', 'changed', 'unchanged']:
query.add(EqualsOperator('latest_report_status', status_arg))
elif status_arg == 'unreported':
unreported = datetime.datetime.utcnow()
unreported = (unreported -
timedelta(hours=app.config['UNRESPONSIVE_HOURS']))
unreported = unreported.replace(microsecond=0).isoformat()
unrep_query = OrOperator()
unrep_query.add(NullOperator('report_timestamp', True))
unrep_query.add(LessEqualOperator('report_timestamp', unreported))
query.add(unrep_query)
if len(query.operations) == 0:
query = None
nodelist = puppetdb.nodes(
query=query,
unreported=app.config['UNRESPONSIVE_HOURS'],
with_status=True,
with_event_numbers=app.config['WITH_EVENT_NUMBERS'])
nodes = []
for node in yield_or_stop(nodelist):
if status_arg:
if node.status == status_arg:
nodes.append(node)
else:
nodes.append(node)
return Response(
stream_with_context(
stream_template('nodes.html',
nodes=nodes,
envs=envs,
current_env=env)))
def fact_value(fact, value):
"""On asking for fact/value get all nodes with that fact."""
facts = get_or_abort(puppetdb.facts, fact, value)
localfacts = [f for f in yield_or_stop(facts)]
return render_template(
'fact.html',
name=fact,
value=value,
facts=localfacts)
def test_stop_conn_error(self):
def my_generator():
yield 1
raise ConnectionError
yield 2
gen = utils.yield_or_stop(my_generator())
for val in gen:
self.assertEqual(1, val)
def test_stop_conn_error(self):
def my_generator():
yield 1
raise ConnectionError
yield 2
gen = utils.yield_or_stop(my_generator())
for val in gen:
self.assertEqual(1, val)
def node(node_name):
"""Display a dashboard for a node showing as much data as we have on that
node. This includes facts and reports but not Resources as that is too
heavy to do within a single request.
"""
node = get_or_abort(puppetdb.node, node_name)
facts = [f for f in yield_or_stop(node.facts())]
for fact in facts:
if fact.name == 'operatingsystem':
node.os = fact.value.lower()
reports = node.reports(order_by='[{"field": "receive-time", "order": "desc"}]', limit=app.config['REPORTS_COUNT'])
return render_template(
'node.html',
node=node,
facts=facts,
reports=yield_or_stop(reports),
reports_count=app.config['REPORTS_COUNT'])
def test_stop_empty(self):
def my_generator():
yield 1
raise EmptyResponseError
yield 2
gen = utils.yield_or_stop(my_generator())
for val in gen:
self.assertEqual(1, val)
def test_stop_empty():
def my_generator():
yield 1
raise EmptyResponseError
yield 2
gen = utils.yield_or_stop(my_generator())
for val in gen:
assert 1 == val
def reports_node(node):
"""Fetches all reports for a node and processes them eventually rendering
a table displaying those reports."""
reports = ten_reports(yield_or_stop(
puppetdb.reports('["=", "certname", "{0}"]'.format(node))))
return render_template(
'reports_node.html',
reports=reports,
nodename=node)
def reports_node(node):
"""Fetches all reports for a node and processes them eventually rendering
a table displaying those reports."""
if app.config["PUPPETDB_API"] > 2:
reports = ten_reports(yield_or_stop(puppetdb.reports('["=", "certname", "{0}"]'.format(node))))
else:
log.warn("PuppetDB API prior to v3 cannot access reports.")
abort(412)
return render_template("reports_node.html", reports=reports, nodename=node)
def reports_node(node_name):
"""Fetches all reports for a node and processes them eventually rendering
a table displaying those reports."""
reports = limit_reports(
yield_or_stop(puppetdb.reports(query='["=", "certname", "{0}"]'.format(node_name))), app.config["REPORTS_COUNT"]
)
return render_template(
"reports_node.html", reports=reports, nodename=node_name, reports_count=app.config["REPORTS_COUNT"]
)
def fact(fact):
"""Fetches the specific fact from PuppetDB and displays its value per
node for which this fact is known."""
# we can only consume the generator once, lists can be doubly consumed
# om nom nom
localfacts = [f for f in yield_or_stop(puppetdb.facts(name=fact))]
return Response(
stream_with_context(
stream_template('fact.html', name=fact, facts=localfacts)))
def reports_node(node):
"""Fetches all reports for a node and processes them eventually rendering
a table displaying those reports."""
if app.config["PUPPETDB_EXPERIMENTAL"]:
reports = ten_reports(yield_or_stop(puppetdb.reports('["=", "certname", "{0}"]'.format(node))))
else:
log.warn("Access to experimental endpoint not allowed.")
abort(412)
return render_template("reports_node.html", reports=reports, nodename=node)
def test_stop_http_error(self):
def my_generator():
yield 1
raise HTTPError
yield 2
gen = utils.yield_or_stop(my_generator())
for val in gen:
self.assertEqual(1, val)
def test_stop_http_error(self):
def my_generator():
yield 1
raise HTTPError
yield 2
gen = utils.yield_or_stop(my_generator())
for val in gen:
self.assertEqual(1, val)
def test_stop_empty(self):
def my_generator():
yield 1
raise EmptyResponseError
yield 2
gen = utils.yield_or_stop(my_generator())
for val in gen:
self.assertEqual(1, val)
def nodes(env):
"""Fetch all (active) nodes from PuppetDB and stream a table displaying
those nodes.
Downside of the streaming aproach is that since we've already sent our
headers we can't abort the request if we detect an error. Because of this
we'll end up with an empty table instead because of how yield_or_stop
works. Once pagination is in place we can change this but we'll need to
provide a search feature instead.
:param env: Search for nodes in this (Catalog and Fact) environment
:type env: :obj:`string`
"""
envs = environments()
status_arg = request.args.get('status', '')
check_env(env, envs)
query = AndOperator()
if env != '*':
query.add(EqualsOperator("catalog_environment", env))
query.add(EqualsOperator("facts_environment", env))
if status_arg in ['failed', 'changed', 'unchanged']:
query.add(EqualsOperator('latest_report_status', status_arg))
elif status_arg == 'unreported':
unreported = datetime.datetime.utcnow()
unreported = (unreported -
timedelta(hours=app.config['UNRESPONSIVE_HOURS']))
unreported = unreported.replace(microsecond=0).isoformat()
unrep_query = OrOperator()
unrep_query.add(NullOperator('report_timestamp', True))
unrep_query.add(LessEqualOperator('report_timestamp', unreported))
query.add(unrep_query)
if len(query.operations) == 0:
query = None
nodelist = puppetdb.nodes(
query=query,
unreported=app.config['UNRESPONSIVE_HOURS'],
with_status=True)
nodes = []
for node in yield_or_stop(nodelist):
if status_arg:
if node.status == status_arg:
nodes.append(node)
else:
nodes.append(node)
return Response(stream_with_context(
stream_template('nodes.html',
nodes=nodes,
envs=envs,
current_env=env)))
def report_latest(node_name):
"""Redirect to the latest report of a given node. This is a workaround
as long as PuppetDB can't filter reports for latest-report? field. This
feature has been requested: http://projects.puppetlabs.com/issues/21554
"""
# TODO: use limit parameter in _query to get just one report
node = get_or_abort(puppetdb.node, node_name)
reports = ten_reports(node.reports())
report = list(yield_or_stop(reports))[0]
return redirect(url_for('report', node=node_name, report_id=report))
def reports_node(node):
"""Fetches all reports for a node and processes them eventually rendering
a table displaying those reports."""
reports = limit_reports(yield_or_stop(
puppetdb.reports('["=", "certname", "{0}"]'.format(node))), app.config['REPORTS_COUNT'])
return render_template(
'reports_node.html',
reports=reports,
nodename=node,
reports_count=app.config['REPORTS_COUNT'])
def report_latest(node_name):
"""Redirect to the latest report of a given node. This is a workaround
as long as PuppetDB can't filter reports for latest-report? field. This
feature has been requested: http://projects.puppetlabs.com/issues/21554
"""
# TODO: use limit parameter in _query to get just one report
node = get_or_abort(puppetdb.node, node_name)
reports = ten_reports(node.reports())
report = list(yield_or_stop(reports))[0]
return redirect(url_for('report', node=node_name, report_id=report))
def status():
"""Fetch all (active) nodes from PuppetDB and construct a scrapable list.
This is useful for feeding puppet data into a monitoring system such as
Prometheus.
"""
nodelist = puppetdb.nodes(unreported=app.config['UNRESPONSIVE_HOURS'],
with_status=True)
facts = collections.defaultdict(dict)
try:
export_facts = set(app.config['EXPORT_FACTS'])
except KeyError:
export_facts = set([
'kernelrelease', 'lsbdistdescription', 'architecture',
'puppetversion', 'kernel', 'processorcount'
])
for fact in puppetdb.facts():
if fact.name not in export_facts:
continue
facts[fact.node][fact.name] = fact.value
nodes = [x for x in yield_or_stop(nodelist)]
def generate():
yield '# HELP puppet_node Puppet inventory\n'
yield '# TYPE puppet_node gauge\n'
for node in nodes:
yield 'puppet_node{'
yield 'status="%s",' % node.status
for fact, value in facts[node.name].iteritems():
yield fact
yield '="'
yield value.replace('\"', '\\"')
yield '",'
yield 'node="%s"' % node.name
yield '} 1\n'
yield '# HELP puppet_latest_report_timestamp '
yield 'UNIX epoch timestmap of the latest report\n'
yield '# TYPE puppet_latest_report_timestamp counter\n'
for node in nodes:
if node.report_timestamp is None:
continue
unix_epoch = datetime.fromtimestamp(0, tzoffset('UTC', 0))
yield 'puppet_last_updated_timestamp{'
yield 'node="%s"' % node.name
yield '} '
yield str(int(
(node.report_timestamp - unix_epoch).total_seconds()))
yield '\n'
resp = Response(stream_with_context(generate()))
resp.headers['Content-Type'.encode('ISO-8859-1')] = 'text/plain'
return resp
def status():
"""Fetch all (active) nodes from PuppetDB and construct a scrapable list.
This is useful for feeding puppet data into a monitoring system such as
Prometheus.
"""
nodelist = puppetdb.nodes(
unreported=app.config['UNRESPONSIVE_HOURS'],
with_status=True)
facts = collections.defaultdict(dict)
try:
export_facts = set(app.config['EXPORT_FACTS'])
except KeyError:
export_facts = set([
'kernelrelease', 'lsbdistdescription', 'architecture',
'puppetversion', 'kernel', 'processorcount'])
for fact in puppetdb.facts():
if fact.name not in export_facts:
continue
facts[fact.node][fact.name] = fact.value
nodes = [x for x in yield_or_stop(nodelist)]
def generate():
yield '# HELP puppet_node Puppet inventory\n'
yield '# TYPE puppet_node gauge\n'
for node in nodes:
yield 'puppet_node{'
yield 'status="%s",' % node.status
for fact, value in facts[node.name].iteritems():
yield fact
yield '="'
yield value.replace('\"', '\\"')
yield '",'
yield 'node="%s"' % node.name
yield '} 1\n'
yield '# HELP puppet_latest_report_timestamp '
yield 'UNIX epoch timestmap of the latest report\n'
yield '# TYPE puppet_latest_report_timestamp counter\n'
for node in nodes:
if node.report_timestamp is None:
continue
unix_epoch = datetime.fromtimestamp(0, tzoffset('UTC', 0))
yield 'puppet_last_updated_timestamp{'
yield 'node="%s"' % node.name
yield '} '
yield str(int((node.report_timestamp-unix_epoch).total_seconds()))
yield '\n'
resp = Response(stream_with_context(generate()))
resp.headers['Content-Type'.encode('ISO-8859-1')] = 'text/plain'
return resp
def nodes():
"""Fetch all (active) nodes from PuppetDB and stream a table displaying
those nodes.
Downside of the streaming aproach is that since we've already sent our
headers we can't abort the request if we detect an error. Because of this
we'll end up with an empty table instead because of how yield_or_stop
works. Once pagination is in place we can change this but we'll need to
provide a search feature instead.
"""
return Response(stream_with_context(stream_template("nodes.html", nodes=yield_or_stop(puppetdb.nodes()))))
def reports_node(node):
"""Fetches all reports for a node and processes them eventually rendering
a table displaying those reports."""
if app.config['PUPPETDB_EXPERIMENTAL']:
reports = ten_reports(yield_or_stop(
puppetdb.reports('["=", "certname", "{0}"]'.format(node))))
else:
log.warn('Access to experimental endpoint not allowed.')
abort(412)
return render_template('reports_node.html', reports=reports,
nodename=node)
def node(env, node_name):
"""Display a dashboard for a node showing as much data as we have on that
node. This includes facts and reports but not Resources as that is too
heavy to do within a single request.
:param env: Ensure that the node, facts and reports are in this environment
:type env: :obj:`string`
"""
check_env(env)
node = get_or_abort(puppetdb.node, node_name)
facts = node.facts()
reports = get_or_abort(puppetdb.reports,
query='["and", ["=", "environment", "{0}"],' \
'["=", "certname", "{1}"]]'.format(env, node_name),
limit=app.config['REPORTS_COUNT'],
order_by='[{"field": "start_time", "order": "desc"}]')
reports, reports_events = tee(reports)
report_event_counts = {}
for report in reports_events:
counts = get_or_abort(puppetdb.event_counts,
query='["and", ["=", "environment", "{0}"],' \
'["=", "certname", "{1}"], ["=", "report", "{2}"]]'.format(
env,
node_name,
report.hash_),
summarize_by="certname")
try:
report_event_counts[report.hash_] = counts[0]
except IndexError:
report_event_counts[report.hash_] = {}
return render_template(
'node.html',
node=node,
facts=yield_or_stop(facts),
reports=yield_or_stop(reports),
reports_count=app.config['REPORTS_COUNT'],
report_event_counts=report_event_counts,
envs=envs,
current_env=env)
def nodes():
"""Fetch all (active) nodes from PuppetDB and stream a table displaying
those nodes.
Downside of the streaming aproach is that since we've already sent our
headers we can't abort the request if we detect an error. Because of this
we'll end up with an empty table instead because of how yield_or_stop
works. Once pagination is in place we can change this but we'll need to
provide a search feature instead.
"""
return Response(stream_with_context(stream_template('nodes.html',
nodes=yield_or_stop(puppetdb.nodes()))))
def test_iter(self):
test_list = (0, 1, 2, 3)
def my_generator():
for i in test_list:
yield i
gen = utils.yield_or_stop(my_generator())
self.assertIsInstance(gen, GeneratorType)
i = 0
for val in gen:
self.assertEqual(i, val)
i = i + 1
def test_iter():
test_list = (0, 1, 2, 3)
def my_generator():
for i in test_list:
yield i
gen = utils.yield_or_stop(my_generator())
assert isinstance(gen, GeneratorType)
i = 0
for val in gen:
assert i == val
i = i + 1
def report(node, report_id):
"""Displays a single report including all the events associated with that
report and their status.
"""
reports = puppetdb.reports('["=", "certname", "{0}"]'.format(node))
for report in reports:
if report.hash_ == report_id:
events = puppetdb.events('["=", "report", "{0}"]'.format(
report.hash_))
return render_template('report.html',
report=report,
events=yield_or_stop(events))
else:
abort(404)
def report(env, node_name, report_id):
"""Displays a single report including all the events associated with that
report and their status.
The report_id may be the puppetdb's report hash or the
configuration_version. This allows for better integration
into puppet-hipchat.
:param env: Search for reports in this environment
:type env: :obj:`string`
:param node_name: Find the reports whose certname match this value
:type node_name: :obj:`string`
:param report_id: The hash or the configuration_version of the desired
report
:type report_id: :obj:`string`
"""
envs = environments()
check_env(env, envs)
query = AndOperator()
report_id_query = OrOperator()
report_id_query.add(EqualsOperator("hash", report_id))
report_id_query.add(EqualsOperator("configuration_version", report_id))
if env != '*':
query.add(EqualsOperator("environment", env))
query.add(EqualsOperator("certname", node_name))
query.add(report_id_query)
reports = puppetdb.reports(query=query)
try:
report = next(reports)
except StopIteration:
abort(404)
report.version = CommonMark.commonmark(report.version)
return render_template(
'report.html',
report=report,
events=yield_or_stop(report.events()),
logs=report.logs,
metrics=report.metrics,
envs=envs,
current_env=env)
def report(node, report_id):
"""Displays a single report including all the events associated with that
report and their status."""
if app.config['PUPPETDB_EXPERIMENTAL']:
reports = puppetdb.reports('["=", "certname", "{0}"]'.format(node))
else:
log.warn('Access to experimental endpoint not allowed.')
abort(412)
for report in reports:
if report.hash_ == report_id:
events = puppetdb.events('["=", "report", "{0}"]'.format(
report.hash_))
return render_template('report.html', report=report,
events=yield_or_stop(events))
else:
abort(404)
def report(node_name, report_id):
"""Displays a single report including all the events associated with that
report and their status.
The report_id may be the puppetdb's report hash or the
configuration_version. This allows for better integration
into puppet-hipchat.
"""
reports = puppetdb.reports('["=", "certname", "{0}"]'.format(node_name))
for report in reports:
if report.hash_ == report_id or report.version == report_id:
events = puppetdb.events('["=", "report", "{0}"]'.format(
report.hash_))
return render_template('report.html',
report=report,
events=yield_or_stop(events))
else:
abort(404)
