def report_log_json(request, report_hash=None):
source_url, source_certs, source_verify = get_server(request)
# Redirects to the events page if GET param latest is true..
context = {}
if report_hash is None:
context['error'] = 'Report Hash not provided.'
return HttpResponse(json.dumps(context), content_type="application/json")
report_logs = puppetdb.api_get(
api_url=source_url,
cert=source_certs,
verify=source_verify,
path='/reports/' + report_hash + '/logs',
api_version='v4',
)
if 'error' in report_logs:
context = report_logs
return HttpResponse(json.dumps(context), content_type="application/json")
# Remove the dict from the list...
for log in report_logs:
# Parse... 2015-09-18T18:02:04.753163330+02:00
# Puppetlabs... has a super long millisecond counter (9 digits!!!)
# We need to trim those down...
time = log['time'][0:26] + log['time'][-6:-3] + log['time'][-2:]
time = arrow.get(time).to('UTC').datetime
log['time'] = filters.date(localtime(time), 'Y-m-d H:i:s')
context['agent_log'] = report_logs
context['report_hash'] = report_hash
return HttpResponse(json.dumps(context), content_type="application/json")
def get_report(key, value, request, timespan='latest', environment=None):
source_url, source_certs, source_verify = get_server(request)
# If key is any of the below, all is good!
allowed_keys = ['certname', 'resource_title', 'resource_type', 'containing_class']
if key in allowed_keys:
pass
# If key does not match above the default will be shown
else:
key = 'containing_class'
events_params = {
'query':
{
'operator': 'and',
2: '["=","' + key + '","' + value + '"]',
3: '["in", "certname",["extract", "certname",["select_nodes",["null?","deactivated",true]]]]'
},
}
if timespan == 'latest':
events_params['query'][1] = '["=","latest_report?",true]'
elif len(timespan) == 2:
events_params['query'][1] = '["and",[">","timestamp","' + timespan[0] + '"],["<", "timestamp", "' + timespan[
1] + '"]]'
results = pdb_api_get(
api_url=source_url,
cert=source_certs,
verify=source_verify,
path='/events',
api_version='v4',
params=mk_puppetdb_query(events_params, request),
)
return results
def reports(request, certname=None):
context = {'timezones': pytz.common_timezones,
'SOURCES': AVAILABLE_SOURCES}
if request.method == 'GET':
if 'source' in request.GET:
source = request.GET.get('source')
set_server(request, source)
if request.method == 'POST':
request.session['django_timezone'] = request.POST['timezone']
return redirect(request.POST['return_url'])
source_url, source_certs, source_verify = get_server(request)
# Redirects to the events page if GET param latest is true..
if request.GET.get('latest', False):
if request.GET.get('latest') == "true":
request.session['report_page'] = 1
latest_report_params = {
'query':
{
1: '["=","certname","' + certname + '"]'
},
'order_by':
{
'order_field':
{
'field': 'start_time',
'order': 'desc',
},
},
'limit': 1,
}
latest_report = puppetdb.api_get(
api_url=source_url,
cert=source_certs,
verify=source_verify,
path='/reports',
api_version='v4',
params=puppetdb.mk_puppetdb_query(latest_report_params, request),
)
report_hash = ""
# If latest reports do not exist, send to the nodes page
# Should only occur if the user is trying to hax their way
# into a node without having the correct permission
if latest_report:
for report in latest_report:
report_env = report['environment']
report_hash = report['hash']
return redirect('/pano/events/' + report_hash + '?report_timestamp=' + request.GET.get(
'report_timestamp') + '&envname=' + report_env)
else:
return redirect('/pano/nodes/')
if certname != request.session.get('last_shown_node', ''):
request.session['last_shown_node'] = certname
request.session['report_page'] = 1
context['certname'] = certname
context['node_facts'] = ','.join(NODES_DEFAULT_FACTS)
return render(request, 'pano/reports.html', context)
def catalogue_json(request, certname=None):
context = dict()
if not certname:
context["error"] = "Must specify certname."
return HttpResponse(json.dumps(context), content_type="application/json")
source_url, source_certs, source_verify = get_server(request)
# Redirects to the events page if GET param latest is true..
show = request.GET.get("show", None)
reports_params = {}
if show is not None and show in ["edges", "resources"]:
if show == "edges":
sort_field = "source_title"
elif show == "resources":
sort_field = "title"
reports_params = {"order_by": {"order_field": {"field": sort_field, "order": "asc"}}}
path = "/catalogs/%s/%s" % (certname, show)
else:
path = "/catalogs/%s" % certname
reports_list = puppetdb.api_get(
path=path, api_url=source_url, api_version="v4", params=puppetdb.mk_puppetdb_query(reports_params, request)
)
data = {"data": reports_list}
return HttpResponse(json.dumps(data, indent=2), content_type="application/json")
def catalogue_compare_json(request, certname1=None, certname2=None):
source_url, source_certs, source_verify = get_server(request)
show = request.GET.get("show", "edges")
if show is not None and show in ["edges", "resources"]:
if show == "edges":
sort_field = "source_title"
elif show == "resources":
sort_field = "title"
certname1_params = {"order_by": {"order_field": {"field": sort_field, "order": "asc"}}}
certname2_params = {"order_by": {"order_field": {"field": sort_field, "order": "asc"}}}
certname1_data = puppetdb.api_get(
path="/catalogs/%s/%s" % (certname1, show),
api_url=source_url,
api_version="v4",
params=puppetdb.mk_puppetdb_query(certname1_params, request),
)
certname1_data = json.dumps(certname1_data, indent=2)
certname2_data = puppetdb.api_get(
path="/catalogs/%s/%s" % (certname2, show),
api_url=source_url,
api_version="v4",
params=puppetdb.mk_puppetdb_query(certname2_params, request),
)
certname2_data = json.dumps(certname2_data, indent=2)
from_split_lines = certname1_data.split("\n")
to_split_lines = certname2_data.split("\n")
diff = difflib.unified_diff(from_split_lines, to_split_lines)
diff = "\n".join(list(diff))
return HttpResponse(diff)
def search_nodes_json(request):
source_url, source_certs, source_verify = get_server(request)
if request.method == 'GET':
if 'search' in request.GET:
search = request.GET.get('search')
# Create a search regex for certname spelt with below.
nodes_params = {
'query':
{
1: '["~","certname","' + search + '"]'
},
'order_by':
{
'order_field':
{
'field': 'certname',
'order': 'desc',
},
},
#'limit': 25,
}
nodes_list = puppetdb.api_get(
api_url=source_url,
cert=source_certs,
verify=source_verify,
path='/nodes',
api_version='v4',
params=puppetdb.mk_puppetdb_query(
nodes_params, request),
)
return HttpResponse(json.dumps(nodes_list), content_type="application/json")
def get_events_summary(request, timespan='latest'):
if timespan == 'latest':
events_params = {
'query':
{
'operator': 'and',
1: '["=","latest_report?",true]',
2: '["in","certname",["extract","certname",["select_nodes",["null?","deactivated",true]]]]',
},
}
source_url, source_certs, source_verify = get_server(request)
events = pdb_api_get(
api_url=source_url,
cert=source_certs,
verify=source_verify,
path='/events',
api_version='v4',
params=mk_puppetdb_query(events_params, request))
summary = summary_of_events(events)
return summary
def get_events_summary(request, timespan='latest', environment=None):
events_params = {
'query':
{
'operator': 'and',
2: '["in","certname",["extract","certname",["select_nodes",["null?","deactivated",true]]]]',
},
}
if timespan == 'latest':
events_params['query'][1] = '["=","latest_report?",true]'
elif len(timespan) == 2:
events_params['query'][1] = '["and",[">","timestamp","' + timespan[0] + '"],["<", "timestamp", "' + timespan[
1] + '"]]'
source_url, source_certs, source_verify = get_server(request)
events = pdb_api_get(
api_url=source_url,
cert=source_certs,
verify=source_verify,
path='/events',
api_version='v4',
params=mk_puppetdb_query(events_params, request))
summary = summary_of_events(events)
return summary
def reports_search_json(request):
context = dict()
if request.method == 'GET':
if 'search' in request.GET:
search = request.GET.get('search')
if 'certname' in request.GET:
certname = request.GET.get('certname')
if not certname or not search:
context['error'] = 'Must specify both certname and search query.'
return HttpResponse(json.dumps(context), content_type="application/json")
source_url, source_certs, source_verify = get_server(request)
# Redirects to the events page if GET param latest is true..
reports_params = {
'query':
{
'operator': 'and',
1: '["=","certname","' + certname + '"]',
2: '["~","hash","^' + search + '"]'
},
'order_by':
{
'order_field':
{
'field': 'start_time',
'order': 'desc',
},
}
}
reports_list = puppetdb.api_get(
path='/reports',
api_url=source_url,
api_version='v4',
params=puppetdb.mk_puppetdb_query(reports_params, request),
)
return HttpResponse(json.dumps(reports_list), content_type="application/json")
def dashboard_json(request):
context = {}
if request.method == 'GET':
if 'source' in request.GET:
source = request.GET.get('source')
set_server(request, source)
if request.method == 'POST':
request.session['django_timezone'] = request.POST['timezone']
return redirect(request.POST['return_url'])
source_url, source_certs, source_verify = get_server(request)
puppet_run_time = get_server(request, type='run_time')
dashboard_show = request.GET.get('show', 'recent')
events_params = {
'query': {
1:
'["and",["=","latest_report?",true],["in", "certname",["extract", "certname",["select_nodes",["null?","deactivated",true]]]]]'
},
'summarize_by': 'certname',
}
reports_params = {
'query': {
1:
'["and",["=","latest_report?",true],["in", "certname",["extract", "certname",["select_nodes",["null?","deactivated",true]]]]]'
}
}
nodes_params = {
'limit': 25,
'order_by': {
'order_field': {
'field': 'report_timestamp',
'order': 'desc',
},
'query_field': {
'field': 'certname'
},
},
}
jobs = {
'tot_resource': {
'url':
source_url,
'certs':
source_certs,
'verify':
source_verify,
'id':
'tot_resource',
'path':
'mbeans/puppetlabs.puppetdb.query.population:type=default,name=num-resources',
},
'avg_resource': {
'url':
source_url,
'certs':
source_certs,
'verify':
source_verify,
'id':
'avg_resource',
'path':
'mbeans/puppetlabs.puppetdb.query.population:type=default,name=avg-resources-per-node',
},
'all_nodes': {
'url': source_url,
'certs': source_certs,
'verify': source_verify,
'api_version': 'v4',
'id': 'all_nodes',
'path': '/nodes',
'request': request
},
'events': {
'url': source_url,
'certs': source_certs,
'verify': source_verify,
'id': 'event_counts',
'path': '/event-counts',
'api_version': 'v4',
'params': events_params,
'request': request
},
'reports': {
'url': source_url,
'certs': source_certs,
'verify': source_verify,
'api_version': 'v4',
'id': 'reports',
'path': '/reports',
'params': reports_params,
'request': request
},
'nodes': {
'url': source_url,
'certs': source_certs,
'verify': source_verify,
'api_version': 'v4',
'id': 'nodes',
'path': '/nodes',
'params': nodes_params,
'request': request
},
}
puppetdb_results = run_puppetdb_jobs(jobs)
# Assign vars from the completed jobs
# Number of results from all_nodes is our population.
puppet_population = len(puppetdb_results['all_nodes'])
# Total resources managed by puppet metric
total_resources = puppetdb_results['tot_resource']
# Average resource per node metric
avg_resource_node = puppetdb_results['avg_resource']
# Information about all active nodes in puppet
all_nodes_list = puppetdb_results['all_nodes']
# All available events for the latest puppet reports
event_list = puppetdb_results['event_counts']
event_dict = {item['subject']['title']: item for item in event_list}
# All of the latest reports
reports_list = puppetdb_results['reports']
reports_dict = {item['certname']: item for item in reports_list}
# 25 Nodes
node_list = puppetdb_results['nodes']
failed_list, changed_list, unreported_list, mismatch_list, pending_list = dictstatus(
all_nodes_list,
reports_dict,
event_dict,
sort=True,
sortby='latestReport',
get_status='notall',
puppet_run_time=puppet_run_time)
pending_list = [x for x in pending_list if x not in unreported_list]
changed_list = [
x for x in changed_list if x not in unreported_list
and x not in failed_list and x not in pending_list
]
failed_list = [x for x in failed_list if x not in unreported_list]
unreported_list = [x for x in unreported_list if x not in failed_list]
node_unreported_count = len(unreported_list)
node_fail_count = len(failed_list)
node_change_count = len(changed_list)
node_off_timestamps_count = len(mismatch_list)
node_pending_count = len(pending_list)
if dashboard_show == 'recent':
merged_nodes_list = dictstatus(node_list,
reports_dict,
event_dict,
sort=False,
get_status="all",
puppet_run_time=puppet_run_time)
elif dashboard_show == 'failed':
merged_nodes_list = failed_list
elif dashboard_show == 'unreported':
merged_nodes_list = unreported_list
elif dashboard_show == 'changed':
merged_nodes_list = changed_list
elif dashboard_show == 'mismatch':
merged_nodes_list = mismatch_list
elif dashboard_show == 'pending':
merged_nodes_list = pending_list
else:
merged_nodes_list = dictstatus(node_list,
reports_dict,
event_dict,
sort=False,
get_status="all",
puppet_run_time=puppet_run_time)
context['node_list'] = merged_nodes_list
context['selected_view'] = dashboard_show
context['population'] = puppet_population
context['total_resource'] = total_resources['Value']
context['avg_resource'] = "{:.2f}".format(avg_resource_node['Value'])
context['failed_nodes'] = node_fail_count
context['changed_nodes'] = node_change_count
context['unreported_nodes'] = node_unreported_count
context['mismatching_timestamps'] = node_off_timestamps_count
context['pending_nodes'] = node_pending_count
return HttpResponse(json.dumps(context), content_type="application/json")
def dashboard_nodes_json(request):
context = {}
if request.method == 'GET':
if 'source' in request.GET:
source = request.GET.get('source')
set_server(request, source)
if request.method == 'POST':
request.session['django_timezone'] = request.POST['timezone']
return redirect(request.POST['return_url'])
source_url, source_certs, source_verify = get_server(request)
puppet_run_time = get_server(request, type='run_time')
# Dashboard to show nodes of "recent, failed, unreported or changed"
dashboard_show = request.GET.get('show', 'recent')
events_params = {
'query': {
1:
'["and",["=","latest_report?",true],["in", "certname",["extract", "certname",["select_nodes",["null?","deactivated",true]]]]]'
},
'summarize_by': 'certname',
}
all_nodes_params = {
'query': {
1:
'["and",["=","latest_report?",true],["in", "certname",["extract", "certname",["select_nodes",["null?","deactivated",true]]]]]'
},
}
reports_params = {
'query': {
1:
'["and",["=","latest_report?",true],["in", "certname",["extract", "certname",["select_nodes",["null?","deactivated",true]]]]]'
}
}
nodes_params = {
'limit': 25,
'order_by': {
'order_field': {
'field': 'report_timestamp',
'order': 'desc',
},
'query_field': {
'field': 'certname'
},
},
}
jobs = {
'all_nodes': {
'url': source_url,
'certs': source_certs,
'verify': source_verify,
'api_version': 'v4',
'id': 'all_nodes',
'path': '/nodes',
'request': request
},
'events': {
'url': source_url,
'certs': source_certs,
'verify': source_verify,
'id': 'event_counts',
'path': 'event-counts',
'api_version': 'v4',
'params': events_params,
'request': request
},
'nodes': {
'url': source_url,
'certs': source_certs,
'verify': source_verify,
'api_version': 'v4',
'id': 'nodes',
'path': '/nodes',
'params': nodes_params,
'request': request
},
'reports': {
'url': source_url,
'certs': source_certs,
'verify': source_verify,
'api_version': 'v4',
'id': 'reports',
'path': '/reports',
'params': reports_params,
'request': request
},
}
puppetdb_results = run_puppetdb_jobs(jobs)
# Information about all active nodes in puppet
all_nodes_list = puppetdb_results['all_nodes']
# All available events for the latest puppet reports
event_list = puppetdb_results['event_counts']
event_dict = {item['subject']['title']: item for item in event_list}
# All of the latest reports
reports_list = puppetdb_results['reports']
reports_dict = {item['certname']: item for item in reports_list}
# 25 Nodes
node_list = puppetdb_results['nodes']
failed_list, changed_list, unreported_list, mismatch_list, pending_list = dictstatus(
all_nodes_list,
reports_dict,
event_dict,
sort=True,
sortby='latestReport',
get_status='notall',
puppet_run_time=puppet_run_time)
pending_list = [x for x in pending_list if x not in unreported_list]
changed_list = [
x for x in changed_list if x not in unreported_list
and x not in failed_list and x not in pending_list
]
failed_list = [x for x in failed_list if x not in unreported_list]
unreported_list = [x for x in unreported_list if x not in failed_list]
if dashboard_show == 'recent':
merged_nodes_list = dictstatus(node_list,
reports_dict,
event_dict,
sort=False,
get_status="all",
puppet_run_time=puppet_run_time)
elif dashboard_show == 'failed':
merged_nodes_list = failed_list
elif dashboard_show == 'unreported':
merged_nodes_list = unreported_list
elif dashboard_show == 'changed':
merged_nodes_list = changed_list
elif dashboard_show == 'mismatch':
merged_nodes_list = mismatch_list
elif dashboard_show == 'pending':
merged_nodes_list = pending_list
else:
merged_nodes_list = dictstatus(node_list,
reports_dict,
event_dict,
sort=False,
get_status="all",
puppet_run_time=puppet_run_time)
context['node_list'] = merged_nodes_list
context['selected_view'] = dashboard_show
return HttpResponse(json.dumps(context), content_type="application/json")
def nodes_json(request):
if request.method == 'GET':
if 'source' in request.GET:
source = request.GET.get('source')
set_server(request, source)
if request.method == 'POST':
request.session['django_timezone'] = request.POST['timezone']
return redirect(request.POST['return_url'])
source_url, source_certs, source_verify = get_server(request)
puppet_run_time = get_server(request, type='run_time')
valid_sort_fields = ('certname', 'catalog_timestamp', 'report_timestamp',
'facts_timestamp', 'successes', 'noops', 'failures',
'skips')
try:
# If user requested to download csv formatted file. Default value is False
dl_csv = request.GET.get('dl_csv', False)
if dl_csv == 'true':
dl_csv = True
else:
dl_csv = False
# Add limits to session
if request.GET.get('limits', False):
if request.session['limits'] != int(request.GET.get('limits', 50)):
request.session['limits'] = int(request.GET.get('limits', 50))
if request.session['limits'] <= 0:
request.session['limits'] = 50
else:
if 'limits' not in request.session:
request.session['limits'] = 50
# Cur Page Number
if request.GET.get('page', False):
if request.session['page'] != int(request.GET.get('page', 1)):
request.session['page'] = int(request.GET.get('page', 1))
if request.session['page'] <= 0:
request.session['page'] = 1
else:
if 'page' not in request.session:
request.session['page'] = 1
# Cur sort field
if request.GET.get('sortfield', False):
if request.session['sortfield'] != request.GET.get('sortfield'):
request.session['sortfield'] = request.GET.get('sortfield')
if request.session['sortfield'] not in valid_sort_fields:
request.session['sortfield'] = 'report_timestamp'
else:
if 'sortfield' not in request.session:
request.session['sortfield'] = 'report_timestamp'
# Cur sort order
if request.GET.get('sortfieldby', False):
avail_sortorder = ['asc', 'desc']
if request.session['sortfieldby'] != request.GET.get(
'sortfieldby'):
request.session['sortfieldby'] = request.GET.get('sortfieldby')
if request.session['sortfieldby'] not in avail_sortorder:
request.session['sortfieldby'] = 'desc'
else:
if 'sortfieldby' not in request.session:
request.session['sortfieldby'] = 'desc'
# Search parameters takes a valid puppetdb query string
if request.GET.get('search', False):
if 'search' in request.session and (request.session['search']
== request.GET.get('search')):
pass
else:
if request.GET.get('search') == 'clear_rules':
request.session['sortfield'] = 'report_timestamp'
request.session['sortfieldby'] = 'desc'
request.session['page'] = 1
request.session['search'] = None
else:
request.session['page'] = 1
request.session['search'] = request.GET.get('search')
else:
if 'search' not in request.session:
request.session['sortfield'] = 'report_timestamp'
request.session['sortfieldby'] = 'desc'
request.session['page'] = 1
request.session['search'] = None
# Set offset
request.session['offset'] = (
request.session['limits'] *
request.session['page']) - request.session['limits']
except:
return HttpResponseBadRequest('Oh no! Your filters were invalid.')
# Valid sort field that the user can search agnaist.
sort_field = request.session['sortfield']
sort_field_order = request.session['sortfieldby']
page_num = request.session['page']
if request.session['search'] is not None:
node_params = {
'query': {
1: request.session['search']
},
}
else:
node_params = {
'query': {},
}
nodes_sort_fields = [
'certname', 'catalog_timestamp', 'report_timestamp', 'facts_timestamp'
]
if sort_field in nodes_sort_fields:
node_params['order_by'] = {
'order_field': {
'field': sort_field,
'order': sort_field_order,
},
}
if dl_csv is False:
node_params['limit'] = request.session['limits']
node_params['offset'] = request.session['offset']
node_params['include_total'] = 'true'
node_sort_fields = [
'certname', 'catalog_timestamp', 'report_timestamp', 'facts_timestamp'
]
try:
node_list, node_headers = puppetdb.api_get(
api_url=source_url,
cert=source_certs,
verify=source_verify,
path='/nodes',
api_version='v4',
params=puppetdb.mk_puppetdb_query(node_params, request),
)
except:
node_list = []
node_headers = dict()
node_headers['X-Records'] = 0
status_sort_fields = ['successes', 'failures', 'skips', 'noops']
# Create a filter part to limit the following API requests to data related to the node_list.
# Skipt the filter completely if a large number of nodes are shown as the query tends to fail.
node_filter = ''
if len(node_list) <= 100 and sort_field not in status_sort_fields:
node_filter = ', ["or"'
for n in node_list:
node_filter += ',["=","certname","%s"]' % n['certname']
node_filter += ']'
# Work out the number of pages from the xrecords response
# return fields that you can sort by
# for each node in the node_list, find out if the latest run has any failures
# v3/event-counts --data-urlencode query='["=","latest-report?",true]'
# --data-urlencode summarize-by='certname'
report_params = {
'query': {
1:
'["and" %s, ["=","latest_report?",true],["in", "certname",["extract", "certname",["select_nodes",["null?","deactivated",true]]]]]'
% node_filter,
},
'summarize_by': 'certname',
}
if sort_field in status_sort_fields:
report_params['order_by'] = {
'order_field': {
'field': sort_field,
'order': sort_field_order,
}
}
report_params['include_total'] = 'true'
report_list, report_headers = puppetdb.api_get(
api_url=source_url,
cert=source_certs,
verify=source_verify,
path='/event-counts',
params=puppetdb.mk_puppetdb_query(report_params, request),
api_version='v4',
)
else:
report_list = puppetdb.api_get(
api_url=source_url,
cert=source_certs,
verify=source_verify,
path='event-counts',
params=puppetdb.mk_puppetdb_query(report_params, request),
api_version='v4',
)
# number of results not depending on sort field.
xrecords = node_headers['X-Records']
total_results = xrecords
num_pages_wdec = float(xrecords) / request.session['limits']
num_pages_wodec = float("{:.0f}".format(num_pages_wdec))
if num_pages_wdec > num_pages_wodec:
num_pages = num_pages_wodec + 1
else:
num_pages = num_pages_wodec
# Converts lists of dicts to dicts.
report_dict = {item['subject']['title']: item
for item in report_list} # /events-count
if sort_field_order == 'desc':
rows = dictstatus(node_list,
None,
report_dict,
sortby=sort_field,
asc=True,
sort=False,
puppet_run_time=puppet_run_time,
format_time=False)
sort_field_order_opposite = 'asc'
elif sort_field_order == 'asc':
rows = dictstatus(node_list,
None,
report_dict,
sortby=sort_field,
asc=False,
sort=False,
puppet_run_time=puppet_run_time,
format_time=False)
sort_field_order_opposite = 'desc'
if dl_csv is True:
if rows is []:
pass
else:
# Generate a sequence of rows. The range is based on the maximum number of
# rows that can be handled by a single sheet in most spreadsheet
# applications.
include_facts = request.GET.get('include_facts', False)
csv_headers = [
'Certname', 'Latest Catalog', 'Latest Report', 'Latest Facts',
'Success', 'Noop', 'Failure', 'Skipped', 'Run Status'
]
if include_facts is not False:
merged_list_facts = []
facts = {}
for fact in include_facts.split(','):
# Sanitize the fact input from the user
fact = fact.strip()
# Add the fact name to the headers list
csv_headers.append(fact)
# build the params for each fact.
facts_params = {
'query': {
1: '["=","name","' + fact + '"]'
},
}
fact_list = puppetdb.api_get(
api_url=source_url,
cert=source_certs,
verify=source_verify,
path='facts',
params=puppetdb.mk_puppetdb_query(facts_params),
api_version='v4',
)
# Populate the facts dict with the facts we have retrieved
# Convert the fact list into a fact dict!
facts[fact] = {
item['certname']: item
for item in fact_list
}
i = 1
jobs = {}
# Add ID to each job so that it can be assembled in
# the same order after we recieve the job results
# We do this via jobs so that we can get faster results.
for node in rows:
jobs[i] = {
'id': i,
'include_facts': include_facts.split(','),
'node': node,
'facts': facts,
}
i += 1
csv_results = generate_csv(jobs)
rows = []
i = 1
# with the job results we can now recreate merged_list
# in the order we sent them.
while i <= len(csv_results):
rows.append(csv_results[i])
i += 1
# Insert the csv header to the top of the list.
rows.insert(0, csv_headers)
pseudo_buffer = Echo()
writer = csv.writer(pseudo_buffer)
response = StreamingHttpResponse(
(writer.writerow(row) for row in rows),
content_type="text/csv")
response[
'Content-Disposition'] = 'attachment; filename="puppetdata-%s.csv"' % (
datetime.datetime.now())
return response
if sort_field in status_sort_fields:
rows = rows[request.session['offset']:(request.session['limits'] +
request.session['offset'])]
"""
c_r_s* = current request sort
c_r_* = current req
r_s* = requests available
"""
context = {
'nodeList': rows,
'total_nodes': total_results,
'c_r_page': page_num,
'c_r_limit': request.session['limits'],
'r_sfield': valid_sort_fields,
'c_r_sfield': sort_field,
'r_sfieldby': ['asc', 'desc'],
'c_r_sfieldby': sort_field_order,
'c_r_sfieldby_o': sort_field_order_opposite,
'tot_pages': '{0:g}'.format(num_pages),
}
return HttpResponse(json.dumps(context), content_type="application/json")
def detailed_events(request, hashid=None):
context = {
'timezones': pytz.common_timezones,
'SOURCES': AVAILABLE_SOURCES
}
if request.method == 'GET':
if 'source' in request.GET:
source = request.GET.get('source')
set_server(request, source)
if request.method == 'POST':
request.session['django_timezone'] = request.POST['timezone']
return redirect(request.POST['return_url'])
source_url, source_certs, source_verify = get_server(request)
report_timestamp = request.GET.get('report_timestamp')
events_params = {
'query': {
1: '["=","report","' + hashid + '"]'
},
'order_by': {
'order_field': {
'field': 'timestamp',
'order': 'asc',
},
'query_field': {
'field': 'certname'
},
},
}
events_list = puppetdb.api_get(
api_url=source_url,
cert=source_certs,
verify=source_verify,
path='/events',
api_version='v4',
params=puppetdb.mk_puppetdb_query(events_params),
)
environment = ''
certname = ''
event_execution_times = []
sorted_events = None
last_event_time = None
last_event_title = None
run_end_time = None
if len(events_list) != 0:
single_event = events_list[0]
environment = single_event['environment']
certname = single_event['certname']
for event in events_list:
event_title = event['resource_title']
event_start_time = json_to_datetime(event['timestamp'])
if last_event_time is None and last_event_title is None:
last_event_time = event_start_time
last_event_title = event_title
run_end_time = json_to_datetime(event['run_end_time'])
continue
else:
event_exec_time = (event_start_time -
last_event_time).total_seconds()
add_event = (last_event_title, event_exec_time)
event_execution_times.append(add_event)
last_event_time = event_start_time
last_event_title = event_title
event_exec_time = (last_event_time - run_end_time).total_seconds()
add_event = [last_event_title, event_exec_time]
event_execution_times.append(add_event)
sorted_events = sorted(event_execution_times,
reverse=True,
key=lambda field: field[1])
if len(sorted_events) > 10:
sorted_events = sorted_events[:10]
else:
events_list = False
context['certname'] = certname
context['report_timestamp'] = report_timestamp
context['hashid'] = hashid
context['events_list'] = events_list
context['event_durations'] = sorted_events
context['environment'] = environment
return render(request, 'pano/detailed_events.html', context)
def dashboard_json(request):
context = {}
if request.method == 'GET':
if 'source' in request.GET:
source = request.GET.get('source')
set_server(request, source)
if request.method == 'POST':
request.session['django_timezone'] = request.POST['timezone']
return redirect(request.POST['return_url'])
source_url, source_certs, source_verify = get_server(request)
puppet_run_time = get_server(request, type='run_time')
dashboard_show = request.GET.get('show', 'recent')
events_params = {
'query':
{
1: '["and",["=","latest_report?",true],["in", "certname",["extract", "certname",["select_nodes",["null?","deactivated",true]]]]]'
},
'summarize_by': 'certname',
}
reports_params = {
'query':
{
1: '["and",["=","latest_report?",true],["in", "certname",["extract", "certname",["select_nodes",["null?","deactivated",true]]]]]'
}
}
nodes_params = {
'limit': 25,
'order_by': {
'order_field': {
'field': 'report_timestamp',
'order': 'desc',
},
'query_field': {'field': 'certname'},
},
}
jobs = {
'tot_resource': {
'url': source_url,
'certs': source_certs,
'verify': source_verify,
'id': 'tot_resource',
'path': 'mbeans/puppetlabs.puppetdb.query.population:type=default,name=num-resources',
},
'avg_resource': {
'url': source_url,
'certs': source_certs,
'verify': source_verify,
'id': 'avg_resource',
'path': 'mbeans/puppetlabs.puppetdb.query.population:type=default,name=avg-resources-per-node',
},
'all_nodes': {
'url': source_url,
'certs': source_certs,
'verify': source_verify,
'api_version': 'v4',
'id': 'all_nodes',
'path': '/nodes',
'request': request
},
'events': {
'url': source_url,
'certs': source_certs,
'verify': source_verify,
'id': 'event_counts',
'path': '/event-counts',
'api_version': 'v4',
'params': events_params,
'request': request
},
'reports': {
'url': source_url,
'certs': source_certs,
'verify': source_verify,
'api_version': 'v4',
'id': 'reports',
'path': '/reports',
'params': reports_params,
'request': request
},
'nodes': {
'url': source_url,
'certs': source_certs,
'verify': source_verify,
'api_version': 'v4',
'id': 'nodes',
'path': '/nodes',
'params': nodes_params,
'request': request
},
}
puppetdb_results = run_puppetdb_jobs(jobs)
# Assign vars from the completed jobs
# Number of results from all_nodes is our population.
puppet_population = len(puppetdb_results['all_nodes'])
# Total resources managed by puppet metric
total_resources = puppetdb_results['tot_resource']
# Average resource per node metric
avg_resource_node = puppetdb_results['avg_resource']
# Information about all active nodes in puppet
all_nodes_list = puppetdb_results['all_nodes']
# All available events for the latest puppet reports
event_list = puppetdb_results['event_counts']
event_dict = {item['subject']['title']: item for item in event_list}
# All of the latest reports
reports_list = puppetdb_results['reports']
reports_dict = {item['certname']: item for item in reports_list}
# 25 Nodes
node_list = puppetdb_results['nodes']
failed_list, changed_list, unreported_list, mismatch_list, pending_list = dictstatus(all_nodes_list,
reports_dict,
event_dict,
sort=True,
sortby='latestReport',
get_status='notall',
puppet_run_time=puppet_run_time)
pending_list = [x for x in pending_list if x not in unreported_list]
changed_list = [x for x in changed_list if
x not in unreported_list and x not in failed_list and x not in pending_list]
failed_list = [x for x in failed_list if x not in unreported_list]
unreported_list = [x for x in unreported_list if x not in failed_list]
node_unreported_count = len(unreported_list)
node_fail_count = len(failed_list)
node_change_count = len(changed_list)
node_off_timestamps_count = len(mismatch_list)
node_pending_count = len(pending_list)
if dashboard_show == 'recent':
merged_nodes_list = dictstatus(
node_list, reports_dict, event_dict, sort=False, get_status="all", puppet_run_time=puppet_run_time)
elif dashboard_show == 'failed':
merged_nodes_list = failed_list
elif dashboard_show == 'unreported':
merged_nodes_list = unreported_list
elif dashboard_show == 'changed':
merged_nodes_list = changed_list
elif dashboard_show == 'mismatch':
merged_nodes_list = mismatch_list
elif dashboard_show == 'pending':
merged_nodes_list = pending_list
else:
merged_nodes_list = dictstatus(
node_list, reports_dict, event_dict, sort=False, get_status="all", puppet_run_time=puppet_run_time)
context['node_list'] = merged_nodes_list
context['selected_view'] = dashboard_show
context['population'] = puppet_population
context['total_resource'] = total_resources['Value']
context['avg_resource'] = "{:.2f}".format(avg_resource_node['Value'])
context['failed_nodes'] = node_fail_count
context['changed_nodes'] = node_change_count
context['unreported_nodes'] = node_unreported_count
context['mismatching_timestamps'] = node_off_timestamps_count
context['pending_nodes'] = node_pending_count
return HttpResponse(json.dumps(context), content_type="application/json")
def dashboard_nodes_json(request):
context = {}
if request.method == 'GET':
if 'source' in request.GET:
source = request.GET.get('source')
set_server(request, source)
if request.method == 'POST':
request.session['django_timezone'] = request.POST['timezone']
return redirect(request.POST['return_url'])
source_url, source_certs, source_verify = get_server(request)
puppet_run_time = get_server(request, type='run_time')
# Dashboard to show nodes of "recent, failed, unreported or changed"
dashboard_show = request.GET.get('show', 'recent')
events_params = {
'query':
{
1: '["and",["=","latest_report?",true],["in", "certname",["extract", "certname",["select_nodes",["null?","deactivated",true]]]]]'
},
'summarize_by': 'certname',
}
all_nodes_params = {
'query':
{
1: '["and",["=","latest_report?",true],["in", "certname",["extract", "certname",["select_nodes",["null?","deactivated",true]]]]]'
},
}
reports_params = {
'query':
{
1: '["and",["=","latest_report?",true],["in", "certname",["extract", "certname",["select_nodes",["null?","deactivated",true]]]]]'
}
}
nodes_params = {
'limit': 25,
'order_by': {
'order_field': {
'field': 'report_timestamp',
'order': 'desc',
},
'query_field': {'field': 'certname'},
},
}
jobs = {
'all_nodes': {
'url': source_url,
'certs': source_certs,
'verify': source_verify,
'api_version': 'v4',
'id': 'all_nodes',
'path': '/nodes',
'request': request
},
'events': {
'url': source_url,
'certs': source_certs,
'verify': source_verify,
'id': 'event_counts',
'path': 'event-counts',
'api_version': 'v4',
'params': events_params,
'request': request
},
'nodes': {
'url': source_url,
'certs': source_certs,
'verify': source_verify,
'api_version': 'v4',
'id': 'nodes',
'path': '/nodes',
'params': nodes_params,
'request': request
},
'reports': {
'url': source_url,
'certs': source_certs,
'verify': source_verify,
'api_version': 'v4',
'id': 'reports',
'path': '/reports',
'params': reports_params,
'request': request
},
}
puppetdb_results = run_puppetdb_jobs(jobs)
# Information about all active nodes in puppet
all_nodes_list = puppetdb_results['all_nodes']
# All available events for the latest puppet reports
event_list = puppetdb_results['event_counts']
event_dict = {item['subject']['title']: item for item in event_list}
# All of the latest reports
reports_list = puppetdb_results['reports']
reports_dict = {item['certname']: item for item in reports_list}
# 25 Nodes
node_list = puppetdb_results['nodes']
failed_list, changed_list, unreported_list, mismatch_list, pending_list = dictstatus(all_nodes_list,
reports_dict,
event_dict,
sort=True,
sortby='latestReport',
get_status='notall',
puppet_run_time=puppet_run_time)
pending_list = [x for x in pending_list if x not in unreported_list]
changed_list = [x for x in changed_list if
x not in unreported_list and x not in failed_list and x not in pending_list]
failed_list = [x for x in failed_list if x not in unreported_list]
unreported_list = [x for x in unreported_list if x not in failed_list]
if dashboard_show == 'recent':
merged_nodes_list = dictstatus(
node_list, reports_dict, event_dict, sort=False, get_status="all", puppet_run_time=puppet_run_time)
elif dashboard_show == 'failed':
merged_nodes_list = failed_list
elif dashboard_show == 'unreported':
merged_nodes_list = unreported_list
elif dashboard_show == 'changed':
merged_nodes_list = changed_list
elif dashboard_show == 'mismatch':
merged_nodes_list = mismatch_list
elif dashboard_show == 'pending':
merged_nodes_list = pending_list
else:
merged_nodes_list = dictstatus(
node_list, reports_dict, event_dict, sort=False, get_status="all", puppet_run_time=puppet_run_time)
context['node_list'] = merged_nodes_list
context['selected_view'] = dashboard_show
return HttpResponse(json.dumps(context), content_type="application/json")
def get_file(request, certname, environment, rtitle, rtype, md5sum_from=None, md5sum_to=None, diff=False,
file_status='from'):
puppetdb_source, puppetdb_certs, puppetdb_verify = get_server(request=request)
filebucket_source, filebucket_certs, filebucket_verify, filebucket_show = get_server(request=request,
type='filebucket')
fileserver_source, fileserver_certs, fileserver_verify, fileserver_show = get_server(request=request,
type='fileserver')
# If Clientbucket is enabled continue else return False
def fetch_filebucket(url, method):
headers = {
'Accept': 's',
}
methods = {'get': requests.get,
'head': requests.head,
}
if method not in methods:
print('No can has method: %s' % method)
return False
resp = methods[method](url,
headers=headers,
verify=filebucket_verify,
cert=filebucket_certs)
if resp.status_code != 200:
return False
else:
return resp.text
def fetch_fileserver(url, method):
methods = {'get': requests.get,
}
if method not in methods:
print('No can has method: %s' % method)
return False
resp = methods[method](url,
verify=fileserver_verify,
cert=fileserver_certs)
if resp.status_code != 200:
return False
else:
return resp.text
def get_resource(certname, rtype, rtitle):
resource_params = {
'query':
{
'operator': 'and',
1: '["=", "certname", "' + certname + '"]',
2: '["=", "type", "' + rtype + '"]',
3: '["=", "title", "' + rtitle + '"]'
},
}
data = pdb_api_get(
api_url=puppetdb_source,
path='resources',
verify=puppetdb_verify,
cert=puppetdb_certs,
params=mk_puppetdb_query(resource_params, request))
if not data:
return False
else:
return data
if not filebucket_show or not fileserver_source:
return False
if file_status == 'both':
if md5sum_to and md5sum_from and certname and rtitle and rtype:
if diff:
# is the hash from puppetdb resource same as md5sum_to
hash_matches = False
md5sum_from = md5sum_from.replace('{md5}', '')
md5sum_to = md5sum_to.replace('{md5}', '')
from_url = filebucket_source + environment + '/file_bucket_file/md5/' + md5sum_from
to_url = filebucket_source + environment + '/file_bucket_file/md5/' + md5sum_to
if fetch_filebucket(from_url, 'head') is not False:
resource_from = fetch_filebucket(from_url, 'get')
else:
# Could not find old MD5 in Filebucket
return False
if fetch_filebucket(to_url, 'head') is not False:
resource_to = fetch_filebucket(to_url, 'get')
# Try puppetdb resources if not found in filebucket.
else:
resource_to = get_resource(certname=certname, rtype=rtype, rtitle=rtitle)
if resource_to is False:
# Could not find new file in Filebucket or as a PuppetDB Resource
return False
else:
resource_to = resource_to[0]
if 'content' in resource_to['parameters']:
resource_to = resource_to['parameters']['content']
hash_of_resource = get_hash(resource_to)
if hash_of_resource == md5sum_to:
# file from resource matches filebucket md5 hash
hash_matches = True
# Solve the viewing of source files by retrieving it from Puppetmaster
elif 'source' in resource_to['parameters'] and fileserver_show is True:
source_path = resource_to['parameters']['source']
if source_path.startswith('puppet://'):
# extract the path for the file
source_path = source_path.split('/') # ['puppet:', '', '', 'files', 'autofs', 'auto.home']
source_path = '/'.join(source_path[3:]) # Skip first 3 entries since they are not needed
# https://puppetmaster.example.com:8140/production/file_content/files/autofs/auto.home
url = fileserver_source + environment + '/file_content/' + source_path
resource_to = fetch_fileserver(url, 'get')
else:
return False
# now that we have come this far, we have both files.
# Lets differentiate the shit out of these files.
from_split_lines = resource_from.split('\n')
to_split_lines = resource_to.split('\n')
diff = difflib.unified_diff(from_split_lines, to_split_lines)
diff = ('\n'.join(list(diff))).split('\n')
return diff
else:
return False
else:
return False
if file_status == 'from' and md5sum_from:
md5sum = md5sum_from.replace('{md5}', '')
elif file_status == 'to' and md5sum_to:
md5sum = md5sum_to.replace('{md5}', '')
else:
return False
# Creates headers and url from the data we got
url_clientbucket = filebucket_source + environment + '/file_bucket_file/md5/' + md5sum
if fetch_filebucket(url_clientbucket, 'head') is False:
# Check if theres a resource available for the latest file available
if file_status == 'to':
resp_pdb = get_resource(certname=certname, rtype=rtype, rtitle=rtitle)
# we got the data lets give the user the good news.
if resp_pdb:
resource_data = resp_pdb[0]
if 'content' in resource_data['parameters']:
prepend_text = 'This file with MD5 %s was found in PuppetDB Resources.\n\n' % (
get_hash(resource_data['parameters']['content']))
return prepend_text + resource_data['parameters']['content']
elif 'source' in resource_data['parameters'] and fileserver_show is True:
source_path = resource_data['parameters']['source']
if source_path.startswith('puppet://'):
# extract the path for the file
source_path = source_path.split('/') # ['puppet:', '', '', 'files', 'autofs', 'auto.home']
source_path = '/'.join(source_path[3:]) # Skip first 3 entries since they are not needed
# https://puppetmaster.example.com:8140/production/file_content/files/autofs/auto.home
url = fileserver_source + environment + '/file_content/' + source_path
source_content = fetch_fileserver(url, 'get')
prepend_text = 'This file with MD5 %s was retrieved from the PuppetMaster Fileserver.\n\n' % (
get_hash(source_content))
return prepend_text + source_content
else:
return False
else:
return False
# the file can't be found as a resource and or fileserver support not enabled
else:
return False
# We probably don't want to search for resources if its the old file.
else:
return False
else:
filebucket_results = fetch_filebucket(url_clientbucket, 'get')
prepend_text = 'This file with MD5 %s was found in Filebucket.\n\n' % (md5sum)
return prepend_text + filebucket_results
def reports_json(request, certname=None):
source_url, source_certs, source_verify = get_server(request)
# Redirects to the events page if GET param latest is true..
context = {}
# Cur Page Number
if request.GET.get('page', False):
if request.session['report_page'] != int(request.GET.get('page', 1)):
request.session['report_page'] = int(request.GET.get('page', 1))
if request.session['report_page'] <= 0:
request.session['report_page'] = 1
else:
if 'report_page' not in request.session:
request.session['report_page'] = 1
if request.session['report_page'] <= 0:
offset = 0
else:
offset = (25 * request.session['report_page']) - 25
reports_params = {
'query': {
1: '["=","certname","' + certname + '"]'
},
'order_by': {
'order_field': {
'field': 'start_time',
'order': 'desc',
},
},
'limit': 25,
'include_total': 'true',
'offset': offset,
}
reports_list, headers = puppetdb.api_get(
api_url=source_url,
cert=source_certs,
verify=source_verify,
path='/reports',
api_version='v4',
params=puppetdb.mk_puppetdb_query(reports_params, request),
)
# Work out the number of pages from the xrecords response
xrecords = headers['X-Records']
num_pages_wdec = float(xrecords) / 25
num_pages_wodec = float("{:.0f}".format(num_pages_wdec))
if num_pages_wdec > num_pages_wodec:
num_pages = num_pages_wodec + 1
else:
num_pages = num_pages_wodec
report_status = []
for report in reports_list:
found_report = False
events_params = {
'query': {
1: '["=","report","' + report['hash'] + '"]'
},
'summarize_by': 'certname',
}
eventcount_list = puppetdb.api_get(
path='event-counts',
api_url=source_url,
api_version='v4',
params=puppetdb.mk_puppetdb_query(events_params, request),
)
# Make list of the results
for event in eventcount_list:
if event['subject']['title'] == report['certname']:
found_report = True
report_status.append({
'hash':
report['hash'],
'certname':
report['certname'],
'environment':
report['environment'],
'start_time':
filters.date(
localtime(json_to_datetime(report['start_time'])),
'Y-m-d H:i:s'),
'end_time':
filters.date(
localtime(json_to_datetime(report['end_time'])),
'Y-m-d H:i:s'),
'events_successes':
event['successes'],
'events_noops':
event['noops'],
'events_failures':
event['failures'],
'events_skipped':
event['skips'],
'report_status':
report['status'],
'config_version':
report['configuration_version'],
'run_duration':
"{0:.0f}".format(
(json_to_datetime(report['end_time']) -
json_to_datetime(
report['start_time'])).total_seconds())
})
break
if found_report is False:
report_status.append({
'hash':
report['hash'],
'certname':
report['certname'],
'environment':
report['environment'],
'start_time':
filters.date(localtime(json_to_datetime(report['start_time'])),
'Y-m-d H:i:s'),
'end_time':
filters.date(localtime(json_to_datetime(report['end_time'])),
'Y-m-d H:i:s'),
'events_successes':
0,
'events_noops':
0,
'events_failures':
0,
'events_skipped':
0,
'report_status':
report['status'],
'config_version':
report['configuration_version'],
'run_duration':
"{0:.0f}".format(
(json_to_datetime(report['end_time']) -
json_to_datetime(report['start_time'])).total_seconds())
})
context['certname'] = certname
context['reports_list'] = report_status
context['curr_page'] = request.session['report_page']
context['tot_pages'] = "{:.0f}".format(num_pages)
return HttpResponse(json.dumps(context), content_type="application/json")
def facts_json(request):
context = {}
if request.method == 'GET':
if 'source' in request.GET:
source = request.GET.get('source')
set_server(request, source)
if request.method == 'POST':
request.session['django_timezone'] = request.POST['timezone']
return redirect(request.POST['return_url'])
source_url, source_certs, source_verify = get_server(request)
certname = None
facts = None
if 'certname' in request.GET:
certname = request.GET.get('certname')
if 'facts' in request.GET:
facts = request.GET.get('facts').split(',')
if not certname:
context['error'] = 'Certname not specified.'
return HttpResponse(json.dumps(context))
if facts:
# validate string for illegal chars
fact_query = list()
for fact in facts:
fact = fact.strip()
# Match for characters that are not a-Z or 0-9 or _
# if theres a match illegal chars exist...
regex = re.compile(r'[^aA-zZ0-9_]')
matches = regex.findall(fact)
if matches:
context['error'] = 'Illegal characters found in facts list. '
context['error'] += 'Facts must not match anything withddd this regex <[^aA-zZ0-9_]>.'
return HttpResponse(json.dumps(context))
fact_query.append('["=","name","' + fact + '"]')
fact_query = ','.join(fact_query)
facts_params = {
'query':
{
1: '["and",["=","certname","' + certname + '"],["or",' + fact_query + ']]'
},
'order-by':
{
'order_field':
{
'field': 'name',
'order': 'asc',
}
}
}
else:
facts_params = {
'query':
{
1: '["=","certname","' + certname + '"]'
},
'order_by':
{
'order_field':
{
'field': 'name',
'order': 'asc',
}
}
}
facts_list = puppetdb.api_get(
api_url=source_url,
cert=source_certs,
verify=source_verify,
path='facts',
params=puppetdb.mk_puppetdb_query(
facts_params, request),
)
context['certname'] = certname
context['facts_list'] = facts_list
return HttpResponse(json.dumps(context, indent=2), content_type="application/json")
def analytics(request):
context = {
'timezones': pytz.common_timezones,
'SOURCES': AVAILABLE_SOURCES
}
if request.method == 'GET':
if 'source' in request.GET:
source = request.GET.get('source')
set_server(request, source)
if request.method == 'POST':
request.session['django_timezone'] = request.POST['timezone']
return redirect(request.POST['return_url'])
source_url, source_certs, source_verify = get_server(request)
events_class_params = {
'query': {
1:
'["and",["=","latest_report?",true],["in","certname",["extract","certname",["select_nodes",["null?","deactivated",true]]]]]'
},
'summarize_by': 'containing_class',
}
events_resource_params = {
'query': {
1:
'["and",["=","latest_report?",true],["in","certname",["extract","certname",["select_nodes",["null?","deactivated",true]]]]]'
},
'summarize_by': 'resource',
}
events_status_params = {
'query': {
1:
'["and",["=","latest_report?",true],["in","certname",["extract","certname",["select_nodes",["null?","deactivated",true]]]]]'
},
'summarize_by': 'resource',
}
reports_runavg_params = {
'limit': 100,
'order_by': {
'order_field': {
'field': 'receive_time',
'order': 'desc',
},
'query_field': {
'field': 'certname'
},
},
}
jobs = {
'events_class_list': {
'url': source_url,
'certs': source_certs,
'verify': source_verify,
'id': 'events_class_list',
'path': '/event-counts',
'api_version': 'v4',
'params': events_class_params,
'request': request
},
'events_resource_list': {
'url': source_url,
'certs': source_certs,
'verify': source_verify,
'id': 'events_resource_list',
'path': '/event-counts',
'api_version': 'v4',
'params': events_resource_params,
'request': request
},
'events_status_list': {
'url': source_url,
'certs': source_certs,
'verify': source_verify,
'id': 'events_status_list',
'path': '/aggregate-event-counts',
'api_version': 'v4',
'params': events_status_params,
'request': request
},
'reports_run_avg': {
'url': source_url,
'certs': source_certs,
'verify': source_verify,
'id': 'reports_run_avg',
'path': '/reports',
'api_version': 'v4',
'params': reports_runavg_params,
'request': request
},
}
job_results = run_puppetdb_jobs(jobs, 4)
reports_run_avg = job_results['reports_run_avg']
events_class_list = job_results['events_class_list']
events_resource_list = job_results['events_resource_list']
events_status_list = job_results['events_status_list']
num_runs_avg = len(reports_run_avg)
run_avg_times = []
avg_run_time = 0
for report in reports_run_avg:
run_time = "{0:.0f}".format(
(json_to_datetime(report['end_time']) -
json_to_datetime(report['start_time'])).total_seconds())
avg_run_time += int(run_time)
run_avg_times.append(run_time)
if num_runs_avg != 0:
avg_run_time = "{0:.0f}".format(avg_run_time / num_runs_avg)
else:
avg_run_time = 0
class_event_results = []
class_resource_results = []
class_status_results = []
for item in events_class_list:
class_name = item['subject']['title']
class_total = item['skips'] + item['failures'] + item['noops'] + item[
'successes']
class_event_results.append((class_name, class_total))
for item in events_resource_list:
class_name = item['subject']['type']
class_total = item['skips'] + item['failures'] + item['noops'] + item[
'successes']
class_resource_results.append((class_name, class_total))
print(events_status_list)
if events_status_list:
for status, value in events_status_list[0].items():
print(status, value)
if value is 0 or status == 'total' or status == 'summarize_by':
continue
class_status_results.append((status, value))
context['class_events'] = class_event_results
context['class_status'] = class_status_results
context['resource_events'] = class_resource_results
context['run_times'] = run_avg_times
context['run_num'] = num_runs_avg
context['run_avg'] = avg_run_time
return render(request, 'pano/analytics/analytics.html', context)
def nodes_json(request):
if request.method == 'GET':
if 'source' in request.GET:
source = request.GET.get('source')
set_server(request, source)
if request.method == 'POST':
request.session['django_timezone'] = request.POST['timezone']
return redirect(request.POST['return_url'])
source_url, source_certs, source_verify = get_server(request)
valid_sort_fields = (
'certname',
'catalog_timestamp',
'report_timestamp',
'facts_timestamp',
'successes',
'noops',
'failures',
'skips')
try:
# If user requested to download csv formatted file. Default value is False
dl_csv = request.GET.get('dl_csv', False)
if dl_csv == 'true':
dl_csv = True
else:
dl_csv = False
# Add limits to session
if request.GET.get('limits', False):
if request.session['limits'] != int(request.GET.get('limits', 50)):
request.session['limits'] = int(request.GET.get('limits', 50))
if request.session['limits'] <= 0:
request.session['limits'] = 50
else:
if 'limits' not in request.session:
request.session['limits'] = 50
# Cur Page Number
if request.GET.get('page', False):
if request.session['page'] != int(request.GET.get('page', 1)):
request.session['page'] = int(request.GET.get('page', 1))
if request.session['page'] <= 0:
request.session['page'] = 1
else:
if 'page' not in request.session:
request.session['page'] = 1
# Cur sort field
if request.GET.get('sortfield', False):
if request.session['sortfield'] != request.GET.get('sortfield'):
request.session['sortfield'] = request.GET.get('sortfield')
if request.session['sortfield'] not in valid_sort_fields:
request.session['sortfield'] = 'report_timestamp'
else:
if 'sortfield' not in request.session:
request.session['sortfield'] = 'report_timestamp'
# Cur sort order
if request.GET.get('sortfieldby', False):
avail_sortorder = ['asc', 'desc']
if request.session['sortfieldby'] != request.GET.get('sortfieldby'):
request.session['sortfieldby'] = request.GET.get('sortfieldby')
if request.session['sortfieldby'] not in avail_sortorder:
request.session['sortfieldby'] = 'desc'
else:
if 'sortfieldby' not in request.session:
request.session['sortfieldby'] = 'desc'
# Search parameters takes a valid puppetdb query string
if request.GET.get('search', False):
if 'search' in request.session and (request.session['search'] == request.GET.get('search')):
pass
else:
if request.GET.get('search') == 'clear_rules':
request.session['sortfield'] = 'report_timestamp'
request.session['sortfieldby'] = 'desc'
request.session['page'] = 1
request.session['search'] = None
else:
request.session['page'] = 1
request.session['search'] = request.GET.get('search')
else:
if 'search' not in request.session:
request.session['sortfield'] = 'report_timestamp'
request.session['sortfieldby'] = 'desc'
request.session['page'] = 1
request.session['search'] = None
# Set offset
request.session['offset'] = (request.session['limits'] * request.session['page']) - request.session[
'limits']
except:
return HttpResponseBadRequest('Oh no! Your filters were invalid.')
# Valid sort field that the user can search agnaist.
sort_field = request.session['sortfield']
sort_field_order = request.session['sortfieldby']
page_num = request.session['page']
if request.session['search'] is not None:
node_params = {
'query':
{
1: request.session['search']
},
}
else:
node_params = {
'query': {},
}
nodes_sort_fields = ['certname', 'catalog_timestamp', 'report_timestamp', 'facts_timestamp']
if sort_field in nodes_sort_fields:
node_params['order_by'] = {
'order_field':
{
'field': sort_field,
'order': sort_field_order,
},
}
if dl_csv is False:
node_params['limit'] = request.session['limits']
node_params['offset'] = request.session['offset']
node_params['include_total'] = 'true'
else:
node_params['order_by'] = {
'order_field':
{
'field': 'report_timestamp',
'order': 'desc',
},
}
node_sort_fields = ['certname', 'catalog_timestamp', 'report_timestamp', 'facts_timestamp']
if sort_field in node_sort_fields:
try:
node_list, node_headers = puppetdb.api_get(
api_url=source_url,
cert=source_certs,
verify=source_verify,
path='/nodes',
api_version='v4',
params=puppetdb.mk_puppetdb_query(
node_params, request),
)
except:
node_list = []
node_headers = dict()
node_headers['X-Records'] = 0
else:
node_list = puppetdb.api_get(
api_url=source_url,
cert=source_certs,
verify=source_verify,
path='/nodes',
api_version='v4',
params=puppetdb.mk_puppetdb_query(
node_params, request),
)
# Work out the number of pages from the xrecords response
# return fields that you can sort by
# for each node in the node_list, find out if the latest run has any failures
# v3/event-counts --data-urlencode query='["=","latest-report?",true]'
# --data-urlencode summarize-by='certname'
report_params = {
'query':
{
1: '["and",["=","latest_report?",true],["in", "certname",["extract", "certname",["select_nodes",["null?","deactivated",true]]]]]'
},
'summarize_by': 'certname',
}
status_sort_fields = ['successes', 'failures', 'skips', 'noops']
report_status_params = {
'query':
{
1: '["and",["=","latest_report?",true],["in", "certname",["extract", "certname",["select_nodes",["null?","deactivated",true]]]]]'
}
}
report_status_list = puppetdb.api_get(
api_url=source_url,
cert=source_certs,
verify=source_verify,
path='/reports',
params=puppetdb.mk_puppetdb_query(report_status_params, request),
api_version='v4',
)
if sort_field in status_sort_fields:
if request.session['search'] is not None:
report_params['query'] = {'operator': 'and',
1: request.session['search'],
2: '["=","latest_report?",true]',
3: '["in", "certname",["extract", "certname",["select_nodes",["null?","deactivated",true]]]]',
}
report_params['order_by'] = {
'order_field':
{
'field': sort_field,
'order': sort_field_order,
}
}
report_params['include_total'] = 'true'
# Don't limit results if its CSV
if dl_csv is False:
report_params['limit'] = request.session['limits']
report_params['offset'] = request.session['offset']
report_list, report_headers = puppetdb.api_get(
api_url=source_url,
cert=source_certs,
verify=source_verify,
path='/event-counts',
params=puppetdb.mk_puppetdb_query(report_params, request),
api_version='v4',
)
else:
report_list = puppetdb.api_get(
api_url=source_url,
cert=source_certs,
verify=source_verify,
path='event-counts',
params=puppetdb.mk_puppetdb_query(report_params, request),
api_version='v4',
)
# number of results depending on sort field.
if sort_field in status_sort_fields:
xrecords = report_headers['X-Records']
total_results = xrecords
elif sort_field in nodes_sort_fields:
xrecords = node_headers['X-Records']
total_results = xrecords
num_pages_wdec = float(xrecords) / request.session['limits']
num_pages_wodec = float("{:.0f}".format(num_pages_wdec))
if num_pages_wdec > num_pages_wodec:
num_pages = num_pages_wodec + 1
else:
num_pages = num_pages_wodec
# Converts lists of dicts to dicts.
status_dict = {item['certname']: item for item in report_status_list}
report_dict = {item['subject']['title']: item for item in report_list}
if sort_field_order == 'desc':
rows = dictstatus(
node_list, status_dict, report_dict, sortby=sort_field, asc=True, sort=False)
sort_field_order_opposite = 'asc'
elif sort_field_order == 'asc':
rows = dictstatus(
node_list, status_dict, report_dict, sortby=sort_field, asc=False, sort=False)
sort_field_order_opposite = 'desc'
if dl_csv is True:
if rows is []:
pass
else:
# Generate a sequence of rows. The range is based on the maximum number of
# rows that can be handled by a single sheet in most spreadsheet
# applications.
include_facts = request.GET.get('include_facts', False)
csv_headers = ['Certname',
'Latest Catalog',
'Latest Report',
'Latest Facts',
'Success',
'Noop',
'Failure',
'Skipped',
'Run Status']
if include_facts is not False:
merged_list_facts = []
facts = {}
for fact in include_facts.split(','):
# Sanitize the fact input from the user
fact = fact.strip()
# Add the fact name to the headers list
csv_headers.append(fact)
# build the params for each fact.
facts_params = {
'query':
{
1: '["=","name","' + fact + '"]'
},
}
fact_list = puppetdb.api_get(
api_url=source_url,
cert=source_certs,
verify=source_verify,
path='facts',
params=puppetdb.mk_puppetdb_query(facts_params),
api_version='v4',
)
# Populate the facts dict with the facts we have retrieved
# Convert the fact list into a fact dict!
facts[fact] = {item['certname']: item for item in fact_list}
i = 1
jobs = {}
# Add ID to each job so that it can be assembled in
# the same order after we recieve the job results
# We do this via jobs so that we can get faster results.
for node in rows:
jobs[i] = {
'id': i,
'include_facts': include_facts.split(','),
'node': node,
'facts': facts,
}
i += 1
csv_results = generate_csv(jobs)
rows = []
i = 1
# with the job results we can now recreate merged_list
# in the order we sent them.
while i <= len(csv_results):
rows.append(csv_results[i])
i += 1
# Insert the csv header to the top of the list.
rows.insert(0, csv_headers)
pseudo_buffer = Echo()
writer = csv.writer(pseudo_buffer)
response = StreamingHttpResponse((writer.writerow(row) for row in rows),
content_type="text/csv")
response['Content-Disposition'] = 'attachment; filename="puppetdata-%s.csv"' % (datetime.datetime.now())
return response
"""
c_r_s* = current request sort
c_r_* = current req
r_s* = requests available
"""
context = {
'nodeList': rows,
'total_nodes': total_results,
'c_r_page': page_num,
'c_r_limit': request.session['limits'],
'r_sfield': valid_sort_fields,
'c_r_sfield': sort_field,
'r_sfieldby': ['asc', 'desc'],
'c_r_sfieldby': sort_field_order,
'c_r_sfieldby_o': sort_field_order_opposite,
'tot_pages': '{0:g}'.format(num_pages),
}
return HttpResponse(json.dumps(context), content_type="application/json")
def facts_json(request):
context = {}
if request.method == 'GET':
if 'source' in request.GET:
source = request.GET.get('source')
set_server(request, source)
if request.method == 'POST':
request.session['django_timezone'] = request.POST['timezone']
return redirect(request.POST['return_url'])
source_url, source_certs, source_verify = get_server(request)
certname = None
facts = None
if 'certname' in request.GET:
certname = request.GET.get('certname')
if 'facts' in request.GET:
facts = request.GET.get('facts').split(',')
if not certname:
context['error'] = 'Certname not specified.'
return HttpResponse(json.dumps(context))
if facts:
# validate string for illegal chars
fact_query = list()
for fact in facts:
fact = fact.strip()
# Match for characters that are not a-Z or 0-9 or _
# if theres a match illegal chars exist...
regex = re.compile(r'[^aA-zZ0-9_]')
matches = regex.findall(fact)
if matches:
context['error'] = 'Illegal characters found in facts list. '
context['error'] += 'Facts must not match anything withddd this regex <[^aA-zZ0-9_]>.'
return HttpResponse(json.dumps(context))
fact_query.append('["=","name","' + fact + '"]')
fact_query = ','.join(fact_query)
facts_params = {
'query':
{
1: '["and",["=","certname","' + certname + '"],["or",' + fact_query + ']]'
},
'order-by':
{
'order_field':
{
'field': 'name',
'order': 'asc',
}
}
}
else:
facts_params = {
'query':
{
1: '["=","certname","' + certname + '"]'
},
'order_by':
{
'order-field':
{
'field': 'name',
'order': 'asc',
}
}
}
facts_list = puppetdb.api_get(
api_url=source_url,
cert=source_certs,
verify=source_verify,
path='facts',
params=puppetdb.mk_puppetdb_query(
facts_params, request),
)
context['certname'] = certname
context['facts_list'] = facts_list
return HttpResponse(json.dumps(context), content_type="application/json")
def analytics(request):
context = {'timezones': pytz.common_timezones,
'SOURCES': AVAILABLE_SOURCES}
if request.method == 'GET':
if 'source' in request.GET:
source = request.GET.get('source')
set_server(request, source)
if request.method == 'POST':
request.session['django_timezone'] = request.POST['timezone']
return redirect(request.POST['return_url'])
source_url, source_certs, source_verify = get_server(request)
events_class_params = {
'query':
{
1: '["and",["=","latest_report?",true],["in","certname",["extract","certname",["select_nodes",["null?","deactivated",true]]]]]'
},
'summarize_by': 'containing_class',
}
events_resource_params = {
'query':
{
1: '["and",["=","latest_report?",true],["in","certname",["extract","certname",["select_nodes",["null?","deactivated",true]]]]]'
},
'summarize_by': 'resource',
}
events_status_params = {
'query':
{
1: '["and",["=","latest_report?",true],["in","certname",["extract","certname",["select_nodes",["null?","deactivated",true]]]]]'
},
'summarize_by': 'resource',
}
reports_runavg_params = {
'limit': 100,
'order_by': {
'order_field': {
'field': 'receive_time',
'order': 'desc',
},
'query_field': {'field': 'certname'},
},
}
jobs = {
'events_class_list': {
'url': source_url,
'certs': source_certs,
'verify': source_verify,
'id': 'events_class_list',
'path': '/event-counts',
'api_version': 'v4',
'params': events_class_params,
'request': request
},
'events_resource_list': {
'url': source_url,
'certs': source_certs,
'verify': source_verify,
'id': 'events_resource_list',
'path': '/event-counts',
'api_version': 'v4',
'params': events_resource_params,
'request': request
},
'events_status_list': {
'url': source_url,
'certs': source_certs,
'verify': source_verify,
'id': 'events_status_list',
'path': '/aggregate-event-counts',
'api_version': 'v4',
'params': events_status_params,
'request': request
},
'reports_run_avg': {
'url': source_url,
'certs': source_certs,
'verify': source_verify,
'id': 'reports_run_avg',
'path': '/reports',
'api_version': 'v4',
'params': reports_runavg_params,
'request': request
},
}
job_results = run_puppetdb_jobs(jobs, 4)
reports_run_avg = job_results['reports_run_avg']
events_class_list = job_results['events_class_list']
events_resource_list = job_results['events_resource_list']
events_status_list = job_results['events_status_list']
num_runs_avg = len(reports_run_avg)
run_avg_times = []
avg_run_time = 0
for report in reports_run_avg:
run_time = "{0:.0f}".format(
(json_to_datetime(report['end_time']) - json_to_datetime(report['start_time'])).total_seconds())
avg_run_time += int(run_time)
run_avg_times.append(run_time)
if num_runs_avg != 0:
avg_run_time = "{0:.0f}".format(avg_run_time / num_runs_avg)
else:
avg_run_time = 0
class_event_results = []
class_resource_results = []
class_status_results = []
for item in events_class_list:
class_name = item['subject']['title']
class_total = item['skips'] + item['failures'] + item['noops'] + item['successes']
class_event_results.append((class_name, class_total))
for item in events_resource_list:
class_name = item['subject']['type']
class_total = item['skips'] + item['failures'] + item['noops'] + item['successes']
class_resource_results.append((class_name, class_total))
if events_status_list:
for status, value in events_status_list[0].items():
if value is 0 or status == 'total' or status == 'summarize_by':
continue
class_status_results.append((status, value))
context['class_events'] = class_event_results
context['class_status'] = class_status_results
context['resource_events'] = class_resource_results
context['run_times'] = run_avg_times
context['run_num'] = num_runs_avg
context['run_avg'] = avg_run_time
return render(request, 'pano/analytics/analytics.html', context)
def catalogue_json(request, certname=None):
context = dict()
data = dict()
if not certname:
context['error'] = 'Must specify certname.'
return HttpResponse(json.dumps(context), content_type="application/json")
source_url, source_certs, source_verify = get_server(request)
# Redirects to the events page if GET param latest is true..
show = request.GET.get('show', 'edges')
save_catalog = request.GET.get('save', 'false')
catalogue_params = {}
path = '/catalogs/%s' % certname
catalogue = puppetdb.api_get(
path=path,
api_url=source_url,
verify=source_verify,
cert=source_certs,
api_version='v4',
params=puppetdb.mk_puppetdb_query(catalogue_params, request),
)
if 'error' not in catalogue and save_catalog == 'true':
report_param = {
'query':
{
'operator': 'and',
1: '["=","latest_report?",true]',
2: '["=","certname","%s"]' % certname
}
}
report_url = '/reports'
latest_report = puppetdb.api_get(
path=report_url,
api_url=source_url,
verify=source_verify,
cert=source_certs,
api_version='v4',
params=puppetdb.mk_puppetdb_query(report_param, request),
)
report_hash = latest_report[0]['hash']
catalogue_hash = catalogue['hash']
catalogue_timestamp = catalogue['producer_timestamp']
try:
saved_catalogue = SavedCatalogs.objects.get(hostname=certname, catalogue_id=catalogue_hash)
if saved_catalogue.linked_report != report_hash:
# Grab the linked report from the result set.
old_linked_report = saved_catalogue.linked_report
# Update the data.
saved_catalogue.linked_report = report_hash
saved_catalogue.timestamp = catalogue_timestamp
# Save the new data.
saved_catalogue.save()
data['success'] = 'Catalogue hash updated.'
data['certname'] = certname
data['old_linked_report'] = old_linked_report
data['new_linked_report'] = report_hash
return HttpResponse(json.dumps(data, indent=2), content_type='application/json')
else:
data['error'] = 'Catalogue hash already exists.'
data['certname'] = certname
data['catalogue_hash'] = catalogue_hash
data['linked_report'] = saved_catalogue.linked_report
return HttpResponseBadRequest(json.dumps(data, indent=2), content_type='application/json')
except SavedCatalogs.DoesNotExist:
# since we couldnt find it in the db its safe to asusme that we can create it!
SavedCatalogs.objects.create(hostname=certname,
catalogue_id=catalogue_hash,
linked_report=report_hash,
timestamp=catalogue_timestamp,
catalogue=json.dumps(catalogue))
data['success'] = 'Saved catalogue.'
data['certname'] = certname
data['catalogue_hash'] = catalogue_hash
data['linked_report'] = report_hash
return HttpResponse(json.dumps(data, indent=2), content_type='application/json')
if show == 'edges':
data['data'] = catalogue['edges']['data']
elif show == 'resources':
data['data'] = catalogue['resources']['data']
else:
data['data'] = catalogue
return HttpResponse(json.dumps(data, indent=2), content_type='application/json')
def reports_json(request, certname=None):
source_url, source_certs, source_verify = get_server(request)
# Redirects to the events page if GET param latest is true..
context = {}
# Cur Page Number
if request.GET.get('page', False):
if request.session['report_page'] != int(request.GET.get('page', 1)):
request.session['report_page'] = int(request.GET.get('page', 1))
if request.session['report_page'] <= 0:
request.session['report_page'] = 1
else:
if 'report_page' not in request.session:
request.session['report_page'] = 1
if request.session['report_page'] <= 0:
offset = 0
else:
offset = (25 * request.session['report_page']) - 25
reports_params = {
'query':
{
1: '["=","certname","' + certname + '"]'
},
'order_by':
{
'order_field':
{
'field': 'start_time',
'order': 'desc',
},
},
'limit': 25,
'include_total': 'true',
'offset': offset,
}
reports_list, headers = puppetdb.api_get(
api_url=source_url,
cert=source_certs,
verify=source_verify,
path='/reports',
api_version='v4',
params=puppetdb.mk_puppetdb_query(
reports_params, request),
)
# Work out the number of pages from the xrecords response
xrecords = headers['X-Records']
num_pages_wdec = float(xrecords) / 25
num_pages_wodec = float("{:.0f}".format(num_pages_wdec))
if num_pages_wdec > num_pages_wodec:
num_pages = num_pages_wodec + 1
else:
num_pages = num_pages_wodec
report_status = []
for report in reports_list:
found_report = False
events_params = {
'query':
{
1: '["=","report","' + report['hash'] + '"]'
},
'summarize_by': 'certname',
}
eventcount_list = puppetdb.api_get(
path='event-counts',
api_url=source_url,
api_version='v4',
params=puppetdb.mk_puppetdb_query(events_params, request),
)
# Make list of the results
for event in eventcount_list:
if event['subject']['title'] == report['certname']:
found_report = True
report_status.append({
'hash': report['hash'],
'certname': report['certname'],
'environment': report['environment'],
'start_time': filters.date(localtime(json_to_datetime(report['start_time'])), 'Y-m-d H:i:s'),
'end_time': filters.date(localtime(json_to_datetime(report['end_time'])), 'Y-m-d H:i:s'),
'events_successes': event['successes'],
'events_noops': event['noops'],
'events_failures': event['failures'],
'events_skipped': event['skips'],
'report_status': report['status'],
'config_version': report['configuration_version'],
'run_duration': "{0:.0f}".format(
(json_to_datetime(report['end_time']) - json_to_datetime(report['start_time'])).total_seconds())
})
break
if found_report is False:
report_status.append({
'hash': report['hash'],
'certname': report['certname'],
'environment': report['environment'],
'start_time': filters.date(localtime(json_to_datetime(report['start_time'])), 'Y-m-d H:i:s'),
'end_time': filters.date(localtime(json_to_datetime(report['end_time'])), 'Y-m-d H:i:s'),
'events_successes': 0,
'events_noops': 0,
'events_failures': 0,
'events_skipped': 0,
'report_status': report['status'],
'config_version': report['configuration_version'],
'run_duration': "{0:.0f}".format(
(json_to_datetime(report['end_time']) - json_to_datetime(report['start_time'])).total_seconds())
})
context['certname'] = certname
context['reports_list'] = report_status
context['curr_page'] = request.session['report_page']
context['tot_pages'] = "{:.0f}".format(num_pages)
return HttpResponse(json.dumps(context), content_type="application/json")
def detailed_events(request, hashid=None):
context = {'timezones': pytz.common_timezones,
'SOURCES': AVAILABLE_SOURCES}
if request.method == 'GET':
if 'source' in request.GET:
source = request.GET.get('source')
set_server(request, source)
if request.method == 'POST':
request.session['django_timezone'] = request.POST['timezone']
return redirect(request.POST['return_url'])
source_url, source_certs, source_verify = get_server(request)
report_timestamp = request.GET.get('report_timestamp')
events_params = {
'query':
{
1: '["=","report","' + hashid + '"]'
},
'order_by':
{
'order_field':
{
'field': 'timestamp',
'order': 'asc',
},
'query_field': {'field': 'certname'},
},
}
events_list = puppetdb.api_get(
api_url=source_url,
cert=source_certs,
verify=source_verify,
path='/events',
api_version='v4',
params=puppetdb.mk_puppetdb_query(events_params),
)
environment = ''
certname = ''
event_execution_times = []
sorted_events = None
last_event_time = None
last_event_title = None
run_end_time = None
if len(events_list) != 0:
single_event = events_list[0]
environment = single_event['environment']
certname = single_event['certname']
for event in events_list:
event_title = event['resource_title']
event_start_time = json_to_datetime(event['timestamp'])
if last_event_time is None and last_event_title is None:
last_event_time = event_start_time
last_event_title = event_title
run_end_time = json_to_datetime(event['run_end_time'])
continue
else:
event_exec_time = (event_start_time - last_event_time).total_seconds()
add_event = (last_event_title, event_exec_time)
event_execution_times.append(add_event)
last_event_time = event_start_time
last_event_title = event_title
event_exec_time = (last_event_time - run_end_time).total_seconds()
add_event = [last_event_title, event_exec_time]
event_execution_times.append(add_event)
sorted_events = sorted(event_execution_times, reverse=True, key=lambda field: field[1])
if len(sorted_events) > 10:
sorted_events = sorted_events[:10]
else:
events_list = False
context['certname'] = certname
context['report_timestamp'] = report_timestamp
context['hashid'] = hashid
context['events_list'] = events_list
context['event_durations'] = sorted_events
context['environment'] = environment
return render(request, 'pano/detailed_events.html', context)
def catalogue_compare_json(request, certname1=None, certname2=None):
source_url, source_certs, source_verify = get_server(request)
show = request.GET.get('show', 'edges')
data = dict()
certname1_hash = request.GET.get('certname1_hash', False)
certname2_hash = request.GET.get('certname2_hash', False)
cata_params = dict()
if certname1_hash:
try:
certname1_result = SavedCatalogs.objects.get(hostname=certname1, catalogue_id=certname1_hash)
certname1_data = json.loads(certname1_result.catalogue)[show]['data']
except SavedCatalogs.DoesNotExist:
data['error'] = 'Catalogue hash not found in DB.'
data['hash_not_found'] = certname1_hash
data['certname'] = certname1
return HttpResponseBadRequest(json.dumps(data, indent=2), content_type="application/json")
else:
certname1_result = puppetdb.api_get(
path='/catalogs/%s' % certname1,
api_url=source_url,
api_version='v4',
params=puppetdb.mk_puppetdb_query(cata_params, request),
)
certname1_data = certname1_result[show]['data']
if certname2_hash:
try:
certname2_result = SavedCatalogs.objects.get(hostname=certname2, catalogue_id=certname2_hash)
certname2_data = json.loads(certname2_result.catalogue)[show]['data']
except SavedCatalogs.DoesNotExist:
data['error'] = 'Catalogue hash not found in DB.'
data['hash_not_found'] = certname2_hash
data['certname'] = certname2
return HttpResponseBadRequest(json.dumps(data, indent=2), content_type="application/json")
else:
certname2_result = puppetdb.api_get(
path='/catalogs/%s' % certname2,
api_url=source_url,
api_version='v4',
params=puppetdb.mk_puppetdb_query(cata_params, request),
)
certname2_data = certname2_result[show]['data']
node_for = dict()
node_agn = dict()
if show == "edges":
for edge in certname1_data:
# remove the certname tag.
try:
edge.pop('certname')
except:
pass
source_type = edge['source_type']
source_title = edge['source_title']
relationship = edge['relationship']
target_type = edge['target_type']
target_title = edge['target_title']
node_for['%s-%s-%s-%s-%s' % (source_type, source_title, relationship, target_type, target_title)] = edge
for edge in certname2_data:
# remove the certname tag.
try:
edge.pop('certname')
except:
pass
source_type = edge['source_type']
source_title = edge['source_title']
relationship = edge['relationship']
target_type = edge['target_type']
target_title = edge['target_title']
node_agn['%s-%s-%s-%s-%s' % (source_type, source_title, relationship, target_type, target_title)] = edge
elif show == "resources":
for resource in certname1_data:
# remove the certname tag.
try:
resource.pop('certname')
except:
pass
resource_title = resource['title']
node_for[resource_title] = resource
for resource in certname2_data:
try:
# remove the certname tag.
resource.pop('certname')
except:
pass
resource_title = resource['title']
node_agn[resource_title] = resource
diff = DictDiffer(node_agn, node_for)
new_entries = list()
rem_entries = list()
cha_entries = list()
# List of new entries
for new_entry in diff.added():
new_entries.append(node_agn[new_entry])
for rem_entry in diff.removed():
rem_entries.append(node_for[rem_entry])
for cha_entry in diff.changed():
for_entry = node_for[cha_entry]
agn_entry = node_agn[cha_entry]
cha_entries.append({
'from': for_entry,
'against': agn_entry
})
output = {
'added_entries': new_entries,
'deleted_entries': rem_entries,
'changed_entries': cha_entries
}
return HttpResponse(json.dumps(output, indent=2), content_type="application/json")
def get_file(request,
certname,
environment,
rtitle,
rtype,
md5sum_from=None,
md5sum_to=None,
diff=False,
file_status='from'):
puppetdb_source, puppetdb_certs, puppetdb_verify = get_server(
request=request)
filebucket_source, filebucket_certs, filebucket_verify, filebucket_show = get_server(
request=request, type='filebucket')
fileserver_source, fileserver_certs, fileserver_verify, fileserver_show = get_server(
request=request, type='fileserver')
# If Clientbucket is enabled continue else return False
def fetch_filebucket(url, method):
headers = {
'Accept': 's',
}
methods = {
'get': requests.get,
'head': requests.head,
}
if method not in methods:
print('No can has method: %s' % method)
return False
resp = methods[method](url,
headers=headers,
verify=filebucket_verify,
cert=filebucket_certs)
if resp.status_code != 200:
return False
else:
return resp.text
def fetch_fileserver(url, method):
methods = {
'get': requests.get,
}
if method not in methods:
print('No can has method: %s' % method)
return False
resp = methods[method](url,
verify=fileserver_verify,
cert=fileserver_certs)
if resp.status_code != 200:
return False
else:
return resp.text
def get_resource(certname, rtype, rtitle):
resource_params = {
'query': {
'operator': 'and',
1: '["=", "certname", "' + certname + '"]',
2: '["=", "type", "' + rtype + '"]',
3: '["=", "title", "' + rtitle + '"]'
},
}
data = pdb_api_get(api_url=puppetdb_source,
path='resources',
verify=puppetdb_verify,
cert=puppetdb_certs,
params=mk_puppetdb_query(resource_params, request))
if not data:
return False
else:
return data
if not filebucket_show or not fileserver_source:
return False
if file_status == 'both':
if md5sum_to and md5sum_from and certname and rtitle and rtype:
if diff:
# is the hash from puppetdb resource same as md5sum_to
hash_matches = False
md5sum_from = md5sum_from.replace('{md5}', '')
md5sum_to = md5sum_to.replace('{md5}', '')
from_url = filebucket_source + environment + '/file_bucket_file/md5/' + md5sum_from
to_url = filebucket_source + environment + '/file_bucket_file/md5/' + md5sum_to
if fetch_filebucket(from_url, 'head') is not False:
resource_from = fetch_filebucket(from_url, 'get')
else:
# Could not find old MD5 in Filebucket
return False
if fetch_filebucket(to_url, 'head') is not False:
resource_to = fetch_filebucket(to_url, 'get')
# Try puppetdb resources if not found in filebucket.
else:
resource_to = get_resource(certname=certname,
rtype=rtype,
rtitle=rtitle)
if resource_to is False:
# Could not find new file in Filebucket or as a PuppetDB Resource
return False
else:
resource_to = resource_to[0]
if 'content' in resource_to['parameters']:
resource_to = resource_to['parameters']['content']
hash_of_resource = get_hash(resource_to)
if hash_of_resource == md5sum_to:
# file from resource matches filebucket md5 hash
hash_matches = True
# Solve the viewing of source files by retrieving it from Puppetmaster
elif 'source' in resource_to[
'parameters'] and fileserver_show is True:
source_path = resource_to['parameters']['source']
if source_path.startswith('puppet://'):
# extract the path for the file
source_path = source_path.split(
'/'
) # ['puppet:', '', '', 'files', 'autofs', 'auto.home']
source_path = '/'.join(
source_path[3:]
) # Skip first 3 entries since they are not needed
# https://puppetmaster.example.com:8140/production/file_content/files/autofs/auto.home
url = fileserver_source + environment + '/file_content/' + source_path
resource_to = fetch_fileserver(url, 'get')
else:
return False
# now that we have come this far, we have both files.
# Lets differentiate the shit out of these files.
from_split_lines = resource_from.split('\n')
to_split_lines = resource_to.split('\n')
diff = difflib.unified_diff(from_split_lines, to_split_lines)
diff = ('\n'.join(list(diff))).split('\n')
return diff
else:
return False
else:
return False
if file_status == 'from' and md5sum_from:
md5sum = md5sum_from.replace('{md5}', '')
elif file_status == 'to' and md5sum_to:
md5sum = md5sum_to.replace('{md5}', '')
else:
return False
# Creates headers and url from the data we got
url_clientbucket = filebucket_source + environment + '/file_bucket_file/md5/' + md5sum
if fetch_filebucket(url_clientbucket, 'head') is False:
# Check if theres a resource available for the latest file available
if file_status == 'to':
resp_pdb = get_resource(certname=certname,
rtype=rtype,
rtitle=rtitle)
# we got the data lets give the user the good news.
if resp_pdb:
resource_data = resp_pdb[0]
if 'content' in resource_data['parameters']:
prepend_text = 'This file with MD5 %s was found in PuppetDB Resources.\n\n' % (
get_hash(resource_data['parameters']['content']))
return prepend_text + resource_data['parameters']['content']
elif 'source' in resource_data[
'parameters'] and fileserver_show is True:
source_path = resource_data['parameters']['source']
if source_path.startswith('puppet://'):
# extract the path for the file
source_path = source_path.split(
'/'
) # ['puppet:', '', '', 'files', 'autofs', 'auto.home']
source_path = '/'.join(
source_path[3:]
) # Skip first 3 entries since they are not needed
# https://puppetmaster.example.com:8140/production/file_content/files/autofs/auto.home
url = fileserver_source + environment + '/file_content/' + source_path
source_content = fetch_fileserver(url, 'get')
prepend_text = 'This file with MD5 %s was retrieved from the PuppetMaster Fileserver.\n\n' % (
get_hash(source_content))
return prepend_text + source_content
else:
return False
else:
return False
# the file can't be found as a resource and or fileserver support not enabled
else:
return False
# We probably don't want to search for resources if its the old file.
else:
return False
else:
filebucket_results = fetch_filebucket(url_clientbucket, 'get')
prepend_text = 'This file with MD5 %s was found in Filebucket.\n\n' % (
md5sum)
return prepend_text + filebucket_results