def catalogue_compare_json(request, certname1=None, certname2=None):
source_url, source_certs, source_verify = get_server(request)
show = request.GET.get('show', 'edges')
if show is not None and show in ['edges', 'resources']:
if show == 'edges':
sort_field = 'source_title'
elif show == 'resources':
sort_field = 'title'
certname1_params = {
'order_by':
{
'order_field':
{
'field': sort_field,
'order': 'asc',
},
}
}
certname2_params = {
'order_by':
{
'order_field':
{
'field': sort_field,
'order': 'asc',
},
}
}
certname1_data = puppetdb.api_get(
path='/catalogs/%s/%s' % (certname1, show),
api_url=source_url,
api_version='v4',
params=puppetdb.mk_puppetdb_query(certname1_params, request),
)
certname1_data = json.dumps(certname1_data, indent=2)
certname2_data = puppetdb.api_get(
path='/catalogs/%s/%s' % (certname2, show),
api_url=source_url,
api_version='v4',
params=puppetdb.mk_puppetdb_query(certname2_params, request),
)
certname2_data = json.dumps(certname2_data, indent=2)
from_split_lines = certname1_data.split('\n')
to_split_lines = certname2_data.split('\n')
diff = difflib.unified_diff(from_split_lines, to_split_lines)
diff = ('\n'.join(list(diff)))
return HttpResponse(diff)
def facts(request, certname=None):
context = {
'timezones': pytz.common_timezones,
'SOURCES': AVAILABLE_SOURCES
}
if request.method == 'GET':
if 'source' in request.GET:
source = request.GET.get('source')
set_server(request, source)
if request.method == 'POST':
request.session['django_timezone'] = request.POST['timezone']
return redirect(request.POST['return_url'])
source_url, source_certs, source_verify = get_server(request)
facts_params = {
'query': {
1: '["=","certname","' + certname + '"]'
},
}
facts_list = puppetdb.api_get(
api_url=source_url,
cert=source_certs,
verify=source_verify,
path='facts',
params=puppetdb.mk_puppetdb_query(facts_params, request),
)
context['certname'] = certname
context['facts_list'] = facts_list
return render(request, 'pano/facts.html', context)
def search_nodes_json(request):
source_url, source_certs, source_verify = get_server(request)
if request.method == 'GET':
if 'search' in request.GET:
search = request.GET.get('search')
# Create a search regex for certname spelt with below.
nodes_params = {
'query':
{
1: '["~","certname","' + search + '"]'
},
'order_by':
{
'order_field':
{
'field': 'certname',
'order': 'desc',
},
},
#'limit': 25,
}
nodes_list = puppetdb.api_get(
api_url=source_url,
cert=source_certs,
verify=source_verify,
path='/nodes',
api_version='v4',
params=puppetdb.mk_puppetdb_query(
nodes_params, request),
)
return HttpResponse(json.dumps(nodes_list), content_type="application/json")
def facts(request, certname=None):
context = {'timezones': pytz.common_timezones,
'SOURCES': AVAILABLE_SOURCES}
if request.method == 'GET':
if 'source' in request.GET:
source = request.GET.get('source')
set_server(request, source)
if request.method == 'POST':
request.session['django_timezone'] = request.POST['timezone']
return redirect(request.POST['return_url'])
source_url, source_certs, source_verify = get_server(request)
facts_params = {
'query':
{
1: '["=","certname","' + certname + '"]'
},
}
facts_list = puppetdb.api_get(
api_url=source_url,
cert=source_certs,
verify=source_verify,
path='facts',
params=puppetdb.mk_puppetdb_query(
facts_params, request),
)
context['certname'] = certname
context['facts_list'] = facts_list
return render(request, 'pano/facts.html', context)
def get_report(key, value, request):
source_url, source_certs, source_verify = get_server(request)
# If key is any of the below, all is good!
allowed_keys = [
'certname', 'resource_title', 'resource_type', 'containing_class'
]
if key in allowed_keys:
pass
# If key does not match above the default will be shown
else:
key = 'containing_class'
events_params = {
'query': {
'operator':
'and',
1:
'["=","' + key + '","' + value + '"]',
2:
'["=","latest_report?",true]',
3:
'["in", "certname",["extract", "certname",["select_nodes",["null?","deactivated",true]]]]'
},
}
results = pdb_api_get(
api_url=source_url,
cert=source_certs,
verify=source_verify,
path='/events',
api_version='v4',
params=mk_puppetdb_query(events_params, request),
)
return results
def reports(request, certname=None):
context = {'timezones': pytz.common_timezones,
'SOURCES': AVAILABLE_SOURCES}
if request.method == 'GET':
if 'source' in request.GET:
source = request.GET.get('source')
set_server(request, source)
if request.method == 'POST':
request.session['django_timezone'] = request.POST['timezone']
return redirect(request.POST['return_url'])
source_url, source_certs, source_verify = get_server(request)
# Redirects to the events page if GET param latest is true..
if request.GET.get('latest', False):
if request.GET.get('latest') == "true":
request.session['report_page'] = 1
latest_report_params = {
'query':
{
1: '["=","certname","' + certname + '"]'
},
'order_by':
{
'order_field':
{
'field': 'start_time',
'order': 'desc',
},
},
'limit': 1,
}
latest_report = puppetdb.api_get(
api_url=source_url,
cert=source_certs,
verify=source_verify,
path='/reports',
api_version='v4',
params=puppetdb.mk_puppetdb_query(latest_report_params, request),
)
report_hash = ""
# If latest reports do not exist, send to the nodes page
# Should only occur if the user is trying to hax their way
# into a node without having the correct permission
if latest_report:
for report in latest_report:
report_env = report['environment']
report_hash = report['hash']
return redirect('/pano/events/' + report_hash + '?report_timestamp=' + request.GET.get(
'report_timestamp') + '&envname=' + report_env)
else:
return redirect('/pano/nodes/')
if certname != request.session.get('last_shown_node', ''):
request.session['last_shown_node'] = certname
request.session['report_page'] = 1
context['certname'] = certname
context['node_facts'] = ','.join(NODES_DEFAULT_FACTS)
return render(request, 'pano/reports.html', context)
def test_summarize_by_query(self):
content = {
'summarize_by': 'containing_class',
}
expected_results = {'summarize_by': 'containing_class'}
results = mk_puppetdb_query(content)
self.assertEqual(expected_results, results)
def reports(request, certname=None):
context = {
'timezones': pytz.common_timezones,
'SOURCES': AVAILABLE_SOURCES
}
if request.method == 'GET':
if 'source' in request.GET:
source = request.GET.get('source')
set_server(request, source)
if request.method == 'POST':
request.session['django_timezone'] = request.POST['timezone']
return redirect(request.POST['return_url'])
source_url, source_certs, source_verify = get_server(request)
# Redirects to the events page if GET param latest is true..
if request.GET.get('latest', False):
if request.GET.get('latest') == "true":
latest_report_params = {
'query': {
1: '["=","certname","' + certname + '"]'
},
'order-by': {
'order-field': {
'field': 'start-time',
'order': 'desc',
},
'query-field': {
'field': 'certname'
},
},
'limit': 1,
}
latest_report = puppetdb.api_get(
api_url=source_url,
cert=source_certs,
verify=source_verify,
path='/reports',
api_version='v4',
params=puppetdb.mk_puppetdb_query(latest_report_params,
request),
)
report_hash = ""
# If latest reports do not exist, send to the nodes page
# Should only occur if the user is trying to hax their way
# into a node without having the correct permission
if latest_report:
for report in latest_report:
report_env = report['environment']
report_hash = report['hash']
return redirect('/pano/events/' + report_hash +
'?report_timestamp=' +
request.GET.get('report_timestamp') +
'&envname=' + report_env)
else:
return redirect('/pano/nodes/')
context['certname'] = certname
context['node_facts'] = ','.join(NODES_DEFAULT_FACTS)
return render(request, 'pano/reports.html', context)
def get_report(key, value, request):
source_url, source_certs, source_verify = get_server(request)
# If key is any of the below, all is good!
allowed_keys = ['certname', 'resource_title', 'resource_type', 'containing_class']
if key in allowed_keys:
pass
# If key does not match above the default will be shown
else:
key = 'containing_class'
events_params = {
'query':
{
'operator': 'and',
1: '["=","' + key + '","' + value + '"]',
2: '["=","latest_report?",true]',
3: '["in", "certname",["extract", "certname",["select_nodes",["null?","deactivated",true]]]]'
},
}
results = pdb_api_get(
api_url=source_url,
cert=source_certs,
verify=source_verify,
path='/events',
api_version='v4',
params=mk_puppetdb_query(events_params, request),
)
return results
def test_summarize_by_query(self):
content = {
'summarize-by': 'containing-class',
}
expected_results = {
'summarize-by': 'containing-class'
}
results = mk_puppetdb_query(content)
self.assertEqual(expected_results, results)
def facts_json(request):
context = {}
if request.method == "GET":
if "source" in request.GET:
source = request.GET.get("source")
set_server(request, source)
if request.method == "POST":
request.session["django_timezone"] = request.POST["timezone"]
return redirect(request.POST["return_url"])
source_url, source_certs, source_verify = get_server(request)
certname = None
facts = None
if "certname" in request.GET:
certname = request.GET.get("certname")
if "facts" in request.GET:
facts = request.GET.get("facts").split(",")
if not certname:
context["error"] = "Certname not specified."
return HttpResponse(json.dumps(context))
if facts:
# validate string for illegal chars
fact_query = list()
for fact in facts:
fact = fact.strip()
# Match for characters that are not a-Z or 0-9 or _
# if theres a match illegal chars exist...
regex = re.compile(r"[^aA-zZ0-9_]")
matches = regex.findall(fact)
if matches:
context["error"] = "Illegal characters found in facts list. "
context["error"] += "Facts must not match anything withddd this regex <[^aA-zZ0-9_]>."
return HttpResponse(json.dumps(context))
fact_query.append('["=","name","' + fact + '"]')
fact_query = ",".join(fact_query)
facts_params = {
"query": {1: '["and",["=","certname","' + certname + '"],["or",' + fact_query + "]]"},
"order-by": {"order_field": {"field": "name", "order": "asc"}},
}
else:
facts_params = {
"query": {1: '["=","certname","' + certname + '"]'},
"order_by": {"order-field": {"field": "name", "order": "asc"}},
}
facts_list = puppetdb.api_get(
api_url=source_url,
cert=source_certs,
verify=source_verify,
path="facts",
params=puppetdb.mk_puppetdb_query(facts_params, request),
)
context["certname"] = certname
context["facts_list"] = facts_list
return HttpResponse(json.dumps(context), content_type="application/json")
def test_single_search_query(self):
content = {
'query': {
1: '["=","certname","hostname.example.com"]'
},
}
expected_results = {
'query': '["and",["=","certname","hostname.example.com"]]'
}
results = mk_puppetdb_query(content)
self.assertEqual(expected_results, results)
def test_single_search_query_with_operator(self):
content = {
'query': {
'operator': 'and',
1: '["=","hash","e4fug294hf3293hf9348g3804hg3084h"]',
},
}
expected_results = {
'query': '["and",["=","hash","e4fug294hf3293hf9348g3804hg3084h"]]'
}
results = mk_puppetdb_query(content)
self.assertEqual(expected_results, results)
def test_single_search_query(self):
content = {
'query':
{
1: '["=","certname","hostname.example.com"]'
},
}
expected_results = {
'query': '["=","certname","hostname.example.com"]'
}
results = mk_puppetdb_query(content)
self.assertEqual(expected_results, results)
def test_single_search_query_with_operator(self):
content = {
'query':
{
'operator': 'and',
1: '["=","hash","e4fug294hf3293hf9348g3804hg3084h"]',
},
}
expected_results = {
'query': '["and", ["=","hash","e4fug294hf3293hf9348g3804hg3084h"]]'
}
results = mk_puppetdb_query(content)
self.assertEqual(expected_results, results)
def test_summarize_by_query_with_single_search_query(self):
content = {
'query': {
1: '["=","certname","hostname.example.com"]'
},
'summarize_by': 'containing_class',
}
expected_results = {
'query': '["and",["=","certname","hostname.example.com"]]',
'summarize_by': 'containing_class'
}
results = mk_puppetdb_query(content)
self.assertEqual(expected_results, results)
def test_order_by_query(self):
content = {
'order_by': {
'order_field': {
'field': 'report_timestamp',
'order': 'desc',
},
}
}
expected_results = {
'order_by': '[{"field":"report_timestamp","order":"desc"}]'
}
results = mk_puppetdb_query(content)
self.assertEqual(expected_results, results)
def test_summarize_by_query_with_single_search_query(self):
content = {
'query':
{
1: '["=","certname","hostname.example.com"]'
},
'summarize-by': 'containing-class',
}
expected_results = {
'query': '["=","certname","hostname.example.com"]',
'summarize-by': 'containing-class'
}
results = mk_puppetdb_query(content)
self.assertEqual(expected_results, results)
def test_order_by_query(self):
content = {
'order-by':
{
'order-field':
{
'field': 'report_timestamp',
'order': 'desc',
},
}
}
expected_results = {
'order-by': '[{"field":"report_timestamp","order":"desc"}]'
}
results = mk_puppetdb_query(content)
self.assertEqual(expected_results, results)
def get_resource(certname, rtype, rtitle):
resource_params = {
'query': {
'operator': 'and',
1: '["=", "certname", "' + certname + '"]',
2: '["=", "type", "' + rtype + '"]',
3: '["=", "title", "' + rtitle + '"]'
},
}
data = pdb_api_get(api_url=puppetdb_source,
path='resources',
verify=puppetdb_verify,
cert=puppetdb_certs,
params=mk_puppetdb_query(resource_params, request))
if not data:
return False
else:
return data
def test_order_by_query_with_single_search_query(self):
content = {
'query': {
1: '["=","certname","hostname.example.com"]'
},
'order_by': {
'order_field': {
'field': 'report_timestamp',
'order': 'desc',
},
}
}
expected_results = {
'order_by': '[{"field":"report_timestamp","order":"desc"}]',
'query': '["and",["=","certname","hostname.example.com"]]'
}
results = mk_puppetdb_query(content)
self.assertEqual(expected_results, results)
def get_events_summary(request, timespan='latest'):
if timespan == 'latest':
events_params = {
'query':
{
1: '["and",["=","latest-report?",true],["in", "certname",["extract", "certname",["select-nodes",["null?","deactivated",true]]]]]'
},
}
source_url, source_certs, source_verify = get_server(request)
events = pdb_api_get(
api_url=source_url,
cert=source_certs,
verify=source_verify,
path='events/',
api_version='v4',
params=mk_puppetdb_query(events_params))
summary = summary_of_events(events)
return summary
def db_threaded_requests(i, q):
while True:
t_job = q.get()
t_path = t_job['path']
t_url = t_job.get('url')
t_certs = t_job.get('certs')
t_verify = t_job.get('verify')
t_params = t_job.get('params', {})
t_api_v = t_job.get('api_version', 'v3')
results = puppetdb.api_get(
api_url=t_url,
verify=t_verify,
cert=t_certs,
path=t_path,
params=puppetdb.mk_puppetdb_query(t_params),
api_version=t_api_v,
)
out_q.put({t_job['id']: results})
q.task_done()
def db_threaded_requests(i, q):
while True:
t_job = q.get()
t_path = t_job['path']
t_url = t_job.get('url')
t_certs = t_job.get('certs')
t_verify = t_job.get('verify')
t_params = t_job.get('params', {})
t_api_v = t_job.get('api_version', 'v3')
t_request = t_job.get('request')
results = puppetdb.api_get(
api_url=t_url,
verify=t_verify,
cert=t_certs,
path=t_path,
params=puppetdb.mk_puppetdb_query(t_params, t_request),
api_version=t_api_v,
)
out_q.put({t_job['id']: results})
q.task_done()
def test_order_by_query_with_single_search_query(self):
content = {
'query':
{
1: '["=","certname","hostname.example.com"]'
},
'order-by':
{
'order-field':
{
'field': 'report_timestamp',
'order': 'desc',
},
}
}
expected_results = {
'order-by': '[{"field":"report_timestamp","order":"desc"}]',
'query': '["=","certname","hostname.example.com"]'
}
results = mk_puppetdb_query(content)
self.assertEqual(expected_results, results)
def get_events_summary(request, timespan='latest'):
if timespan == 'latest':
events_params = {
'query': {
'operator':
'and',
1:
'["=","latest_report?",true]',
2:
'["in","certname",["extract","certname",["select_nodes",["null?","deactivated",true]]]]',
},
}
source_url, source_certs, source_verify = get_server(request)
events = pdb_api_get(api_url=source_url,
cert=source_certs,
verify=source_verify,
path='/events',
api_version='v4',
params=mk_puppetdb_query(events_params, request))
summary = summary_of_events(events)
return summary
def get_resource(certname, rtype, rtitle):
resource_params = {
'query':
{
'operator': 'and',
1: '["=", "certname", "' + certname + '"]',
2: '["=", "type", "' + rtype + '"]',
3: '["=", "title", "' + rtitle + '"]'
},
}
data = pdb_api_get(
api_url=puppetdb_source,
path='resources',
verify=puppetdb_verify,
cert=puppetdb_certs,
params=mk_puppetdb_query(resource_params, request))
if not data:
return False
else:
return data
def test_order_by_query_with_double_search_query_with_operator(self):
content = {
'query':
{
'operator': 'or',
1: '["=","certname","hostname1.example.com"]',
2: '["=","certname","hostname2.example.com"]'
},
'order_by':
{
'order_field':
{
'field': 'report_timestamp',
'order': 'desc',
},
}
}
expected_results = {
'query': '["and",["=","certname","hostname1.example.com"],["=","certname","hostname2.example.com"]]',
'order_by': '[{"field":"report_timestamp","order":"desc"}]'
}
results = mk_puppetdb_query(content)
self.assertEqual(expected_results, results)
def catalogue_json(request, certname=None):
context = dict()
if not certname:
context['error'] = 'Must specify certname.'
return HttpResponse(json.dumps(context), content_type="application/json")
source_url, source_certs, source_verify = get_server(request)
# Redirects to the events page if GET param latest is true..
show = request.GET.get('show', None)
reports_params = {}
if show is not None and show in ['edges', 'resources']:
if show == 'edges':
sort_field = 'source_title'
elif show == 'resources':
sort_field = 'title'
reports_params = {
'order_by':
{
'order_field':
{
'field': sort_field,
'order': 'asc',
},
}
}
path = '/catalogs/%s/%s' % (certname, show)
else:
path = '/catalogs/%s' % certname
reports_list = puppetdb.api_get(
path=path,
api_url=source_url,
api_version='v4',
params=puppetdb.mk_puppetdb_query(reports_params, request),
)
data = {'data': reports_list}
return HttpResponse(json.dumps(data, indent=2), content_type="application/json")
def get_events_summary(request, timespan='latest', environment=None):
events_params = {
'query':
{
'operator': 'and',
2: '["in","certname",["extract","certname",["select_nodes",["null?","deactivated",true]]]]',
},
}
if timespan == 'latest':
events_params['query'][1] = '["=","latest_report?",true]'
elif len(timespan) == 2:
events_params['query'][1] = '["and",[">","timestamp","' + timespan[0] + '"],["<", "timestamp", "' + timespan[
1] + '"]]'
source_url, source_certs, source_verify = get_server(request)
events = pdb_api_get(
api_url=source_url,
cert=source_certs,
verify=source_verify,
path='/events',
api_version='v4',
params=mk_puppetdb_query(events_params, request))
summary = summary_of_events(events)
return summary
def reports_search_json(request):
context = dict()
if request.method == 'GET':
if 'search' in request.GET:
search = request.GET.get('search')
if 'certname' in request.GET:
certname = request.GET.get('certname')
if not certname or not search:
context['error'] = 'Must specify both certname and search query.'
return HttpResponse(json.dumps(context), content_type="application/json")
source_url, source_certs, source_verify = get_server(request)
# Redirects to the events page if GET param latest is true..
reports_params = {
'query':
{
'operator': 'and',
1: '["=","certname","' + certname + '"]',
2: '["~","hash","^' + search + '"]'
},
'order_by':
{
'order_field':
{
'field': 'start_time',
'order': 'desc',
},
}
}
reports_list = puppetdb.api_get(
path='/reports',
api_url=source_url,
api_version='v4',
params=puppetdb.mk_puppetdb_query(reports_params, request),
)
return HttpResponse(json.dumps(reports_list), content_type="application/json")
def reports(request, certname=None):
context = default_context
if request.method == "GET":
if "source" in request.GET:
source = request.GET.get("source")
set_server(request, source)
if request.method == "POST":
request.session["django_timezone"] = request.POST["timezone"]
return redirect(request.POST["return_url"])
source_url, source_certs, source_verify = get_server(request)
# Redirects to the events page if GET param latest is true..
if request.GET.get("latest", False):
if request.GET.get("latest") == "true":
latest_report_params = {
"query": {1: '["=","certname","' + certname + '"]'},
"order-by": {
"order-field": {"field": "start-time", "order": "desc"},
"query-field": {"field": "certname"},
},
"limit": 1,
}
latest_report = puppetdb.api_get(
api_url=source_url,
cert=source_certs,
verify=source_verify,
path="/reports",
api_version="v4",
params=puppetdb.mk_puppetdb_query(latest_report_params),
)
report_hash = ""
for report in latest_report:
report_env = report["environment"]
report_hash = report["hash"]
return redirect(
"/pano/events/"
+ certname
+ "/"
+ report_hash
+ "?report_timestamp="
+ request.GET.get("report_timestamp")
+ "&envname="
+ report_env
)
page_num = int(request.GET.get("page", 0))
if page_num <= 0:
offset = 0
else:
offset = "{:.0f}".format(page_num * 25)
reports_params = {
"query": {1: '["=","certname","' + certname + '"]'},
"order-by": {"order-field": {"field": "start-time", "order": "desc"}, "query-field": {"field": "certname"}},
"limit": 25,
"include-total": "true",
"offset": offset,
}
reports_list, headers = puppetdb.api_get(
api_url=source_url,
cert=source_certs,
verify=source_verify,
path="/reports",
api_version="v4",
params=puppetdb.mk_puppetdb_query(reports_params),
)
# Work out the number of pages from the xrecords response
xrecords = headers["X-Records"]
num_pages_wdec = float(xrecords) / 25
num_pages_wodec = float("{:.0f}".format(num_pages_wdec))
if num_pages_wdec > num_pages_wodec:
num_pages = num_pages_wodec + 1
else:
num_pages = num_pages_wodec
report_status = []
for report in reports_list:
found_report = False
events_params = {"query": {1: '["=","report","' + report["hash"] + '"]'}, "summarize-by": "certname"}
eventcount_list = puppetdb.api_get(
path="event-counts", api_url=source_url, api_version="v4", params=puppetdb.mk_puppetdb_query(events_params)
)
# Make list of the results
for event in eventcount_list:
if event["subject"]["title"] == report["certname"]:
found_report = True
# hashid, certname, environment, time start, time end, success, noop, failure, pending
report_status.append(
[
report["hash"],
report["certname"],
report["environment"],
report["start-time"],
report["end-time"],
event["successes"],
event["noops"],
event["failures"],
event["skips"],
]
)
break
if found_report is False:
report_status.append(
[
report["hash"],
report["certname"],
report["environment"],
report["start-time"],
report["end-time"],
0,
0,
0,
0,
]
)
report_status = sort_tables(report_status, order=True, col=3)
context["certname"] = certname
context["reports"] = report_status
context["curr_page"] = page_num
context["tot_pages"] = "{:.0f}".format(num_pages)
return render(request, "pano/reports.html", context)
def facts_json(request):
context = {}
if request.method == 'GET':
if 'source' in request.GET:
source = request.GET.get('source')
set_server(request, source)
if request.method == 'POST':
request.session['django_timezone'] = request.POST['timezone']
return redirect(request.POST['return_url'])
source_url, source_certs, source_verify = get_server(request)
certname = None
facts = None
if 'certname' in request.GET:
certname = request.GET.get('certname')
if 'facts' in request.GET:
facts = request.GET.get('facts').split(',')
if not certname:
context['error'] = 'Certname not specified.'
return HttpResponse(json.dumps(context))
if facts:
# validate string for illegal chars
fact_query = list()
for fact in facts:
fact = fact.strip()
# Match for characters that are not a-Z or 0-9 or _
# if theres a match illegal chars exist...
regex = re.compile(r'[^aA-zZ0-9_]')
matches = regex.findall(fact)
if matches:
context['error'] = 'Illegal characters found in facts list. '
context[
'error'] += 'Facts must not match anything withddd this regex <[^aA-zZ0-9_]>.'
return HttpResponse(json.dumps(context))
fact_query.append('["=","name","' + fact + '"]')
fact_query = ','.join(fact_query)
facts_params = {
'query': {
1:
'["and",["=","certname","' + certname + '"],["or",' +
fact_query + ']]'
},
'order-by': {
'order_field': {
'field': 'name',
'order': 'asc',
}
}
}
else:
facts_params = {
'query': {
1: '["=","certname","' + certname + '"]'
},
'order_by': {
'order-field': {
'field': 'name',
'order': 'asc',
}
}
}
facts_list = puppetdb.api_get(
api_url=source_url,
cert=source_certs,
verify=source_verify,
path='facts',
params=puppetdb.mk_puppetdb_query(facts_params, request),
)
context['certname'] = certname
context['facts_list'] = facts_list
return HttpResponse(json.dumps(context), content_type="application/json")
def facts_json(request):
context = {}
if request.method == 'GET':
if 'source' in request.GET:
source = request.GET.get('source')
set_server(request, source)
if request.method == 'POST':
request.session['django_timezone'] = request.POST['timezone']
return redirect(request.POST['return_url'])
source_url, source_certs, source_verify = get_server(request)
certname = None
facts = None
if 'certname' in request.GET:
certname = request.GET.get('certname')
if 'facts' in request.GET:
facts = request.GET.get('facts').split(',')
if not certname:
context['error'] = 'Certname not specified.'
return HttpResponse(json.dumps(context))
if facts:
# validate string for illegal chars
fact_query = list()
for fact in facts:
fact = fact.strip()
# Match for characters that are not a-Z or 0-9 or _
# if theres a match illegal chars exist...
regex = re.compile(r'[^aA-zZ0-9_]')
matches = regex.findall(fact)
if matches:
context['error'] = 'Illegal characters found in facts list. '
context['error'] += 'Facts must not match anything withddd this regex <[^aA-zZ0-9_]>.'
return HttpResponse(json.dumps(context))
fact_query.append('["=","name","' + fact + '"]')
fact_query = ','.join(fact_query)
facts_params = {
'query':
{
1: '["and",["=","certname","' + certname + '"],["or",' + fact_query + ']]'
},
'order-by':
{
'order_field':
{
'field': 'name',
'order': 'asc',
}
}
}
else:
facts_params = {
'query':
{
1: '["=","certname","' + certname + '"]'
},
'order_by':
{
'order-field':
{
'field': 'name',
'order': 'asc',
}
}
}
facts_list = puppetdb.api_get(
api_url=source_url,
cert=source_certs,
verify=source_verify,
path='facts',
params=puppetdb.mk_puppetdb_query(
facts_params, request),
)
context['certname'] = certname
context['facts_list'] = facts_list
return HttpResponse(json.dumps(context, indent=2), content_type="application/json")
def reports_json(request, certname=None):
source_url, source_certs, source_verify = get_server(request)
# Redirects to the events page if GET param latest is true..
context = {}
# Cur Page Number
if request.GET.get('page', False):
if request.session['report_page'] != int(request.GET.get('page', 1)):
request.session['report_page'] = int(request.GET.get('page', 1))
if request.session['report_page'] <= 0:
request.session['report_page'] = 1
else:
if 'report_page' not in request.session:
request.session['report_page'] = 1
if request.session['report_page'] <= 0:
offset = 0
else:
offset = (25 * request.session['report_page']) - 25
reports_params = {
'query':
{
1: '["=","certname","' + certname + '"]'
},
'order-by':
{
'order-field':
{
'field': 'start_time',
'order': 'desc',
},
'query-field': {'field': 'certname'},
},
'limit': 25,
'include_total': 'true',
'offset': offset,
}
reports_list, headers = puppetdb.api_get(
api_url=source_url,
cert=source_certs,
verify=source_verify,
path='/reports',
api_version='v4',
params=puppetdb.mk_puppetdb_query(
reports_params, request),
)
# Work out the number of pages from the xrecords response
xrecords = headers['X-Records']
num_pages_wdec = float(xrecords) / 25
num_pages_wodec = float("{:.0f}".format(num_pages_wdec))
if num_pages_wdec > num_pages_wodec:
num_pages = num_pages_wodec + 1
else:
num_pages = num_pages_wodec
report_status = []
for report in reports_list:
found_report = False
events_params = {
'query':
{
1: '["=","report","' + report['hash'] + '"]'
},
'summarize_by': 'certname',
}
eventcount_list = puppetdb.api_get(
path='event-counts',
api_url=source_url,
api_version='v4',
params=puppetdb.mk_puppetdb_query(events_params, request),
)
# Make list of the results
for event in eventcount_list:
if event['subject']['title'] == report['certname']:
found_report = True
report_status.append({
'hash': report['hash'],
'certname': report['certname'],
'environment': report['environment'],
'start_time': filters.date(localtime(json_to_datetime(report['start_time'])), 'Y-m-d H:i:s'),
'end_time': filters.date(localtime(json_to_datetime(report['end_time'])), 'Y-m-d H:i:s'),
'events_successes': event['successes'],
'events_noops': event['noops'],
'events_failures': event['failures'],
'events_skipped': event['skips'],
'report_status': report['status'],
'config_version': report['configuration_version'],
'run_duration': "{0:.0f}".format(
(json_to_datetime(report['end_time']) - json_to_datetime(report['start_time'])).total_seconds())
})
break
if found_report is False:
report_status.append({
'hash': report['hash'],
'certname': report['certname'],
'environment': report['environment'],
'start_time': filters.date(localtime(json_to_datetime(report['start_time'])), 'Y-m-d H:i:s'),
'end_time': filters.date(localtime(json_to_datetime(report['end_time'])), 'Y-m-d H:i:s'),
'events_successes': 0,
'events_noops': 0,
'events_failures': 0,
'events_skipped': 0,
'report_status': report['status'],
'config_version': report['configuration_version'],
'run_duration': "{0:.0f}".format(
(json_to_datetime(report['end_time']) - json_to_datetime(report['start_time'])).total_seconds())
})
context['certname'] = certname
context['reports_list'] = report_status
context['curr_page'] = request.session['report_page']
context['tot_pages'] = "{:.0f}".format(num_pages)
return HttpResponse(json.dumps(context), content_type="application/json")
def detailed_events(request, hashid=None):
context = {
'timezones': pytz.common_timezones,
'SOURCES': AVAILABLE_SOURCES
}
if request.method == 'GET':
if 'source' in request.GET:
source = request.GET.get('source')
set_server(request, source)
if request.method == 'POST':
request.session['django_timezone'] = request.POST['timezone']
return redirect(request.POST['return_url'])
source_url, source_certs, source_verify = get_server(request)
report_timestamp = request.GET.get('report_timestamp')
events_params = {
'query': {
1: '["=","report","' + hashid + '"]'
},
'order_by': {
'order_field': {
'field': 'timestamp',
'order': 'asc',
},
'query_field': {
'field': 'certname'
},
},
}
events_list = puppetdb.api_get(
api_url=source_url,
cert=source_certs,
verify=source_verify,
path='/events',
api_version='v4',
params=puppetdb.mk_puppetdb_query(events_params),
)
environment = ''
certname = ''
event_execution_times = []
sorted_events = None
last_event_time = None
last_event_title = None
run_end_time = None
if len(events_list) != 0:
single_event = events_list[0]
environment = single_event['environment']
certname = single_event['certname']
for event in events_list:
event_title = event['resource_title']
event_start_time = json_to_datetime(event['timestamp'])
if last_event_time is None and last_event_title is None:
last_event_time = event_start_time
last_event_title = event_title
run_end_time = json_to_datetime(event['run_end_time'])
continue
else:
event_exec_time = (event_start_time -
last_event_time).total_seconds()
add_event = (last_event_title, event_exec_time)
event_execution_times.append(add_event)
last_event_time = event_start_time
last_event_title = event_title
event_exec_time = (last_event_time - run_end_time).total_seconds()
add_event = [last_event_title, event_exec_time]
event_execution_times.append(add_event)
sorted_events = sorted(event_execution_times,
reverse=True,
key=lambda field: field[1])
if len(sorted_events) > 10:
sorted_events = sorted_events[:10]
else:
events_list = False
context['certname'] = certname
context['report_timestamp'] = report_timestamp
context['hashid'] = hashid
context['events_list'] = events_list
context['event_durations'] = sorted_events
context['environment'] = environment
return render(request, 'pano/detailed_events.html', context)
def reports_json(request, certname=None):
source_url, source_certs, source_verify = get_server(request)
# Redirects to the events page if GET param latest is true..
context = {}
# Cur Page Number
if request.GET.get('page', False):
if request.session['report_page'] != int(request.GET.get('page', 1)):
request.session['report_page'] = int(request.GET.get('page', 1))
if request.session['report_page'] <= 0:
request.session['report_page'] = 1
else:
if 'report_page' not in request.session:
request.session['report_page'] = 1
if request.session['report_page'] <= 0:
offset = 0
else:
offset = (25 * request.session['report_page']) - 25
reports_params = {
'query':
{
1: '["=","certname","' + certname + '"]'
},
'order_by':
{
'order_field':
{
'field': 'start_time',
'order': 'desc',
},
},
'limit': 25,
'include_total': 'true',
'offset': offset,
}
reports_list, headers = puppetdb.api_get(
api_url=source_url,
cert=source_certs,
verify=source_verify,
path='/reports',
api_version='v4',
params=puppetdb.mk_puppetdb_query(
reports_params, request),
)
# Work out the number of pages from the xrecords response
xrecords = headers['X-Records']
num_pages_wdec = float(xrecords) / 25
num_pages_wodec = float("{:.0f}".format(num_pages_wdec))
if num_pages_wdec > num_pages_wodec:
num_pages = num_pages_wodec + 1
else:
num_pages = num_pages_wodec
report_status = []
for report in reports_list:
found_report = False
events_params = {
'query':
{
1: '["=","report","' + report['hash'] + '"]'
},
'summarize_by': 'certname',
}
eventcount_list = puppetdb.api_get(
path='event-counts',
api_url=source_url,
api_version='v4',
params=puppetdb.mk_puppetdb_query(events_params, request),
)
# Make list of the results
for event in eventcount_list:
if event['subject']['title'] == report['certname']:
found_report = True
report_status.append({
'hash': report['hash'],
'certname': report['certname'],
'environment': report['environment'],
'start_time': filters.date(localtime(json_to_datetime(report['start_time'])), 'Y-m-d H:i:s'),
'end_time': filters.date(localtime(json_to_datetime(report['end_time'])), 'Y-m-d H:i:s'),
'events_successes': event['successes'],
'events_noops': event['noops'],
'events_failures': event['failures'],
'events_skipped': event['skips'],
'report_status': report['status'],
'config_version': report['configuration_version'],
'run_duration': "{0:.0f}".format(
(json_to_datetime(report['end_time']) - json_to_datetime(report['start_time'])).total_seconds())
})
break
if found_report is False:
report_status.append({
'hash': report['hash'],
'certname': report['certname'],
'environment': report['environment'],
'start_time': filters.date(localtime(json_to_datetime(report['start_time'])), 'Y-m-d H:i:s'),
'end_time': filters.date(localtime(json_to_datetime(report['end_time'])), 'Y-m-d H:i:s'),
'events_successes': 0,
'events_noops': 0,
'events_failures': 0,
'events_skipped': 0,
'report_status': report['status'],
'config_version': report['configuration_version'],
'run_duration': "{0:.0f}".format(
(json_to_datetime(report['end_time']) - json_to_datetime(report['start_time'])).total_seconds())
})
context['certname'] = certname
context['reports_list'] = report_status
context['curr_page'] = request.session['report_page']
context['tot_pages'] = "{:.0f}".format(num_pages)
return HttpResponse(json.dumps(context), content_type="application/json")
def detailed_events(request, hashid=None):
context = {'timezones': pytz.common_timezones,
'SOURCES': AVAILABLE_SOURCES}
if request.method == 'GET':
if 'source' in request.GET:
source = request.GET.get('source')
set_server(request, source)
if request.method == 'POST':
request.session['django_timezone'] = request.POST['timezone']
return redirect(request.POST['return_url'])
source_url, source_certs, source_verify = get_server(request)
report_timestamp = request.GET.get('report_timestamp')
events_params = {
'query':
{
1: '["=","report","' + hashid + '"]'
},
'order_by':
{
'order_field':
{
'field': 'timestamp',
'order': 'asc',
},
'query_field': {'field': 'certname'},
},
}
events_list = puppetdb.api_get(
api_url=source_url,
cert=source_certs,
verify=source_verify,
path='/events',
api_version='v4',
params=puppetdb.mk_puppetdb_query(events_params),
)
environment = ''
certname = ''
event_execution_times = []
sorted_events = None
last_event_time = None
last_event_title = None
run_end_time = None
if len(events_list) != 0:
single_event = events_list[0]
environment = single_event['environment']
certname = single_event['certname']
for event in events_list:
event_title = event['resource_title']
event_start_time = json_to_datetime(event['timestamp'])
if last_event_time is None and last_event_title is None:
last_event_time = event_start_time
last_event_title = event_title
run_end_time = json_to_datetime(event['run_end_time'])
continue
else:
event_exec_time = (event_start_time - last_event_time).total_seconds()
add_event = (last_event_title, event_exec_time)
event_execution_times.append(add_event)
last_event_time = event_start_time
last_event_title = event_title
event_exec_time = (last_event_time - run_end_time).total_seconds()
add_event = [last_event_title, event_exec_time]
event_execution_times.append(add_event)
sorted_events = sorted(event_execution_times, reverse=True, key=lambda field: field[1])
if len(sorted_events) > 10:
sorted_events = sorted_events[:10]
else:
events_list = False
context['certname'] = certname
context['report_timestamp'] = report_timestamp
context['hashid'] = hashid
context['events_list'] = events_list
context['event_durations'] = sorted_events
context['environment'] = environment
return render(request, 'pano/detailed_events.html', context)
def nodes_json(request):
if request.method == 'GET':
if 'source' in request.GET:
source = request.GET.get('source')
set_server(request, source)
if request.method == 'POST':
request.session['django_timezone'] = request.POST['timezone']
return redirect(request.POST['return_url'])
source_url, source_certs, source_verify = get_server(request)
valid_sort_fields = ('certname', 'catalog_timestamp', 'report_timestamp',
'facts_timestamp', 'successes', 'noops', 'failures',
'skips')
try:
# If user requested to download csv formatted file. Default value is False
dl_csv = request.GET.get('dl_csv', False)
if dl_csv == 'true':
dl_csv = True
else:
dl_csv = False
# Add limits to session
if request.GET.get('limits', False):
if request.session['limits'] != int(request.GET.get('limits', 50)):
request.session['limits'] = int(request.GET.get('limits', 50))
if request.session['limits'] <= 0:
request.session['limits'] = 50
else:
if 'limits' not in request.session:
request.session['limits'] = 50
# Cur Page Number
if request.GET.get('page', False):
if request.session['page'] != int(request.GET.get('page', 1)):
request.session['page'] = int(request.GET.get('page', 1))
if request.session['page'] <= 0:
request.session['page'] = 1
else:
if 'page' not in request.session:
request.session['page'] = 1
# Cur sort field
if request.GET.get('sortfield', False):
if request.session['sortfield'] != request.GET.get('sortfield'):
request.session['sortfield'] = request.GET.get('sortfield')
if request.session['sortfield'] not in valid_sort_fields:
request.session['sortfield'] = 'report_timestamp'
else:
if 'sortfield' not in request.session:
request.session['sortfield'] = 'report_timestamp'
# Cur sort order
if request.GET.get('sortfieldby', False):
avail_sortorder = ['asc', 'desc']
if request.session['sortfieldby'] != request.GET.get(
'sortfieldby'):
request.session['sortfieldby'] = request.GET.get('sortfieldby')
if request.session['sortfieldby'] not in avail_sortorder:
request.session['sortfieldby'] = 'desc'
else:
if 'sortfieldby' not in request.session:
request.session['sortfieldby'] = 'desc'
# Search parameters takes a valid puppetdb query string
if request.GET.get('search', False):
if 'search' in request.session and (request.session['search']
== request.GET.get('search')):
pass
else:
if request.GET.get('search') == 'clear_rules':
request.session['sortfield'] = 'report_timestamp'
request.session['sortfieldby'] = 'desc'
request.session['page'] = 1
request.session['search'] = None
else:
request.session['page'] = 1
request.session['search'] = request.GET.get('search')
else:
if 'search' not in request.session:
request.session['sortfield'] = 'report_timestamp'
request.session['sortfieldby'] = 'desc'
request.session['page'] = 1
request.session['search'] = None
# Set offset
request.session['offset'] = (
request.session['limits'] *
request.session['page']) - request.session['limits']
except:
return HttpResponseBadRequest('Oh no! Your filters were invalid.')
# Valid sort field that the user can search agnaist.
sort_field = request.session['sortfield']
sort_field_order = request.session['sortfieldby']
page_num = request.session['page']
if request.session['search'] is not None:
node_params = {
'query': {
1: request.session['search']
},
}
else:
node_params = {
'query': {},
}
nodes_sort_fields = [
'certname', 'catalog_timestamp', 'report_timestamp', 'facts_timestamp'
]
if sort_field in nodes_sort_fields:
node_params['order_by'] = {
'order_field': {
'field': sort_field,
'order': sort_field_order,
},
}
if dl_csv is False:
node_params['limit'] = request.session['limits']
node_params['offset'] = request.session['offset']
node_params['include_total'] = 'true'
else:
node_params['order_by'] = {
'order_field': {
'field': 'report_timestamp',
'order': 'desc',
},
}
node_sort_fields = [
'certname', 'catalog_timestamp', 'report_timestamp', 'facts_timestamp'
]
if sort_field in node_sort_fields:
try:
node_list, node_headers = puppetdb.api_get(
api_url=source_url,
cert=source_certs,
verify=source_verify,
path='/nodes',
api_version='v4',
params=puppetdb.mk_puppetdb_query(node_params, request),
)
except:
node_list = []
node_headers = dict()
node_headers['X-Records'] = 0
else:
node_list = puppetdb.api_get(
api_url=source_url,
cert=source_certs,
verify=source_verify,
path='/nodes',
api_version='v4',
params=puppetdb.mk_puppetdb_query(node_params, request),
)
# Work out the number of pages from the xrecords response
# return fields that you can sort by
# for each node in the node_list, find out if the latest run has any failures
# v3/event-counts --data-urlencode query='["=","latest-report?",true]'
# --data-urlencode summarize-by='certname'
report_params = {
'query': {
1:
'["and",["=","latest_report?",true],["in", "certname",["extract", "certname",["select_nodes",["null?","deactivated",true]]]]]'
},
'summarize_by': 'certname',
}
status_sort_fields = ['successes', 'failures', 'skips', 'noops']
report_status_params = {
'query': {
1:
'["and",["=","latest_report?",true],["in", "certname",["extract", "certname",["select_nodes",["null?","deactivated",true]]]]]'
}
}
report_status_list = puppetdb.api_get(
api_url=source_url,
cert=source_certs,
verify=source_verify,
path='/reports',
params=puppetdb.mk_puppetdb_query(report_status_params, request),
api_version='v4',
)
if sort_field in status_sort_fields:
if request.session['search'] is not None:
report_params['query'] = {
'operator':
'and',
1:
request.session['search'],
2:
'["=","latest_report?",true]',
3:
'["in", "certname",["extract", "certname",["select_nodes",["null?","deactivated",true]]]]',
}
report_params['order_by'] = {
'order_field': {
'field': sort_field,
'order': sort_field_order,
}
}
report_params['include_total'] = 'true'
# Don't limit results if its CSV
if dl_csv is False:
report_params['limit'] = request.session['limits']
report_params['offset'] = request.session['offset']
report_list, report_headers = puppetdb.api_get(
api_url=source_url,
cert=source_certs,
verify=source_verify,
path='/event-counts',
params=puppetdb.mk_puppetdb_query(report_params, request),
api_version='v4',
)
else:
report_list = puppetdb.api_get(
api_url=source_url,
cert=source_certs,
verify=source_verify,
path='event-counts',
params=puppetdb.mk_puppetdb_query(report_params, request),
api_version='v4',
)
# number of results depending on sort field.
if sort_field in status_sort_fields:
xrecords = report_headers['X-Records']
total_results = xrecords
elif sort_field in nodes_sort_fields:
xrecords = node_headers['X-Records']
total_results = xrecords
num_pages_wdec = float(xrecords) / request.session['limits']
num_pages_wodec = float("{:.0f}".format(num_pages_wdec))
if num_pages_wdec > num_pages_wodec:
num_pages = num_pages_wodec + 1
else:
num_pages = num_pages_wodec
# Converts lists of dicts to dicts.
node_dict = {item['certname']: item for item in node_list}
status_dict = {item['certname']: item for item in report_status_list}
report_dict = {item['subject']['title']: item for item in report_list}
if sort_field_order == 'desc':
rows = dictstatus(node_dict,
status_dict,
report_dict,
sortby=sort_field,
asc=True,
sort=False)
sort_field_order_opposite = 'asc'
elif sort_field_order == 'asc':
rows = dictstatus(node_dict,
status_dict,
report_dict,
sortby=sort_field,
asc=False,
sort=False)
sort_field_order_opposite = 'desc'
if dl_csv is True:
if rows is []:
pass
else:
# Generate a sequence of rows. The range is based on the maximum number of
# rows that can be handled by a single sheet in most spreadsheet
# applications.
include_facts = request.GET.get('include_facts', False)
csv_headers = [
'Certname', 'Latest Catalog', 'Latest Report', 'Latest Facts',
'Success', 'Noop', 'Failure', 'Skipped'
]
if include_facts is not False:
merged_list_facts = []
facts = {}
for fact in include_facts.split(','):
# Sanitize the fact input from the user
fact = fact.strip()
# Add the fact name to the headers list
csv_headers.append(fact)
# build the params for each fact.
facts_params = {
'query': {
1: '["=","name","' + fact + '"]'
},
}
fact_list = puppetdb.api_get(
api_url=source_url,
cert=source_certs,
verify=source_verify,
path='facts',
params=puppetdb.mk_puppetdb_query(facts_params),
api_version='v4',
)
# Populate the facts dict with the facts we have retrieved
# Convert the fact list into a fact dict!
facts[fact] = {
item['certname']: item
for item in fact_list
}
i = 1
jobs = {}
# Add ID to each job so that it can be assembled in
# the same order after we recieve the job results
# We do this via jobs so that we can get faster results.
for node in rows:
jobs[i] = {
'id': i,
'include_facts': include_facts.split(','),
'node': node,
'facts': facts,
}
i += 1
csv_results = generate_csv(jobs)
rows = []
i = 1
# with the job results we can now recreate merged_list
# in the order we sent them.
while i <= len(csv_results):
rows.append(csv_results[i])
i += 1
# Insert the csv header to the top of the list.
rows.insert(0, csv_headers)
pseudo_buffer = Echo()
writer = csv.writer(pseudo_buffer)
response = StreamingHttpResponse(
(writer.writerow(row) for row in rows),
content_type="text/csv")
response[
'Content-Disposition'] = 'attachment; filename="puppetdata-%s.csv"' % (
datetime.datetime.now())
return response
"""
c_r_s* = current request sort
c_r_* = current req
r_s* = requests available
"""
context = {
'nodeList': rows,
'total_nodes': total_results,
'c_r_page': page_num,
'c_r_limit': request.session['limits'],
'r_sfield': valid_sort_fields,
'c_r_sfield': sort_field,
'r_sfieldby': ['asc', 'desc'],
'c_r_sfieldby': sort_field_order,
'c_r_sfieldby_o': sort_field_order_opposite,
'tot_pages': '{0:g}'.format(num_pages),
}
return HttpResponse(json.dumps(context), content_type="application/json")
def catalogue_compare_json(request, certname1=None, certname2=None):
source_url, source_certs, source_verify = get_server(request)
show = request.GET.get('show', 'edges')
data = dict()
certname1_hash = request.GET.get('certname1_hash', False)
certname2_hash = request.GET.get('certname2_hash', False)
cata_params = dict()
if certname1_hash:
try:
certname1_result = SavedCatalogs.objects.get(hostname=certname1, catalogue_id=certname1_hash)
certname1_data = json.loads(certname1_result.catalogue)[show]['data']
except SavedCatalogs.DoesNotExist:
data['error'] = 'Catalogue hash not found in DB.'
data['hash_not_found'] = certname1_hash
data['certname'] = certname1
return HttpResponseBadRequest(json.dumps(data, indent=2), content_type="application/json")
else:
certname1_result = puppetdb.api_get(
path='/catalogs/%s' % certname1,
api_url=source_url,
api_version='v4',
params=puppetdb.mk_puppetdb_query(cata_params, request),
)
certname1_data = certname1_result[show]['data']
if certname2_hash:
try:
certname2_result = SavedCatalogs.objects.get(hostname=certname2, catalogue_id=certname2_hash)
certname2_data = json.loads(certname2_result.catalogue)[show]['data']
except SavedCatalogs.DoesNotExist:
data['error'] = 'Catalogue hash not found in DB.'
data['hash_not_found'] = certname2_hash
data['certname'] = certname2
return HttpResponseBadRequest(json.dumps(data, indent=2), content_type="application/json")
else:
certname2_result = puppetdb.api_get(
path='/catalogs/%s' % certname2,
api_url=source_url,
api_version='v4',
params=puppetdb.mk_puppetdb_query(cata_params, request),
)
certname2_data = certname2_result[show]['data']
node_for = dict()
node_agn = dict()
if show == "edges":
for edge in certname1_data:
# remove the certname tag.
try:
edge.pop('certname')
except:
pass
source_type = edge['source_type']
source_title = edge['source_title']
relationship = edge['relationship']
target_type = edge['target_type']
target_title = edge['target_title']
node_for['%s-%s-%s-%s-%s' % (source_type, source_title, relationship, target_type, target_title)] = edge
for edge in certname2_data:
# remove the certname tag.
try:
edge.pop('certname')
except:
pass
source_type = edge['source_type']
source_title = edge['source_title']
relationship = edge['relationship']
target_type = edge['target_type']
target_title = edge['target_title']
node_agn['%s-%s-%s-%s-%s' % (source_type, source_title, relationship, target_type, target_title)] = edge
elif show == "resources":
for resource in certname1_data:
# remove the certname tag.
try:
resource.pop('certname')
except:
pass
resource_title = resource['title']
node_for[resource_title] = resource
for resource in certname2_data:
try:
# remove the certname tag.
resource.pop('certname')
except:
pass
resource_title = resource['title']
node_agn[resource_title] = resource
diff = DictDiffer(node_agn, node_for)
new_entries = list()
rem_entries = list()
cha_entries = list()
# List of new entries
for new_entry in diff.added():
new_entries.append(node_agn[new_entry])
for rem_entry in diff.removed():
rem_entries.append(node_for[rem_entry])
for cha_entry in diff.changed():
for_entry = node_for[cha_entry]
agn_entry = node_agn[cha_entry]
cha_entries.append({
'from': for_entry,
'against': agn_entry
})
output = {
'added_entries': new_entries,
'deleted_entries': rem_entries,
'changed_entries': cha_entries
}
return HttpResponse(json.dumps(output, indent=2), content_type="application/json")
def nodes_json(request):
if request.method == 'GET':
if 'source' in request.GET:
source = request.GET.get('source')
set_server(request, source)
if request.method == 'POST':
request.session['django_timezone'] = request.POST['timezone']
return redirect(request.POST['return_url'])
source_url, source_certs, source_verify = get_server(request)
valid_sort_fields = (
'certname',
'catalog_timestamp',
'report_timestamp',
'facts_timestamp',
'successes',
'noops',
'failures',
'skips')
try:
# If user requested to download csv formatted file. Default value is False
dl_csv = request.GET.get('dl_csv', False)
if dl_csv == 'true':
dl_csv = True
else:
dl_csv = False
# Add limits to session
if request.GET.get('limits', False):
if request.session['limits'] != int(request.GET.get('limits', 50)):
request.session['limits'] = int(request.GET.get('limits', 50))
if request.session['limits'] <= 0:
request.session['limits'] = 50
else:
if 'limits' not in request.session:
request.session['limits'] = 50
# Cur Page Number
if request.GET.get('page', False):
if request.session['page'] != int(request.GET.get('page', 1)):
request.session['page'] = int(request.GET.get('page', 1))
if request.session['page'] <= 0:
request.session['page'] = 1
else:
if 'page' not in request.session:
request.session['page'] = 1
# Cur sort field
if request.GET.get('sortfield', False):
if request.session['sortfield'] != request.GET.get('sortfield'):
request.session['sortfield'] = request.GET.get('sortfield')
if request.session['sortfield'] not in valid_sort_fields:
request.session['sortfield'] = 'report_timestamp'
else:
if 'sortfield' not in request.session:
request.session['sortfield'] = 'report_timestamp'
# Cur sort order
if request.GET.get('sortfieldby', False):
avail_sortorder = ['asc', 'desc']
if request.session['sortfieldby'] != request.GET.get('sortfieldby'):
request.session['sortfieldby'] = request.GET.get('sortfieldby')
if request.session['sortfieldby'] not in avail_sortorder:
request.session['sortfieldby'] = 'desc'
else:
if 'sortfieldby' not in request.session:
request.session['sortfieldby'] = 'desc'
# Search parameters takes a valid puppetdb query string
if request.GET.get('search', False):
if 'search' in request.session and (request.session['search'] == request.GET.get('search')):
pass
else:
if request.GET.get('search') == 'clear_rules':
request.session['sortfield'] = 'report_timestamp'
request.session['sortfieldby'] = 'desc'
request.session['page'] = 1
request.session['search'] = None
else:
request.session['page'] = 1
request.session['search'] = request.GET.get('search')
else:
if 'search' not in request.session:
request.session['sortfield'] = 'report_timestamp'
request.session['sortfieldby'] = 'desc'
request.session['page'] = 1
request.session['search'] = None
# Set offset
request.session['offset'] = (request.session['limits'] * request.session['page']) - request.session[
'limits']
except:
return HttpResponseBadRequest('Oh no! Your filters were invalid.')
# Valid sort field that the user can search agnaist.
sort_field = request.session['sortfield']
sort_field_order = request.session['sortfieldby']
page_num = request.session['page']
if request.session['search'] is not None:
node_params = {
'query':
{
1: request.session['search']
},
}
else:
node_params = {
'query': {},
}
nodes_sort_fields = ['certname', 'catalog_timestamp', 'report_timestamp', 'facts_timestamp']
if sort_field in nodes_sort_fields:
node_params['order_by'] = {
'order_field':
{
'field': sort_field,
'order': sort_field_order,
},
}
if dl_csv is False:
node_params['limit'] = request.session['limits']
node_params['offset'] = request.session['offset']
node_params['include_total'] = 'true'
else:
node_params['order_by'] = {
'order_field':
{
'field': 'report_timestamp',
'order': 'desc',
},
}
node_sort_fields = ['certname', 'catalog_timestamp', 'report_timestamp', 'facts_timestamp']
if sort_field in node_sort_fields:
try:
node_list, node_headers = puppetdb.api_get(
api_url=source_url,
cert=source_certs,
verify=source_verify,
path='/nodes',
api_version='v4',
params=puppetdb.mk_puppetdb_query(
node_params, request),
)
except:
node_list = []
node_headers = dict()
node_headers['X-Records'] = 0
else:
node_list = puppetdb.api_get(
api_url=source_url,
cert=source_certs,
verify=source_verify,
path='/nodes',
api_version='v4',
params=puppetdb.mk_puppetdb_query(
node_params, request),
)
# Work out the number of pages from the xrecords response
# return fields that you can sort by
# for each node in the node_list, find out if the latest run has any failures
# v3/event-counts --data-urlencode query='["=","latest-report?",true]'
# --data-urlencode summarize-by='certname'
report_params = {
'query':
{
1: '["and",["=","latest_report?",true],["in", "certname",["extract", "certname",["select_nodes",["null?","deactivated",true]]]]]'
},
'summarize_by': 'certname',
}
status_sort_fields = ['successes', 'failures', 'skips', 'noops']
report_status_params = {
'query':
{
1: '["and",["=","latest_report?",true],["in", "certname",["extract", "certname",["select_nodes",["null?","deactivated",true]]]]]'
}
}
report_status_list = puppetdb.api_get(
api_url=source_url,
cert=source_certs,
verify=source_verify,
path='/reports',
params=puppetdb.mk_puppetdb_query(report_status_params, request),
api_version='v4',
)
if sort_field in status_sort_fields:
if request.session['search'] is not None:
report_params['query'] = {'operator': 'and',
1: request.session['search'],
2: '["=","latest_report?",true]',
3: '["in", "certname",["extract", "certname",["select_nodes",["null?","deactivated",true]]]]',
}
report_params['order_by'] = {
'order_field':
{
'field': sort_field,
'order': sort_field_order,
}
}
report_params['include_total'] = 'true'
# Don't limit results if its CSV
if dl_csv is False:
report_params['limit'] = request.session['limits']
report_params['offset'] = request.session['offset']
report_list, report_headers = puppetdb.api_get(
api_url=source_url,
cert=source_certs,
verify=source_verify,
path='/event-counts',
params=puppetdb.mk_puppetdb_query(report_params, request),
api_version='v4',
)
else:
report_list = puppetdb.api_get(
api_url=source_url,
cert=source_certs,
verify=source_verify,
path='event-counts',
params=puppetdb.mk_puppetdb_query(report_params, request),
api_version='v4',
)
# number of results depending on sort field.
if sort_field in status_sort_fields:
xrecords = report_headers['X-Records']
total_results = xrecords
elif sort_field in nodes_sort_fields:
xrecords = node_headers['X-Records']
total_results = xrecords
num_pages_wdec = float(xrecords) / request.session['limits']
num_pages_wodec = float("{:.0f}".format(num_pages_wdec))
if num_pages_wdec > num_pages_wodec:
num_pages = num_pages_wodec + 1
else:
num_pages = num_pages_wodec
# Converts lists of dicts to dicts.
node_dict = {item['certname']: item for item in node_list}
status_dict = {item['certname']: item for item in report_status_list}
report_dict = {item['subject']['title']: item for item in report_list}
if sort_field_order == 'desc':
rows = dictstatus(
node_dict, status_dict, report_dict, sortby=sort_field, asc=True, sort=False)
sort_field_order_opposite = 'asc'
elif sort_field_order == 'asc':
rows = dictstatus(
node_dict, status_dict, report_dict, sortby=sort_field, asc=False, sort=False)
sort_field_order_opposite = 'desc'
if dl_csv is True:
if rows is []:
pass
else:
# Generate a sequence of rows. The range is based on the maximum number of
# rows that can be handled by a single sheet in most spreadsheet
# applications.
include_facts = request.GET.get('include_facts', False)
csv_headers = ['Certname',
'Latest Catalog',
'Latest Report',
'Latest Facts',
'Success',
'Noop',
'Failure',
'Skipped']
if include_facts is not False:
merged_list_facts = []
facts = {}
for fact in include_facts.split(','):
# Sanitize the fact input from the user
fact = fact.strip()
# Add the fact name to the headers list
csv_headers.append(fact)
# build the params for each fact.
facts_params = {
'query':
{
1: '["=","name","' + fact + '"]'
},
}
fact_list = puppetdb.api_get(
api_url=source_url,
cert=source_certs,
verify=source_verify,
path='facts',
params=puppetdb.mk_puppetdb_query(facts_params),
api_version='v4',
)
# Populate the facts dict with the facts we have retrieved
# Convert the fact list into a fact dict!
facts[fact] = {item['certname']: item for item in fact_list}
i = 1
jobs = {}
# Add ID to each job so that it can be assembled in
# the same order after we recieve the job results
# We do this via jobs so that we can get faster results.
for node in rows:
jobs[i] = {
'id': i,
'include_facts': include_facts.split(','),
'node': node,
'facts': facts,
}
i += 1
csv_results = generate_csv(jobs)
rows = []
i = 1
# with the job results we can now recreate merged_list
# in the order we sent them.
while i <= len(csv_results):
rows.append(csv_results[i])
i += 1
# Insert the csv header to the top of the list.
rows.insert(0, csv_headers)
pseudo_buffer = Echo()
writer = csv.writer(pseudo_buffer)
response = StreamingHttpResponse((writer.writerow(row) for row in rows),
content_type="text/csv")
response['Content-Disposition'] = 'attachment; filename="puppetdata-%s.csv"' % (datetime.datetime.now())
return response
"""
c_r_s* = current request sort
c_r_* = current req
r_s* = requests available
"""
context = {
'nodeList': rows,
'total_nodes': total_results,
'c_r_page': page_num,
'c_r_limit': request.session['limits'],
'r_sfield': valid_sort_fields,
'c_r_sfield': sort_field,
'r_sfieldby': ['asc', 'desc'],
'c_r_sfieldby': sort_field_order,
'c_r_sfieldby_o': sort_field_order_opposite,
'tot_pages': '{0:g}'.format(num_pages),
}
return HttpResponse(json.dumps(context), content_type="application/json")
def catalogue_json(request, certname=None):
context = dict()
data = dict()
if not certname:
context['error'] = 'Must specify certname.'
return HttpResponse(json.dumps(context), content_type="application/json")
source_url, source_certs, source_verify = get_server(request)
# Redirects to the events page if GET param latest is true..
show = request.GET.get('show', 'edges')
save_catalog = request.GET.get('save', 'false')
catalogue_params = {}
path = '/catalogs/%s' % certname
catalogue = puppetdb.api_get(
path=path,
api_url=source_url,
api_version='v4',
params=puppetdb.mk_puppetdb_query(catalogue_params, request),
)
if 'error' not in catalogue and save_catalog == 'true':
report_param = {
'query':
{
'operator': 'and',
1: '["=","latest_report?",true]',
2: '["=","certname","%s"]' % certname
}
}
report_url = '/reports'
latest_report = puppetdb.api_get(
path=report_url,
api_url=source_url,
api_version='v4',
params=puppetdb.mk_puppetdb_query(report_param, request),
)
report_hash = latest_report[0]['hash']
catalogue_hash = catalogue['hash']
catalogue_timestamp = catalogue['producer_timestamp']
try:
saved_catalogue = SavedCatalogs.objects.get(hostname=certname, catalogue_id=catalogue_hash)
if saved_catalogue.linked_report != report_hash:
# Grab the linked report from the result set.
old_linked_report = saved_catalogue.linked_report
# Update the data.
saved_catalogue.linked_report = report_hash
saved_catalogue.timestamp = catalogue_timestamp
# Save the new data.
saved_catalogue.save()
data['success'] = 'Catalogue hash updated.'
data['certname'] = certname
data['old_linked_report'] = old_linked_report
data['new_linked_report'] = report_hash
return HttpResponse(json.dumps(data, indent=2), content_type='application/json')
else:
data['error'] = 'Catalogue hash already exists.'
data['certname'] = certname
data['catalogue_hash'] = catalogue_hash
data['linked_report'] = saved_catalogue.linked_report
return HttpResponseBadRequest(json.dumps(data, indent=2), content_type='application/json')
except SavedCatalogs.DoesNotExist:
# since we couldnt find it in the db its safe to asusme that we can create it!
SavedCatalogs.objects.create(hostname=certname,
catalogue_id=catalogue_hash,
linked_report=report_hash,
timestamp=catalogue_timestamp,
catalogue=json.dumps(catalogue))
data['success'] = 'Saved catalogue.'
data['certname'] = certname
data['catalogue_hash'] = catalogue_hash
data['linked_report'] = report_hash
return HttpResponse(json.dumps(data, indent=2), content_type='application/json')
if show == 'edges':
data['data'] = catalogue['edges']['data']
elif show == 'resources':
data['data'] = catalogue['resources']['data']
else:
data['data'] = catalogue
return HttpResponse(json.dumps(data, indent=2), content_type='application/json')
