def catalogue_compare_json(request, certname1=None, certname2=None): source_url, source_certs, source_verify = get_server(request) show = request.GET.get('show', 'edges') if show is not None and show in ['edges', 'resources']: if show == 'edges': sort_field = 'source_title' elif show == 'resources': sort_field = 'title' certname1_params = { 'order_by': { 'order_field': { 'field': sort_field, 'order': 'asc', }, } } certname2_params = { 'order_by': { 'order_field': { 'field': sort_field, 'order': 'asc', }, } } certname1_data = puppetdb.api_get( path='/catalogs/%s/%s' % (certname1, show), api_url=source_url, api_version='v4', params=puppetdb.mk_puppetdb_query(certname1_params, request), ) certname1_data = json.dumps(certname1_data, indent=2) certname2_data = puppetdb.api_get( path='/catalogs/%s/%s' % (certname2, show), api_url=source_url, api_version='v4', params=puppetdb.mk_puppetdb_query(certname2_params, request), ) certname2_data = json.dumps(certname2_data, indent=2) from_split_lines = certname1_data.split('\n') to_split_lines = certname2_data.split('\n') diff = difflib.unified_diff(from_split_lines, to_split_lines) diff = ('\n'.join(list(diff))) return HttpResponse(diff)
def facts(request, certname=None): context = { 'timezones': pytz.common_timezones, 'SOURCES': AVAILABLE_SOURCES } if request.method == 'GET': if 'source' in request.GET: source = request.GET.get('source') set_server(request, source) if request.method == 'POST': request.session['django_timezone'] = request.POST['timezone'] return redirect(request.POST['return_url']) source_url, source_certs, source_verify = get_server(request) facts_params = { 'query': { 1: '["=","certname","' + certname + '"]' }, } facts_list = puppetdb.api_get( api_url=source_url, cert=source_certs, verify=source_verify, path='facts', params=puppetdb.mk_puppetdb_query(facts_params, request), ) context['certname'] = certname context['facts_list'] = facts_list return render(request, 'pano/facts.html', context)
def search_nodes_json(request): source_url, source_certs, source_verify = get_server(request) if request.method == 'GET': if 'search' in request.GET: search = request.GET.get('search') # Create a search regex for certname spelt with below. nodes_params = { 'query': { 1: '["~","certname","' + search + '"]' }, 'order_by': { 'order_field': { 'field': 'certname', 'order': 'desc', }, }, #'limit': 25, } nodes_list = puppetdb.api_get( api_url=source_url, cert=source_certs, verify=source_verify, path='/nodes', api_version='v4', params=puppetdb.mk_puppetdb_query( nodes_params, request), ) return HttpResponse(json.dumps(nodes_list), content_type="application/json")
def facts(request, certname=None): context = {'timezones': pytz.common_timezones, 'SOURCES': AVAILABLE_SOURCES} if request.method == 'GET': if 'source' in request.GET: source = request.GET.get('source') set_server(request, source) if request.method == 'POST': request.session['django_timezone'] = request.POST['timezone'] return redirect(request.POST['return_url']) source_url, source_certs, source_verify = get_server(request) facts_params = { 'query': { 1: '["=","certname","' + certname + '"]' }, } facts_list = puppetdb.api_get( api_url=source_url, cert=source_certs, verify=source_verify, path='facts', params=puppetdb.mk_puppetdb_query( facts_params, request), ) context['certname'] = certname context['facts_list'] = facts_list return render(request, 'pano/facts.html', context)
def get_report(key, value, request): source_url, source_certs, source_verify = get_server(request) # If key is any of the below, all is good! allowed_keys = [ 'certname', 'resource_title', 'resource_type', 'containing_class' ] if key in allowed_keys: pass # If key does not match above the default will be shown else: key = 'containing_class' events_params = { 'query': { 'operator': 'and', 1: '["=","' + key + '","' + value + '"]', 2: '["=","latest_report?",true]', 3: '["in", "certname",["extract", "certname",["select_nodes",["null?","deactivated",true]]]]' }, } results = pdb_api_get( api_url=source_url, cert=source_certs, verify=source_verify, path='/events', api_version='v4', params=mk_puppetdb_query(events_params, request), ) return results
def reports(request, certname=None): context = {'timezones': pytz.common_timezones, 'SOURCES': AVAILABLE_SOURCES} if request.method == 'GET': if 'source' in request.GET: source = request.GET.get('source') set_server(request, source) if request.method == 'POST': request.session['django_timezone'] = request.POST['timezone'] return redirect(request.POST['return_url']) source_url, source_certs, source_verify = get_server(request) # Redirects to the events page if GET param latest is true.. if request.GET.get('latest', False): if request.GET.get('latest') == "true": request.session['report_page'] = 1 latest_report_params = { 'query': { 1: '["=","certname","' + certname + '"]' }, 'order_by': { 'order_field': { 'field': 'start_time', 'order': 'desc', }, }, 'limit': 1, } latest_report = puppetdb.api_get( api_url=source_url, cert=source_certs, verify=source_verify, path='/reports', api_version='v4', params=puppetdb.mk_puppetdb_query(latest_report_params, request), ) report_hash = "" # If latest reports do not exist, send to the nodes page # Should only occur if the user is trying to hax their way # into a node without having the correct permission if latest_report: for report in latest_report: report_env = report['environment'] report_hash = report['hash'] return redirect('/pano/events/' + report_hash + '?report_timestamp=' + request.GET.get( 'report_timestamp') + '&envname=' + report_env) else: return redirect('/pano/nodes/') if certname != request.session.get('last_shown_node', ''): request.session['last_shown_node'] = certname request.session['report_page'] = 1 context['certname'] = certname context['node_facts'] = ','.join(NODES_DEFAULT_FACTS) return render(request, 'pano/reports.html', context)
def test_summarize_by_query(self): content = { 'summarize_by': 'containing_class', } expected_results = {'summarize_by': 'containing_class'} results = mk_puppetdb_query(content) self.assertEqual(expected_results, results)
def reports(request, certname=None): context = { 'timezones': pytz.common_timezones, 'SOURCES': AVAILABLE_SOURCES } if request.method == 'GET': if 'source' in request.GET: source = request.GET.get('source') set_server(request, source) if request.method == 'POST': request.session['django_timezone'] = request.POST['timezone'] return redirect(request.POST['return_url']) source_url, source_certs, source_verify = get_server(request) # Redirects to the events page if GET param latest is true.. if request.GET.get('latest', False): if request.GET.get('latest') == "true": latest_report_params = { 'query': { 1: '["=","certname","' + certname + '"]' }, 'order-by': { 'order-field': { 'field': 'start-time', 'order': 'desc', }, 'query-field': { 'field': 'certname' }, }, 'limit': 1, } latest_report = puppetdb.api_get( api_url=source_url, cert=source_certs, verify=source_verify, path='/reports', api_version='v4', params=puppetdb.mk_puppetdb_query(latest_report_params, request), ) report_hash = "" # If latest reports do not exist, send to the nodes page # Should only occur if the user is trying to hax their way # into a node without having the correct permission if latest_report: for report in latest_report: report_env = report['environment'] report_hash = report['hash'] return redirect('/pano/events/' + report_hash + '?report_timestamp=' + request.GET.get('report_timestamp') + '&envname=' + report_env) else: return redirect('/pano/nodes/') context['certname'] = certname context['node_facts'] = ','.join(NODES_DEFAULT_FACTS) return render(request, 'pano/reports.html', context)
def get_report(key, value, request): source_url, source_certs, source_verify = get_server(request) # If key is any of the below, all is good! allowed_keys = ['certname', 'resource_title', 'resource_type', 'containing_class'] if key in allowed_keys: pass # If key does not match above the default will be shown else: key = 'containing_class' events_params = { 'query': { 'operator': 'and', 1: '["=","' + key + '","' + value + '"]', 2: '["=","latest_report?",true]', 3: '["in", "certname",["extract", "certname",["select_nodes",["null?","deactivated",true]]]]' }, } results = pdb_api_get( api_url=source_url, cert=source_certs, verify=source_verify, path='/events', api_version='v4', params=mk_puppetdb_query(events_params, request), ) return results
def test_summarize_by_query(self): content = { 'summarize-by': 'containing-class', } expected_results = { 'summarize-by': 'containing-class' } results = mk_puppetdb_query(content) self.assertEqual(expected_results, results)
def facts_json(request): context = {} if request.method == "GET": if "source" in request.GET: source = request.GET.get("source") set_server(request, source) if request.method == "POST": request.session["django_timezone"] = request.POST["timezone"] return redirect(request.POST["return_url"]) source_url, source_certs, source_verify = get_server(request) certname = None facts = None if "certname" in request.GET: certname = request.GET.get("certname") if "facts" in request.GET: facts = request.GET.get("facts").split(",") if not certname: context["error"] = "Certname not specified." return HttpResponse(json.dumps(context)) if facts: # validate string for illegal chars fact_query = list() for fact in facts: fact = fact.strip() # Match for characters that are not a-Z or 0-9 or _ # if theres a match illegal chars exist... regex = re.compile(r"[^aA-zZ0-9_]") matches = regex.findall(fact) if matches: context["error"] = "Illegal characters found in facts list. " context["error"] += "Facts must not match anything withddd this regex <[^aA-zZ0-9_]>." return HttpResponse(json.dumps(context)) fact_query.append('["=","name","' + fact + '"]') fact_query = ",".join(fact_query) facts_params = { "query": {1: '["and",["=","certname","' + certname + '"],["or",' + fact_query + "]]"}, "order-by": {"order_field": {"field": "name", "order": "asc"}}, } else: facts_params = { "query": {1: '["=","certname","' + certname + '"]'}, "order_by": {"order-field": {"field": "name", "order": "asc"}}, } facts_list = puppetdb.api_get( api_url=source_url, cert=source_certs, verify=source_verify, path="facts", params=puppetdb.mk_puppetdb_query(facts_params, request), ) context["certname"] = certname context["facts_list"] = facts_list return HttpResponse(json.dumps(context), content_type="application/json")
def test_single_search_query(self): content = { 'query': { 1: '["=","certname","hostname.example.com"]' }, } expected_results = { 'query': '["and",["=","certname","hostname.example.com"]]' } results = mk_puppetdb_query(content) self.assertEqual(expected_results, results)
def test_single_search_query_with_operator(self): content = { 'query': { 'operator': 'and', 1: '["=","hash","e4fug294hf3293hf9348g3804hg3084h"]', }, } expected_results = { 'query': '["and",["=","hash","e4fug294hf3293hf9348g3804hg3084h"]]' } results = mk_puppetdb_query(content) self.assertEqual(expected_results, results)
def test_single_search_query(self): content = { 'query': { 1: '["=","certname","hostname.example.com"]' }, } expected_results = { 'query': '["=","certname","hostname.example.com"]' } results = mk_puppetdb_query(content) self.assertEqual(expected_results, results)
def test_single_search_query_with_operator(self): content = { 'query': { 'operator': 'and', 1: '["=","hash","e4fug294hf3293hf9348g3804hg3084h"]', }, } expected_results = { 'query': '["and", ["=","hash","e4fug294hf3293hf9348g3804hg3084h"]]' } results = mk_puppetdb_query(content) self.assertEqual(expected_results, results)
def test_summarize_by_query_with_single_search_query(self): content = { 'query': { 1: '["=","certname","hostname.example.com"]' }, 'summarize_by': 'containing_class', } expected_results = { 'query': '["and",["=","certname","hostname.example.com"]]', 'summarize_by': 'containing_class' } results = mk_puppetdb_query(content) self.assertEqual(expected_results, results)
def test_order_by_query(self): content = { 'order_by': { 'order_field': { 'field': 'report_timestamp', 'order': 'desc', }, } } expected_results = { 'order_by': '[{"field":"report_timestamp","order":"desc"}]' } results = mk_puppetdb_query(content) self.assertEqual(expected_results, results)
def test_summarize_by_query_with_single_search_query(self): content = { 'query': { 1: '["=","certname","hostname.example.com"]' }, 'summarize-by': 'containing-class', } expected_results = { 'query': '["=","certname","hostname.example.com"]', 'summarize-by': 'containing-class' } results = mk_puppetdb_query(content) self.assertEqual(expected_results, results)
def test_order_by_query(self): content = { 'order-by': { 'order-field': { 'field': 'report_timestamp', 'order': 'desc', }, } } expected_results = { 'order-by': '[{"field":"report_timestamp","order":"desc"}]' } results = mk_puppetdb_query(content) self.assertEqual(expected_results, results)
def get_resource(certname, rtype, rtitle): resource_params = { 'query': { 'operator': 'and', 1: '["=", "certname", "' + certname + '"]', 2: '["=", "type", "' + rtype + '"]', 3: '["=", "title", "' + rtitle + '"]' }, } data = pdb_api_get(api_url=puppetdb_source, path='resources', verify=puppetdb_verify, cert=puppetdb_certs, params=mk_puppetdb_query(resource_params, request)) if not data: return False else: return data
def test_order_by_query_with_single_search_query(self): content = { 'query': { 1: '["=","certname","hostname.example.com"]' }, 'order_by': { 'order_field': { 'field': 'report_timestamp', 'order': 'desc', }, } } expected_results = { 'order_by': '[{"field":"report_timestamp","order":"desc"}]', 'query': '["and",["=","certname","hostname.example.com"]]' } results = mk_puppetdb_query(content) self.assertEqual(expected_results, results)
def get_events_summary(request, timespan='latest'): if timespan == 'latest': events_params = { 'query': { 1: '["and",["=","latest-report?",true],["in", "certname",["extract", "certname",["select-nodes",["null?","deactivated",true]]]]]' }, } source_url, source_certs, source_verify = get_server(request) events = pdb_api_get( api_url=source_url, cert=source_certs, verify=source_verify, path='events/', api_version='v4', params=mk_puppetdb_query(events_params)) summary = summary_of_events(events) return summary
def db_threaded_requests(i, q): while True: t_job = q.get() t_path = t_job['path'] t_url = t_job.get('url') t_certs = t_job.get('certs') t_verify = t_job.get('verify') t_params = t_job.get('params', {}) t_api_v = t_job.get('api_version', 'v3') results = puppetdb.api_get( api_url=t_url, verify=t_verify, cert=t_certs, path=t_path, params=puppetdb.mk_puppetdb_query(t_params), api_version=t_api_v, ) out_q.put({t_job['id']: results}) q.task_done()
def db_threaded_requests(i, q): while True: t_job = q.get() t_path = t_job['path'] t_url = t_job.get('url') t_certs = t_job.get('certs') t_verify = t_job.get('verify') t_params = t_job.get('params', {}) t_api_v = t_job.get('api_version', 'v3') t_request = t_job.get('request') results = puppetdb.api_get( api_url=t_url, verify=t_verify, cert=t_certs, path=t_path, params=puppetdb.mk_puppetdb_query(t_params, t_request), api_version=t_api_v, ) out_q.put({t_job['id']: results}) q.task_done()
def test_order_by_query_with_single_search_query(self): content = { 'query': { 1: '["=","certname","hostname.example.com"]' }, 'order-by': { 'order-field': { 'field': 'report_timestamp', 'order': 'desc', }, } } expected_results = { 'order-by': '[{"field":"report_timestamp","order":"desc"}]', 'query': '["=","certname","hostname.example.com"]' } results = mk_puppetdb_query(content) self.assertEqual(expected_results, results)
def get_events_summary(request, timespan='latest'): if timespan == 'latest': events_params = { 'query': { 'operator': 'and', 1: '["=","latest_report?",true]', 2: '["in","certname",["extract","certname",["select_nodes",["null?","deactivated",true]]]]', }, } source_url, source_certs, source_verify = get_server(request) events = pdb_api_get(api_url=source_url, cert=source_certs, verify=source_verify, path='/events', api_version='v4', params=mk_puppetdb_query(events_params, request)) summary = summary_of_events(events) return summary
def get_resource(certname, rtype, rtitle): resource_params = { 'query': { 'operator': 'and', 1: '["=", "certname", "' + certname + '"]', 2: '["=", "type", "' + rtype + '"]', 3: '["=", "title", "' + rtitle + '"]' }, } data = pdb_api_get( api_url=puppetdb_source, path='resources', verify=puppetdb_verify, cert=puppetdb_certs, params=mk_puppetdb_query(resource_params, request)) if not data: return False else: return data
def test_order_by_query_with_double_search_query_with_operator(self): content = { 'query': { 'operator': 'or', 1: '["=","certname","hostname1.example.com"]', 2: '["=","certname","hostname2.example.com"]' }, 'order_by': { 'order_field': { 'field': 'report_timestamp', 'order': 'desc', }, } } expected_results = { 'query': '["and",["=","certname","hostname1.example.com"],["=","certname","hostname2.example.com"]]', 'order_by': '[{"field":"report_timestamp","order":"desc"}]' } results = mk_puppetdb_query(content) self.assertEqual(expected_results, results)
def catalogue_json(request, certname=None): context = dict() if not certname: context['error'] = 'Must specify certname.' return HttpResponse(json.dumps(context), content_type="application/json") source_url, source_certs, source_verify = get_server(request) # Redirects to the events page if GET param latest is true.. show = request.GET.get('show', None) reports_params = {} if show is not None and show in ['edges', 'resources']: if show == 'edges': sort_field = 'source_title' elif show == 'resources': sort_field = 'title' reports_params = { 'order_by': { 'order_field': { 'field': sort_field, 'order': 'asc', }, } } path = '/catalogs/%s/%s' % (certname, show) else: path = '/catalogs/%s' % certname reports_list = puppetdb.api_get( path=path, api_url=source_url, api_version='v4', params=puppetdb.mk_puppetdb_query(reports_params, request), ) data = {'data': reports_list} return HttpResponse(json.dumps(data, indent=2), content_type="application/json")
def get_events_summary(request, timespan='latest', environment=None): events_params = { 'query': { 'operator': 'and', 2: '["in","certname",["extract","certname",["select_nodes",["null?","deactivated",true]]]]', }, } if timespan == 'latest': events_params['query'][1] = '["=","latest_report?",true]' elif len(timespan) == 2: events_params['query'][1] = '["and",[">","timestamp","' + timespan[0] + '"],["<", "timestamp", "' + timespan[ 1] + '"]]' source_url, source_certs, source_verify = get_server(request) events = pdb_api_get( api_url=source_url, cert=source_certs, verify=source_verify, path='/events', api_version='v4', params=mk_puppetdb_query(events_params, request)) summary = summary_of_events(events) return summary
def reports_search_json(request): context = dict() if request.method == 'GET': if 'search' in request.GET: search = request.GET.get('search') if 'certname' in request.GET: certname = request.GET.get('certname') if not certname or not search: context['error'] = 'Must specify both certname and search query.' return HttpResponse(json.dumps(context), content_type="application/json") source_url, source_certs, source_verify = get_server(request) # Redirects to the events page if GET param latest is true.. reports_params = { 'query': { 'operator': 'and', 1: '["=","certname","' + certname + '"]', 2: '["~","hash","^' + search + '"]' }, 'order_by': { 'order_field': { 'field': 'start_time', 'order': 'desc', }, } } reports_list = puppetdb.api_get( path='/reports', api_url=source_url, api_version='v4', params=puppetdb.mk_puppetdb_query(reports_params, request), ) return HttpResponse(json.dumps(reports_list), content_type="application/json")
def reports(request, certname=None): context = default_context if request.method == "GET": if "source" in request.GET: source = request.GET.get("source") set_server(request, source) if request.method == "POST": request.session["django_timezone"] = request.POST["timezone"] return redirect(request.POST["return_url"]) source_url, source_certs, source_verify = get_server(request) # Redirects to the events page if GET param latest is true.. if request.GET.get("latest", False): if request.GET.get("latest") == "true": latest_report_params = { "query": {1: '["=","certname","' + certname + '"]'}, "order-by": { "order-field": {"field": "start-time", "order": "desc"}, "query-field": {"field": "certname"}, }, "limit": 1, } latest_report = puppetdb.api_get( api_url=source_url, cert=source_certs, verify=source_verify, path="/reports", api_version="v4", params=puppetdb.mk_puppetdb_query(latest_report_params), ) report_hash = "" for report in latest_report: report_env = report["environment"] report_hash = report["hash"] return redirect( "/pano/events/" + certname + "/" + report_hash + "?report_timestamp=" + request.GET.get("report_timestamp") + "&envname=" + report_env ) page_num = int(request.GET.get("page", 0)) if page_num <= 0: offset = 0 else: offset = "{:.0f}".format(page_num * 25) reports_params = { "query": {1: '["=","certname","' + certname + '"]'}, "order-by": {"order-field": {"field": "start-time", "order": "desc"}, "query-field": {"field": "certname"}}, "limit": 25, "include-total": "true", "offset": offset, } reports_list, headers = puppetdb.api_get( api_url=source_url, cert=source_certs, verify=source_verify, path="/reports", api_version="v4", params=puppetdb.mk_puppetdb_query(reports_params), ) # Work out the number of pages from the xrecords response xrecords = headers["X-Records"] num_pages_wdec = float(xrecords) / 25 num_pages_wodec = float("{:.0f}".format(num_pages_wdec)) if num_pages_wdec > num_pages_wodec: num_pages = num_pages_wodec + 1 else: num_pages = num_pages_wodec report_status = [] for report in reports_list: found_report = False events_params = {"query": {1: '["=","report","' + report["hash"] + '"]'}, "summarize-by": "certname"} eventcount_list = puppetdb.api_get( path="event-counts", api_url=source_url, api_version="v4", params=puppetdb.mk_puppetdb_query(events_params) ) # Make list of the results for event in eventcount_list: if event["subject"]["title"] == report["certname"]: found_report = True # hashid, certname, environment, time start, time end, success, noop, failure, pending report_status.append( [ report["hash"], report["certname"], report["environment"], report["start-time"], report["end-time"], event["successes"], event["noops"], event["failures"], event["skips"], ] ) break if found_report is False: report_status.append( [ report["hash"], report["certname"], report["environment"], report["start-time"], report["end-time"], 0, 0, 0, 0, ] ) report_status = sort_tables(report_status, order=True, col=3) context["certname"] = certname context["reports"] = report_status context["curr_page"] = page_num context["tot_pages"] = "{:.0f}".format(num_pages) return render(request, "pano/reports.html", context)
def facts_json(request): context = {} if request.method == 'GET': if 'source' in request.GET: source = request.GET.get('source') set_server(request, source) if request.method == 'POST': request.session['django_timezone'] = request.POST['timezone'] return redirect(request.POST['return_url']) source_url, source_certs, source_verify = get_server(request) certname = None facts = None if 'certname' in request.GET: certname = request.GET.get('certname') if 'facts' in request.GET: facts = request.GET.get('facts').split(',') if not certname: context['error'] = 'Certname not specified.' return HttpResponse(json.dumps(context)) if facts: # validate string for illegal chars fact_query = list() for fact in facts: fact = fact.strip() # Match for characters that are not a-Z or 0-9 or _ # if theres a match illegal chars exist... regex = re.compile(r'[^aA-zZ0-9_]') matches = regex.findall(fact) if matches: context['error'] = 'Illegal characters found in facts list. ' context[ 'error'] += 'Facts must not match anything withddd this regex <[^aA-zZ0-9_]>.' return HttpResponse(json.dumps(context)) fact_query.append('["=","name","' + fact + '"]') fact_query = ','.join(fact_query) facts_params = { 'query': { 1: '["and",["=","certname","' + certname + '"],["or",' + fact_query + ']]' }, 'order-by': { 'order_field': { 'field': 'name', 'order': 'asc', } } } else: facts_params = { 'query': { 1: '["=","certname","' + certname + '"]' }, 'order_by': { 'order-field': { 'field': 'name', 'order': 'asc', } } } facts_list = puppetdb.api_get( api_url=source_url, cert=source_certs, verify=source_verify, path='facts', params=puppetdb.mk_puppetdb_query(facts_params, request), ) context['certname'] = certname context['facts_list'] = facts_list return HttpResponse(json.dumps(context), content_type="application/json")
def facts_json(request): context = {} if request.method == 'GET': if 'source' in request.GET: source = request.GET.get('source') set_server(request, source) if request.method == 'POST': request.session['django_timezone'] = request.POST['timezone'] return redirect(request.POST['return_url']) source_url, source_certs, source_verify = get_server(request) certname = None facts = None if 'certname' in request.GET: certname = request.GET.get('certname') if 'facts' in request.GET: facts = request.GET.get('facts').split(',') if not certname: context['error'] = 'Certname not specified.' return HttpResponse(json.dumps(context)) if facts: # validate string for illegal chars fact_query = list() for fact in facts: fact = fact.strip() # Match for characters that are not a-Z or 0-9 or _ # if theres a match illegal chars exist... regex = re.compile(r'[^aA-zZ0-9_]') matches = regex.findall(fact) if matches: context['error'] = 'Illegal characters found in facts list. ' context['error'] += 'Facts must not match anything withddd this regex <[^aA-zZ0-9_]>.' return HttpResponse(json.dumps(context)) fact_query.append('["=","name","' + fact + '"]') fact_query = ','.join(fact_query) facts_params = { 'query': { 1: '["and",["=","certname","' + certname + '"],["or",' + fact_query + ']]' }, 'order-by': { 'order_field': { 'field': 'name', 'order': 'asc', } } } else: facts_params = { 'query': { 1: '["=","certname","' + certname + '"]' }, 'order_by': { 'order-field': { 'field': 'name', 'order': 'asc', } } } facts_list = puppetdb.api_get( api_url=source_url, cert=source_certs, verify=source_verify, path='facts', params=puppetdb.mk_puppetdb_query( facts_params, request), ) context['certname'] = certname context['facts_list'] = facts_list return HttpResponse(json.dumps(context, indent=2), content_type="application/json")
def reports_json(request, certname=None): source_url, source_certs, source_verify = get_server(request) # Redirects to the events page if GET param latest is true.. context = {} # Cur Page Number if request.GET.get('page', False): if request.session['report_page'] != int(request.GET.get('page', 1)): request.session['report_page'] = int(request.GET.get('page', 1)) if request.session['report_page'] <= 0: request.session['report_page'] = 1 else: if 'report_page' not in request.session: request.session['report_page'] = 1 if request.session['report_page'] <= 0: offset = 0 else: offset = (25 * request.session['report_page']) - 25 reports_params = { 'query': { 1: '["=","certname","' + certname + '"]' }, 'order-by': { 'order-field': { 'field': 'start_time', 'order': 'desc', }, 'query-field': {'field': 'certname'}, }, 'limit': 25, 'include_total': 'true', 'offset': offset, } reports_list, headers = puppetdb.api_get( api_url=source_url, cert=source_certs, verify=source_verify, path='/reports', api_version='v4', params=puppetdb.mk_puppetdb_query( reports_params, request), ) # Work out the number of pages from the xrecords response xrecords = headers['X-Records'] num_pages_wdec = float(xrecords) / 25 num_pages_wodec = float("{:.0f}".format(num_pages_wdec)) if num_pages_wdec > num_pages_wodec: num_pages = num_pages_wodec + 1 else: num_pages = num_pages_wodec report_status = [] for report in reports_list: found_report = False events_params = { 'query': { 1: '["=","report","' + report['hash'] + '"]' }, 'summarize_by': 'certname', } eventcount_list = puppetdb.api_get( path='event-counts', api_url=source_url, api_version='v4', params=puppetdb.mk_puppetdb_query(events_params, request), ) # Make list of the results for event in eventcount_list: if event['subject']['title'] == report['certname']: found_report = True report_status.append({ 'hash': report['hash'], 'certname': report['certname'], 'environment': report['environment'], 'start_time': filters.date(localtime(json_to_datetime(report['start_time'])), 'Y-m-d H:i:s'), 'end_time': filters.date(localtime(json_to_datetime(report['end_time'])), 'Y-m-d H:i:s'), 'events_successes': event['successes'], 'events_noops': event['noops'], 'events_failures': event['failures'], 'events_skipped': event['skips'], 'report_status': report['status'], 'config_version': report['configuration_version'], 'run_duration': "{0:.0f}".format( (json_to_datetime(report['end_time']) - json_to_datetime(report['start_time'])).total_seconds()) }) break if found_report is False: report_status.append({ 'hash': report['hash'], 'certname': report['certname'], 'environment': report['environment'], 'start_time': filters.date(localtime(json_to_datetime(report['start_time'])), 'Y-m-d H:i:s'), 'end_time': filters.date(localtime(json_to_datetime(report['end_time'])), 'Y-m-d H:i:s'), 'events_successes': 0, 'events_noops': 0, 'events_failures': 0, 'events_skipped': 0, 'report_status': report['status'], 'config_version': report['configuration_version'], 'run_duration': "{0:.0f}".format( (json_to_datetime(report['end_time']) - json_to_datetime(report['start_time'])).total_seconds()) }) context['certname'] = certname context['reports_list'] = report_status context['curr_page'] = request.session['report_page'] context['tot_pages'] = "{:.0f}".format(num_pages) return HttpResponse(json.dumps(context), content_type="application/json")
def detailed_events(request, hashid=None): context = { 'timezones': pytz.common_timezones, 'SOURCES': AVAILABLE_SOURCES } if request.method == 'GET': if 'source' in request.GET: source = request.GET.get('source') set_server(request, source) if request.method == 'POST': request.session['django_timezone'] = request.POST['timezone'] return redirect(request.POST['return_url']) source_url, source_certs, source_verify = get_server(request) report_timestamp = request.GET.get('report_timestamp') events_params = { 'query': { 1: '["=","report","' + hashid + '"]' }, 'order_by': { 'order_field': { 'field': 'timestamp', 'order': 'asc', }, 'query_field': { 'field': 'certname' }, }, } events_list = puppetdb.api_get( api_url=source_url, cert=source_certs, verify=source_verify, path='/events', api_version='v4', params=puppetdb.mk_puppetdb_query(events_params), ) environment = '' certname = '' event_execution_times = [] sorted_events = None last_event_time = None last_event_title = None run_end_time = None if len(events_list) != 0: single_event = events_list[0] environment = single_event['environment'] certname = single_event['certname'] for event in events_list: event_title = event['resource_title'] event_start_time = json_to_datetime(event['timestamp']) if last_event_time is None and last_event_title is None: last_event_time = event_start_time last_event_title = event_title run_end_time = json_to_datetime(event['run_end_time']) continue else: event_exec_time = (event_start_time - last_event_time).total_seconds() add_event = (last_event_title, event_exec_time) event_execution_times.append(add_event) last_event_time = event_start_time last_event_title = event_title event_exec_time = (last_event_time - run_end_time).total_seconds() add_event = [last_event_title, event_exec_time] event_execution_times.append(add_event) sorted_events = sorted(event_execution_times, reverse=True, key=lambda field: field[1]) if len(sorted_events) > 10: sorted_events = sorted_events[:10] else: events_list = False context['certname'] = certname context['report_timestamp'] = report_timestamp context['hashid'] = hashid context['events_list'] = events_list context['event_durations'] = sorted_events context['environment'] = environment return render(request, 'pano/detailed_events.html', context)
def reports_json(request, certname=None): source_url, source_certs, source_verify = get_server(request) # Redirects to the events page if GET param latest is true.. context = {} # Cur Page Number if request.GET.get('page', False): if request.session['report_page'] != int(request.GET.get('page', 1)): request.session['report_page'] = int(request.GET.get('page', 1)) if request.session['report_page'] <= 0: request.session['report_page'] = 1 else: if 'report_page' not in request.session: request.session['report_page'] = 1 if request.session['report_page'] <= 0: offset = 0 else: offset = (25 * request.session['report_page']) - 25 reports_params = { 'query': { 1: '["=","certname","' + certname + '"]' }, 'order_by': { 'order_field': { 'field': 'start_time', 'order': 'desc', }, }, 'limit': 25, 'include_total': 'true', 'offset': offset, } reports_list, headers = puppetdb.api_get( api_url=source_url, cert=source_certs, verify=source_verify, path='/reports', api_version='v4', params=puppetdb.mk_puppetdb_query( reports_params, request), ) # Work out the number of pages from the xrecords response xrecords = headers['X-Records'] num_pages_wdec = float(xrecords) / 25 num_pages_wodec = float("{:.0f}".format(num_pages_wdec)) if num_pages_wdec > num_pages_wodec: num_pages = num_pages_wodec + 1 else: num_pages = num_pages_wodec report_status = [] for report in reports_list: found_report = False events_params = { 'query': { 1: '["=","report","' + report['hash'] + '"]' }, 'summarize_by': 'certname', } eventcount_list = puppetdb.api_get( path='event-counts', api_url=source_url, api_version='v4', params=puppetdb.mk_puppetdb_query(events_params, request), ) # Make list of the results for event in eventcount_list: if event['subject']['title'] == report['certname']: found_report = True report_status.append({ 'hash': report['hash'], 'certname': report['certname'], 'environment': report['environment'], 'start_time': filters.date(localtime(json_to_datetime(report['start_time'])), 'Y-m-d H:i:s'), 'end_time': filters.date(localtime(json_to_datetime(report['end_time'])), 'Y-m-d H:i:s'), 'events_successes': event['successes'], 'events_noops': event['noops'], 'events_failures': event['failures'], 'events_skipped': event['skips'], 'report_status': report['status'], 'config_version': report['configuration_version'], 'run_duration': "{0:.0f}".format( (json_to_datetime(report['end_time']) - json_to_datetime(report['start_time'])).total_seconds()) }) break if found_report is False: report_status.append({ 'hash': report['hash'], 'certname': report['certname'], 'environment': report['environment'], 'start_time': filters.date(localtime(json_to_datetime(report['start_time'])), 'Y-m-d H:i:s'), 'end_time': filters.date(localtime(json_to_datetime(report['end_time'])), 'Y-m-d H:i:s'), 'events_successes': 0, 'events_noops': 0, 'events_failures': 0, 'events_skipped': 0, 'report_status': report['status'], 'config_version': report['configuration_version'], 'run_duration': "{0:.0f}".format( (json_to_datetime(report['end_time']) - json_to_datetime(report['start_time'])).total_seconds()) }) context['certname'] = certname context['reports_list'] = report_status context['curr_page'] = request.session['report_page'] context['tot_pages'] = "{:.0f}".format(num_pages) return HttpResponse(json.dumps(context), content_type="application/json")
def detailed_events(request, hashid=None): context = {'timezones': pytz.common_timezones, 'SOURCES': AVAILABLE_SOURCES} if request.method == 'GET': if 'source' in request.GET: source = request.GET.get('source') set_server(request, source) if request.method == 'POST': request.session['django_timezone'] = request.POST['timezone'] return redirect(request.POST['return_url']) source_url, source_certs, source_verify = get_server(request) report_timestamp = request.GET.get('report_timestamp') events_params = { 'query': { 1: '["=","report","' + hashid + '"]' }, 'order_by': { 'order_field': { 'field': 'timestamp', 'order': 'asc', }, 'query_field': {'field': 'certname'}, }, } events_list = puppetdb.api_get( api_url=source_url, cert=source_certs, verify=source_verify, path='/events', api_version='v4', params=puppetdb.mk_puppetdb_query(events_params), ) environment = '' certname = '' event_execution_times = [] sorted_events = None last_event_time = None last_event_title = None run_end_time = None if len(events_list) != 0: single_event = events_list[0] environment = single_event['environment'] certname = single_event['certname'] for event in events_list: event_title = event['resource_title'] event_start_time = json_to_datetime(event['timestamp']) if last_event_time is None and last_event_title is None: last_event_time = event_start_time last_event_title = event_title run_end_time = json_to_datetime(event['run_end_time']) continue else: event_exec_time = (event_start_time - last_event_time).total_seconds() add_event = (last_event_title, event_exec_time) event_execution_times.append(add_event) last_event_time = event_start_time last_event_title = event_title event_exec_time = (last_event_time - run_end_time).total_seconds() add_event = [last_event_title, event_exec_time] event_execution_times.append(add_event) sorted_events = sorted(event_execution_times, reverse=True, key=lambda field: field[1]) if len(sorted_events) > 10: sorted_events = sorted_events[:10] else: events_list = False context['certname'] = certname context['report_timestamp'] = report_timestamp context['hashid'] = hashid context['events_list'] = events_list context['event_durations'] = sorted_events context['environment'] = environment return render(request, 'pano/detailed_events.html', context)
def nodes_json(request): if request.method == 'GET': if 'source' in request.GET: source = request.GET.get('source') set_server(request, source) if request.method == 'POST': request.session['django_timezone'] = request.POST['timezone'] return redirect(request.POST['return_url']) source_url, source_certs, source_verify = get_server(request) valid_sort_fields = ('certname', 'catalog_timestamp', 'report_timestamp', 'facts_timestamp', 'successes', 'noops', 'failures', 'skips') try: # If user requested to download csv formatted file. Default value is False dl_csv = request.GET.get('dl_csv', False) if dl_csv == 'true': dl_csv = True else: dl_csv = False # Add limits to session if request.GET.get('limits', False): if request.session['limits'] != int(request.GET.get('limits', 50)): request.session['limits'] = int(request.GET.get('limits', 50)) if request.session['limits'] <= 0: request.session['limits'] = 50 else: if 'limits' not in request.session: request.session['limits'] = 50 # Cur Page Number if request.GET.get('page', False): if request.session['page'] != int(request.GET.get('page', 1)): request.session['page'] = int(request.GET.get('page', 1)) if request.session['page'] <= 0: request.session['page'] = 1 else: if 'page' not in request.session: request.session['page'] = 1 # Cur sort field if request.GET.get('sortfield', False): if request.session['sortfield'] != request.GET.get('sortfield'): request.session['sortfield'] = request.GET.get('sortfield') if request.session['sortfield'] not in valid_sort_fields: request.session['sortfield'] = 'report_timestamp' else: if 'sortfield' not in request.session: request.session['sortfield'] = 'report_timestamp' # Cur sort order if request.GET.get('sortfieldby', False): avail_sortorder = ['asc', 'desc'] if request.session['sortfieldby'] != request.GET.get( 'sortfieldby'): request.session['sortfieldby'] = request.GET.get('sortfieldby') if request.session['sortfieldby'] not in avail_sortorder: request.session['sortfieldby'] = 'desc' else: if 'sortfieldby' not in request.session: request.session['sortfieldby'] = 'desc' # Search parameters takes a valid puppetdb query string if request.GET.get('search', False): if 'search' in request.session and (request.session['search'] == request.GET.get('search')): pass else: if request.GET.get('search') == 'clear_rules': request.session['sortfield'] = 'report_timestamp' request.session['sortfieldby'] = 'desc' request.session['page'] = 1 request.session['search'] = None else: request.session['page'] = 1 request.session['search'] = request.GET.get('search') else: if 'search' not in request.session: request.session['sortfield'] = 'report_timestamp' request.session['sortfieldby'] = 'desc' request.session['page'] = 1 request.session['search'] = None # Set offset request.session['offset'] = ( request.session['limits'] * request.session['page']) - request.session['limits'] except: return HttpResponseBadRequest('Oh no! Your filters were invalid.') # Valid sort field that the user can search agnaist. sort_field = request.session['sortfield'] sort_field_order = request.session['sortfieldby'] page_num = request.session['page'] if request.session['search'] is not None: node_params = { 'query': { 1: request.session['search'] }, } else: node_params = { 'query': {}, } nodes_sort_fields = [ 'certname', 'catalog_timestamp', 'report_timestamp', 'facts_timestamp' ] if sort_field in nodes_sort_fields: node_params['order_by'] = { 'order_field': { 'field': sort_field, 'order': sort_field_order, }, } if dl_csv is False: node_params['limit'] = request.session['limits'] node_params['offset'] = request.session['offset'] node_params['include_total'] = 'true' else: node_params['order_by'] = { 'order_field': { 'field': 'report_timestamp', 'order': 'desc', }, } node_sort_fields = [ 'certname', 'catalog_timestamp', 'report_timestamp', 'facts_timestamp' ] if sort_field in node_sort_fields: try: node_list, node_headers = puppetdb.api_get( api_url=source_url, cert=source_certs, verify=source_verify, path='/nodes', api_version='v4', params=puppetdb.mk_puppetdb_query(node_params, request), ) except: node_list = [] node_headers = dict() node_headers['X-Records'] = 0 else: node_list = puppetdb.api_get( api_url=source_url, cert=source_certs, verify=source_verify, path='/nodes', api_version='v4', params=puppetdb.mk_puppetdb_query(node_params, request), ) # Work out the number of pages from the xrecords response # return fields that you can sort by # for each node in the node_list, find out if the latest run has any failures # v3/event-counts --data-urlencode query='["=","latest-report?",true]' # --data-urlencode summarize-by='certname' report_params = { 'query': { 1: '["and",["=","latest_report?",true],["in", "certname",["extract", "certname",["select_nodes",["null?","deactivated",true]]]]]' }, 'summarize_by': 'certname', } status_sort_fields = ['successes', 'failures', 'skips', 'noops'] report_status_params = { 'query': { 1: '["and",["=","latest_report?",true],["in", "certname",["extract", "certname",["select_nodes",["null?","deactivated",true]]]]]' } } report_status_list = puppetdb.api_get( api_url=source_url, cert=source_certs, verify=source_verify, path='/reports', params=puppetdb.mk_puppetdb_query(report_status_params, request), api_version='v4', ) if sort_field in status_sort_fields: if request.session['search'] is not None: report_params['query'] = { 'operator': 'and', 1: request.session['search'], 2: '["=","latest_report?",true]', 3: '["in", "certname",["extract", "certname",["select_nodes",["null?","deactivated",true]]]]', } report_params['order_by'] = { 'order_field': { 'field': sort_field, 'order': sort_field_order, } } report_params['include_total'] = 'true' # Don't limit results if its CSV if dl_csv is False: report_params['limit'] = request.session['limits'] report_params['offset'] = request.session['offset'] report_list, report_headers = puppetdb.api_get( api_url=source_url, cert=source_certs, verify=source_verify, path='/event-counts', params=puppetdb.mk_puppetdb_query(report_params, request), api_version='v4', ) else: report_list = puppetdb.api_get( api_url=source_url, cert=source_certs, verify=source_verify, path='event-counts', params=puppetdb.mk_puppetdb_query(report_params, request), api_version='v4', ) # number of results depending on sort field. if sort_field in status_sort_fields: xrecords = report_headers['X-Records'] total_results = xrecords elif sort_field in nodes_sort_fields: xrecords = node_headers['X-Records'] total_results = xrecords num_pages_wdec = float(xrecords) / request.session['limits'] num_pages_wodec = float("{:.0f}".format(num_pages_wdec)) if num_pages_wdec > num_pages_wodec: num_pages = num_pages_wodec + 1 else: num_pages = num_pages_wodec # Converts lists of dicts to dicts. node_dict = {item['certname']: item for item in node_list} status_dict = {item['certname']: item for item in report_status_list} report_dict = {item['subject']['title']: item for item in report_list} if sort_field_order == 'desc': rows = dictstatus(node_dict, status_dict, report_dict, sortby=sort_field, asc=True, sort=False) sort_field_order_opposite = 'asc' elif sort_field_order == 'asc': rows = dictstatus(node_dict, status_dict, report_dict, sortby=sort_field, asc=False, sort=False) sort_field_order_opposite = 'desc' if dl_csv is True: if rows is []: pass else: # Generate a sequence of rows. The range is based on the maximum number of # rows that can be handled by a single sheet in most spreadsheet # applications. include_facts = request.GET.get('include_facts', False) csv_headers = [ 'Certname', 'Latest Catalog', 'Latest Report', 'Latest Facts', 'Success', 'Noop', 'Failure', 'Skipped' ] if include_facts is not False: merged_list_facts = [] facts = {} for fact in include_facts.split(','): # Sanitize the fact input from the user fact = fact.strip() # Add the fact name to the headers list csv_headers.append(fact) # build the params for each fact. facts_params = { 'query': { 1: '["=","name","' + fact + '"]' }, } fact_list = puppetdb.api_get( api_url=source_url, cert=source_certs, verify=source_verify, path='facts', params=puppetdb.mk_puppetdb_query(facts_params), api_version='v4', ) # Populate the facts dict with the facts we have retrieved # Convert the fact list into a fact dict! facts[fact] = { item['certname']: item for item in fact_list } i = 1 jobs = {} # Add ID to each job so that it can be assembled in # the same order after we recieve the job results # We do this via jobs so that we can get faster results. for node in rows: jobs[i] = { 'id': i, 'include_facts': include_facts.split(','), 'node': node, 'facts': facts, } i += 1 csv_results = generate_csv(jobs) rows = [] i = 1 # with the job results we can now recreate merged_list # in the order we sent them. while i <= len(csv_results): rows.append(csv_results[i]) i += 1 # Insert the csv header to the top of the list. rows.insert(0, csv_headers) pseudo_buffer = Echo() writer = csv.writer(pseudo_buffer) response = StreamingHttpResponse( (writer.writerow(row) for row in rows), content_type="text/csv") response[ 'Content-Disposition'] = 'attachment; filename="puppetdata-%s.csv"' % ( datetime.datetime.now()) return response """ c_r_s* = current request sort c_r_* = current req r_s* = requests available """ context = { 'nodeList': rows, 'total_nodes': total_results, 'c_r_page': page_num, 'c_r_limit': request.session['limits'], 'r_sfield': valid_sort_fields, 'c_r_sfield': sort_field, 'r_sfieldby': ['asc', 'desc'], 'c_r_sfieldby': sort_field_order, 'c_r_sfieldby_o': sort_field_order_opposite, 'tot_pages': '{0:g}'.format(num_pages), } return HttpResponse(json.dumps(context), content_type="application/json")
def catalogue_compare_json(request, certname1=None, certname2=None): source_url, source_certs, source_verify = get_server(request) show = request.GET.get('show', 'edges') data = dict() certname1_hash = request.GET.get('certname1_hash', False) certname2_hash = request.GET.get('certname2_hash', False) cata_params = dict() if certname1_hash: try: certname1_result = SavedCatalogs.objects.get(hostname=certname1, catalogue_id=certname1_hash) certname1_data = json.loads(certname1_result.catalogue)[show]['data'] except SavedCatalogs.DoesNotExist: data['error'] = 'Catalogue hash not found in DB.' data['hash_not_found'] = certname1_hash data['certname'] = certname1 return HttpResponseBadRequest(json.dumps(data, indent=2), content_type="application/json") else: certname1_result = puppetdb.api_get( path='/catalogs/%s' % certname1, api_url=source_url, api_version='v4', params=puppetdb.mk_puppetdb_query(cata_params, request), ) certname1_data = certname1_result[show]['data'] if certname2_hash: try: certname2_result = SavedCatalogs.objects.get(hostname=certname2, catalogue_id=certname2_hash) certname2_data = json.loads(certname2_result.catalogue)[show]['data'] except SavedCatalogs.DoesNotExist: data['error'] = 'Catalogue hash not found in DB.' data['hash_not_found'] = certname2_hash data['certname'] = certname2 return HttpResponseBadRequest(json.dumps(data, indent=2), content_type="application/json") else: certname2_result = puppetdb.api_get( path='/catalogs/%s' % certname2, api_url=source_url, api_version='v4', params=puppetdb.mk_puppetdb_query(cata_params, request), ) certname2_data = certname2_result[show]['data'] node_for = dict() node_agn = dict() if show == "edges": for edge in certname1_data: # remove the certname tag. try: edge.pop('certname') except: pass source_type = edge['source_type'] source_title = edge['source_title'] relationship = edge['relationship'] target_type = edge['target_type'] target_title = edge['target_title'] node_for['%s-%s-%s-%s-%s' % (source_type, source_title, relationship, target_type, target_title)] = edge for edge in certname2_data: # remove the certname tag. try: edge.pop('certname') except: pass source_type = edge['source_type'] source_title = edge['source_title'] relationship = edge['relationship'] target_type = edge['target_type'] target_title = edge['target_title'] node_agn['%s-%s-%s-%s-%s' % (source_type, source_title, relationship, target_type, target_title)] = edge elif show == "resources": for resource in certname1_data: # remove the certname tag. try: resource.pop('certname') except: pass resource_title = resource['title'] node_for[resource_title] = resource for resource in certname2_data: try: # remove the certname tag. resource.pop('certname') except: pass resource_title = resource['title'] node_agn[resource_title] = resource diff = DictDiffer(node_agn, node_for) new_entries = list() rem_entries = list() cha_entries = list() # List of new entries for new_entry in diff.added(): new_entries.append(node_agn[new_entry]) for rem_entry in diff.removed(): rem_entries.append(node_for[rem_entry]) for cha_entry in diff.changed(): for_entry = node_for[cha_entry] agn_entry = node_agn[cha_entry] cha_entries.append({ 'from': for_entry, 'against': agn_entry }) output = { 'added_entries': new_entries, 'deleted_entries': rem_entries, 'changed_entries': cha_entries } return HttpResponse(json.dumps(output, indent=2), content_type="application/json")
def nodes_json(request): if request.method == 'GET': if 'source' in request.GET: source = request.GET.get('source') set_server(request, source) if request.method == 'POST': request.session['django_timezone'] = request.POST['timezone'] return redirect(request.POST['return_url']) source_url, source_certs, source_verify = get_server(request) valid_sort_fields = ( 'certname', 'catalog_timestamp', 'report_timestamp', 'facts_timestamp', 'successes', 'noops', 'failures', 'skips') try: # If user requested to download csv formatted file. Default value is False dl_csv = request.GET.get('dl_csv', False) if dl_csv == 'true': dl_csv = True else: dl_csv = False # Add limits to session if request.GET.get('limits', False): if request.session['limits'] != int(request.GET.get('limits', 50)): request.session['limits'] = int(request.GET.get('limits', 50)) if request.session['limits'] <= 0: request.session['limits'] = 50 else: if 'limits' not in request.session: request.session['limits'] = 50 # Cur Page Number if request.GET.get('page', False): if request.session['page'] != int(request.GET.get('page', 1)): request.session['page'] = int(request.GET.get('page', 1)) if request.session['page'] <= 0: request.session['page'] = 1 else: if 'page' not in request.session: request.session['page'] = 1 # Cur sort field if request.GET.get('sortfield', False): if request.session['sortfield'] != request.GET.get('sortfield'): request.session['sortfield'] = request.GET.get('sortfield') if request.session['sortfield'] not in valid_sort_fields: request.session['sortfield'] = 'report_timestamp' else: if 'sortfield' not in request.session: request.session['sortfield'] = 'report_timestamp' # Cur sort order if request.GET.get('sortfieldby', False): avail_sortorder = ['asc', 'desc'] if request.session['sortfieldby'] != request.GET.get('sortfieldby'): request.session['sortfieldby'] = request.GET.get('sortfieldby') if request.session['sortfieldby'] not in avail_sortorder: request.session['sortfieldby'] = 'desc' else: if 'sortfieldby' not in request.session: request.session['sortfieldby'] = 'desc' # Search parameters takes a valid puppetdb query string if request.GET.get('search', False): if 'search' in request.session and (request.session['search'] == request.GET.get('search')): pass else: if request.GET.get('search') == 'clear_rules': request.session['sortfield'] = 'report_timestamp' request.session['sortfieldby'] = 'desc' request.session['page'] = 1 request.session['search'] = None else: request.session['page'] = 1 request.session['search'] = request.GET.get('search') else: if 'search' not in request.session: request.session['sortfield'] = 'report_timestamp' request.session['sortfieldby'] = 'desc' request.session['page'] = 1 request.session['search'] = None # Set offset request.session['offset'] = (request.session['limits'] * request.session['page']) - request.session[ 'limits'] except: return HttpResponseBadRequest('Oh no! Your filters were invalid.') # Valid sort field that the user can search agnaist. sort_field = request.session['sortfield'] sort_field_order = request.session['sortfieldby'] page_num = request.session['page'] if request.session['search'] is not None: node_params = { 'query': { 1: request.session['search'] }, } else: node_params = { 'query': {}, } nodes_sort_fields = ['certname', 'catalog_timestamp', 'report_timestamp', 'facts_timestamp'] if sort_field in nodes_sort_fields: node_params['order_by'] = { 'order_field': { 'field': sort_field, 'order': sort_field_order, }, } if dl_csv is False: node_params['limit'] = request.session['limits'] node_params['offset'] = request.session['offset'] node_params['include_total'] = 'true' else: node_params['order_by'] = { 'order_field': { 'field': 'report_timestamp', 'order': 'desc', }, } node_sort_fields = ['certname', 'catalog_timestamp', 'report_timestamp', 'facts_timestamp'] if sort_field in node_sort_fields: try: node_list, node_headers = puppetdb.api_get( api_url=source_url, cert=source_certs, verify=source_verify, path='/nodes', api_version='v4', params=puppetdb.mk_puppetdb_query( node_params, request), ) except: node_list = [] node_headers = dict() node_headers['X-Records'] = 0 else: node_list = puppetdb.api_get( api_url=source_url, cert=source_certs, verify=source_verify, path='/nodes', api_version='v4', params=puppetdb.mk_puppetdb_query( node_params, request), ) # Work out the number of pages from the xrecords response # return fields that you can sort by # for each node in the node_list, find out if the latest run has any failures # v3/event-counts --data-urlencode query='["=","latest-report?",true]' # --data-urlencode summarize-by='certname' report_params = { 'query': { 1: '["and",["=","latest_report?",true],["in", "certname",["extract", "certname",["select_nodes",["null?","deactivated",true]]]]]' }, 'summarize_by': 'certname', } status_sort_fields = ['successes', 'failures', 'skips', 'noops'] report_status_params = { 'query': { 1: '["and",["=","latest_report?",true],["in", "certname",["extract", "certname",["select_nodes",["null?","deactivated",true]]]]]' } } report_status_list = puppetdb.api_get( api_url=source_url, cert=source_certs, verify=source_verify, path='/reports', params=puppetdb.mk_puppetdb_query(report_status_params, request), api_version='v4', ) if sort_field in status_sort_fields: if request.session['search'] is not None: report_params['query'] = {'operator': 'and', 1: request.session['search'], 2: '["=","latest_report?",true]', 3: '["in", "certname",["extract", "certname",["select_nodes",["null?","deactivated",true]]]]', } report_params['order_by'] = { 'order_field': { 'field': sort_field, 'order': sort_field_order, } } report_params['include_total'] = 'true' # Don't limit results if its CSV if dl_csv is False: report_params['limit'] = request.session['limits'] report_params['offset'] = request.session['offset'] report_list, report_headers = puppetdb.api_get( api_url=source_url, cert=source_certs, verify=source_verify, path='/event-counts', params=puppetdb.mk_puppetdb_query(report_params, request), api_version='v4', ) else: report_list = puppetdb.api_get( api_url=source_url, cert=source_certs, verify=source_verify, path='event-counts', params=puppetdb.mk_puppetdb_query(report_params, request), api_version='v4', ) # number of results depending on sort field. if sort_field in status_sort_fields: xrecords = report_headers['X-Records'] total_results = xrecords elif sort_field in nodes_sort_fields: xrecords = node_headers['X-Records'] total_results = xrecords num_pages_wdec = float(xrecords) / request.session['limits'] num_pages_wodec = float("{:.0f}".format(num_pages_wdec)) if num_pages_wdec > num_pages_wodec: num_pages = num_pages_wodec + 1 else: num_pages = num_pages_wodec # Converts lists of dicts to dicts. node_dict = {item['certname']: item for item in node_list} status_dict = {item['certname']: item for item in report_status_list} report_dict = {item['subject']['title']: item for item in report_list} if sort_field_order == 'desc': rows = dictstatus( node_dict, status_dict, report_dict, sortby=sort_field, asc=True, sort=False) sort_field_order_opposite = 'asc' elif sort_field_order == 'asc': rows = dictstatus( node_dict, status_dict, report_dict, sortby=sort_field, asc=False, sort=False) sort_field_order_opposite = 'desc' if dl_csv is True: if rows is []: pass else: # Generate a sequence of rows. The range is based on the maximum number of # rows that can be handled by a single sheet in most spreadsheet # applications. include_facts = request.GET.get('include_facts', False) csv_headers = ['Certname', 'Latest Catalog', 'Latest Report', 'Latest Facts', 'Success', 'Noop', 'Failure', 'Skipped'] if include_facts is not False: merged_list_facts = [] facts = {} for fact in include_facts.split(','): # Sanitize the fact input from the user fact = fact.strip() # Add the fact name to the headers list csv_headers.append(fact) # build the params for each fact. facts_params = { 'query': { 1: '["=","name","' + fact + '"]' }, } fact_list = puppetdb.api_get( api_url=source_url, cert=source_certs, verify=source_verify, path='facts', params=puppetdb.mk_puppetdb_query(facts_params), api_version='v4', ) # Populate the facts dict with the facts we have retrieved # Convert the fact list into a fact dict! facts[fact] = {item['certname']: item for item in fact_list} i = 1 jobs = {} # Add ID to each job so that it can be assembled in # the same order after we recieve the job results # We do this via jobs so that we can get faster results. for node in rows: jobs[i] = { 'id': i, 'include_facts': include_facts.split(','), 'node': node, 'facts': facts, } i += 1 csv_results = generate_csv(jobs) rows = [] i = 1 # with the job results we can now recreate merged_list # in the order we sent them. while i <= len(csv_results): rows.append(csv_results[i]) i += 1 # Insert the csv header to the top of the list. rows.insert(0, csv_headers) pseudo_buffer = Echo() writer = csv.writer(pseudo_buffer) response = StreamingHttpResponse((writer.writerow(row) for row in rows), content_type="text/csv") response['Content-Disposition'] = 'attachment; filename="puppetdata-%s.csv"' % (datetime.datetime.now()) return response """ c_r_s* = current request sort c_r_* = current req r_s* = requests available """ context = { 'nodeList': rows, 'total_nodes': total_results, 'c_r_page': page_num, 'c_r_limit': request.session['limits'], 'r_sfield': valid_sort_fields, 'c_r_sfield': sort_field, 'r_sfieldby': ['asc', 'desc'], 'c_r_sfieldby': sort_field_order, 'c_r_sfieldby_o': sort_field_order_opposite, 'tot_pages': '{0:g}'.format(num_pages), } return HttpResponse(json.dumps(context), content_type="application/json")
def catalogue_json(request, certname=None): context = dict() data = dict() if not certname: context['error'] = 'Must specify certname.' return HttpResponse(json.dumps(context), content_type="application/json") source_url, source_certs, source_verify = get_server(request) # Redirects to the events page if GET param latest is true.. show = request.GET.get('show', 'edges') save_catalog = request.GET.get('save', 'false') catalogue_params = {} path = '/catalogs/%s' % certname catalogue = puppetdb.api_get( path=path, api_url=source_url, api_version='v4', params=puppetdb.mk_puppetdb_query(catalogue_params, request), ) if 'error' not in catalogue and save_catalog == 'true': report_param = { 'query': { 'operator': 'and', 1: '["=","latest_report?",true]', 2: '["=","certname","%s"]' % certname } } report_url = '/reports' latest_report = puppetdb.api_get( path=report_url, api_url=source_url, api_version='v4', params=puppetdb.mk_puppetdb_query(report_param, request), ) report_hash = latest_report[0]['hash'] catalogue_hash = catalogue['hash'] catalogue_timestamp = catalogue['producer_timestamp'] try: saved_catalogue = SavedCatalogs.objects.get(hostname=certname, catalogue_id=catalogue_hash) if saved_catalogue.linked_report != report_hash: # Grab the linked report from the result set. old_linked_report = saved_catalogue.linked_report # Update the data. saved_catalogue.linked_report = report_hash saved_catalogue.timestamp = catalogue_timestamp # Save the new data. saved_catalogue.save() data['success'] = 'Catalogue hash updated.' data['certname'] = certname data['old_linked_report'] = old_linked_report data['new_linked_report'] = report_hash return HttpResponse(json.dumps(data, indent=2), content_type='application/json') else: data['error'] = 'Catalogue hash already exists.' data['certname'] = certname data['catalogue_hash'] = catalogue_hash data['linked_report'] = saved_catalogue.linked_report return HttpResponseBadRequest(json.dumps(data, indent=2), content_type='application/json') except SavedCatalogs.DoesNotExist: # since we couldnt find it in the db its safe to asusme that we can create it! SavedCatalogs.objects.create(hostname=certname, catalogue_id=catalogue_hash, linked_report=report_hash, timestamp=catalogue_timestamp, catalogue=json.dumps(catalogue)) data['success'] = 'Saved catalogue.' data['certname'] = certname data['catalogue_hash'] = catalogue_hash data['linked_report'] = report_hash return HttpResponse(json.dumps(data, indent=2), content_type='application/json') if show == 'edges': data['data'] = catalogue['edges']['data'] elif show == 'resources': data['data'] = catalogue['resources']['data'] else: data['data'] = catalogue return HttpResponse(json.dumps(data, indent=2), content_type='application/json')