def append_list(n_data, s_data, m_list, r_status): if type(n_data) is not dict or type(s_data) is not dict and type( m_list) is not list and not r_status: raise ValueError( 'Incorrect type given as input. Expects n_data, s_data as dict and m_list as list.' ) m_list.append(( n_data['certname'], filters.date( localtime(json_to_datetime(n_data['catalog_timestamp'])), 'Y-m-d H:i:s') if n_data['catalog_timestamp'] is not None else '', filters.date( localtime(json_to_datetime(n_data['report_timestamp'])), 'Y-m-d H:i:s') if n_data['report_timestamp'] is not None else '', filters.date( localtime(json_to_datetime(n_data['facts_timestamp'])), 'Y-m-d H:i:s') if n_data['facts_timestamp'] is not None else '', s_data.get('successes', 0), s_data.get('noops', 0), s_data.get('failures', 0), s_data.get('skips', 0), r_status, )) return m_list
def append_list(n_data, s_data, m_list, r_status): if type(n_data) is not dict or type(s_data) is not dict and type(m_list) is not list and not r_status: raise ValueError('Incorrect type given as input. Expects n_data, s_data as dict and m_list as list.') m_list.append(( n_data['certname'], filters.date(localtime(json_to_datetime(n_data['catalog_timestamp'])), 'Y-m-d H:i:s') if n_data[ 'catalog_timestamp'] is not None else '', filters.date(localtime(json_to_datetime(n_data['report_timestamp'])), 'Y-m-d H:i:s') if n_data[ 'report_timestamp'] is not None else '', filters.date(localtime(json_to_datetime(n_data['facts_timestamp'])), 'Y-m-d H:i:s') if n_data[ 'facts_timestamp'] is not None else '', s_data.get('successes', 0), s_data.get('noops', 0), s_data.get('failures', 0), s_data.get('skips', 0), r_status, )) return m_list
def check_failed_compile(report_timestamp, fact_timestamp, catalog_timestamp, puppet_run_interval=puppet_run_time): """ :param report_timestamp: str :param fact_timestamp: str :param catalog_timestamp: str :return: Bool Returns False if the compiled run has not failed Returns True if the compiled run has failed """ if report_timestamp is None or catalog_timestamp is None or fact_timestamp is None: return True # check if the fact report is older than puppet_run_time by double the run time report_time = json_to_datetime(report_timestamp) fact_time = json_to_datetime(fact_timestamp) catalog_time = json_to_datetime(catalog_timestamp) # Report time, fact time and catalog time should all be run within (PUPPET_RUN_INTERVAL / 2) # minutes of each other diffs = dict() # Time elapsed between fact time and catalog time diffs['catalog_fact'] = catalog_time - fact_time diffs['fact_catalog'] = fact_time - catalog_time # Time elapsed between fact time and report time diffs['report_fact'] = report_time - fact_time diffs['fact_report'] = fact_time - report_time # Time elapsed between report and catalog diffs['report_catalog'] = report_time - catalog_time diffs['catalog_report'] = catalog_time - report_time for key, value in diffs.items(): if value > timedelta(minutes=puppet_run_interval / 2): return True return False
def append_list(n_data, s_data, m_list, r_status, format_time=True): if type(n_data) is not dict or type(s_data) is not dict and type( m_list) is not list and not r_status: raise ValueError( 'Incorrect type given as input. Expects n_data, s_data as dict and m_list as list.' ) catalog_timestamp = n_data['catalog_timestamp'] if n_data[ 'catalog_timestamp'] is not None else '' report_timestamp = n_data['report_timestamp'] if n_data[ 'report_timestamp'] is not None else '' facts_timestamp = n_data['facts_timestamp'] if n_data[ 'facts_timestamp'] is not None else '' if format_time: if catalog_timestamp is not '': catalog_timestamp = filters.date( localtime(json_to_datetime(catalog_timestamp)), 'Y-m-d H:i:s') if report_timestamp is not '': report_timestamp = filters.date( localtime(json_to_datetime(report_timestamp)), 'Y-m-d H:i:s') if facts_timestamp is not '': facts_timestamp = filters.date( localtime(json_to_datetime(facts_timestamp)), 'Y-m-d H:i:s') m_list.append(( n_data['certname'], catalog_timestamp, report_timestamp, facts_timestamp, s_data.get('successes', 0), s_data.get('noops', 0), s_data.get('failures', 0), s_data.get('skips', 0), r_status, ))
def check_failed_compile(report_timestamp, fact_timestamp, catalog_timestamp, puppet_run_interval=puppet_run_time): """ :param report_timestamp: str :param fact_timestamp: str :param catalog_timestamp: str :return: Bool Returns False if the compiled run has not failed Returns True if the compiled run has failed """ if report_timestamp is None or catalog_timestamp is None or fact_timestamp is None: return True # check if the fact report is older than puppet_run_time by double the run time report_time = json_to_datetime(report_timestamp) fact_time = json_to_datetime(fact_timestamp) catalog_time = json_to_datetime(catalog_timestamp) # Report time, fact time and catalog time should all be run within (PUPPET_RUN_INTERVAL / 2) # minutes of each other diffs = dict() # Time elapsed between fact time and catalog time diffs['catalog_fact'] = catalog_time - fact_time diffs['fact_catalog'] = fact_time - catalog_time # Time elapsed between fact time and report time diffs['report_fact'] = report_time - fact_time diffs['fact_report'] = fact_time - report_time # Time elapsed between report and catalog diffs['report_catalog'] = report_time - catalog_time diffs['catalog_report'] = catalog_time - report_time for key, value in diffs.items(): if value > timedelta(minutes=puppet_run_interval / 2): return True return False
def reports_json(request, certname=None): source_url, source_certs, source_verify = get_server(request) # Redirects to the events page if GET param latest is true.. context = {} # Cur Page Number if request.GET.get('page', False): if request.session['report_page'] != int(request.GET.get('page', 1)): request.session['report_page'] = int(request.GET.get('page', 1)) if request.session['report_page'] <= 0: request.session['report_page'] = 1 else: if 'report_page' not in request.session: request.session['report_page'] = 1 if request.session['report_page'] <= 0: offset = 0 else: offset = (25 * request.session['report_page']) - 25 reports_params = { 'query': { 1: '["=","certname","' + certname + '"]' }, 'order_by': { 'order_field': { 'field': 'start_time', 'order': 'desc', }, }, 'limit': 25, 'include_total': 'true', 'offset': offset, } reports_list, headers = puppetdb.api_get( api_url=source_url, cert=source_certs, verify=source_verify, path='/reports', api_version='v4', params=puppetdb.mk_puppetdb_query(reports_params, request), ) # Work out the number of pages from the xrecords response xrecords = headers['X-Records'] num_pages_wdec = float(xrecords) / 25 num_pages_wodec = float("{:.0f}".format(num_pages_wdec)) if num_pages_wdec > num_pages_wodec: num_pages = num_pages_wodec + 1 else: num_pages = num_pages_wodec report_status = [] for report in reports_list: found_report = False events_params = { 'query': { 1: '["=","report","' + report['hash'] + '"]' }, 'summarize_by': 'certname', } eventcount_list = puppetdb.api_get( path='event-counts', api_url=source_url, api_version='v4', verify=source_verify, cert=source_certs, params=puppetdb.mk_puppetdb_query(events_params, request), ) # Make list of the results for event in eventcount_list: if event['subject']['title'] == report['certname']: found_report = True report_status.append({ 'hash': report['hash'], 'certname': report['certname'], 'environment': report['environment'], 'is_noop': report['noop'], 'start_time': filters.date( localtime(json_to_datetime(report['start_time'])), 'Y-m-d H:i:s'), 'end_time': filters.date( localtime(json_to_datetime(report['end_time'])), 'Y-m-d H:i:s'), 'events_successes': event['successes'], 'events_noops': event['noops'], 'events_failures': event['failures'], 'events_skipped': event['skips'], 'report_status': report['status'], 'config_version': report['configuration_version'], 'run_duration': "{0:.0f}".format( (json_to_datetime(report['end_time']) - json_to_datetime( report['start_time'])).total_seconds()) }) break if found_report is False: report_status.append({ 'hash': report['hash'], 'certname': report['certname'], 'environment': report['environment'], 'is_noop': report['noop'], 'start_time': filters.date(localtime(json_to_datetime(report['start_time'])), 'Y-m-d H:i:s'), 'end_time': filters.date(localtime(json_to_datetime(report['end_time'])), 'Y-m-d H:i:s'), 'events_successes': 0, 'events_noops': 0, 'events_failures': 0, 'events_skipped': 0, 'report_status': report['status'], 'config_version': report['configuration_version'], 'run_duration': "{0:.0f}".format( (json_to_datetime(report['end_time']) - json_to_datetime(report['start_time'])).total_seconds()) }) context['certname'] = certname context['reports_list'] = report_status context['curr_page'] = request.session['report_page'] context['tot_pages'] = "{:.0f}".format(num_pages) return HttpResponse(json.dumps(context, indent=2), content_type="application/json")
def reports_json(request, certname=None): source_url, source_certs, source_verify = get_server(request) # Redirects to the events page if GET param latest is true.. context = {} # Cur Page Number if request.GET.get('page', False): if request.session['report_page'] != int(request.GET.get('page', 1)): request.session['report_page'] = int(request.GET.get('page', 1)) if request.session['report_page'] <= 0: request.session['report_page'] = 1 else: if 'report_page' not in request.session: request.session['report_page'] = 1 if request.session['report_page'] <= 0: offset = 0 else: offset = (25 * request.session['report_page']) - 25 reports_params = { 'query': { 1: '["=","certname","' + certname + '"]' }, 'order_by': { 'order_field': { 'field': 'start_time', 'order': 'desc', }, }, 'limit': 25, 'include_total': 'true', 'offset': offset, } reports_list, headers = puppetdb.api_get( api_url=source_url, cert=source_certs, verify=source_verify, path='/reports', api_version='v4', params=puppetdb.mk_puppetdb_query( reports_params, request), ) # Work out the number of pages from the xrecords response xrecords = headers['X-Records'] num_pages_wdec = float(xrecords) / 25 num_pages_wodec = float("{:.0f}".format(num_pages_wdec)) if num_pages_wdec > num_pages_wodec: num_pages = num_pages_wodec + 1 else: num_pages = num_pages_wodec report_status = [] for report in reports_list: found_report = False events_params = { 'query': { 1: '["=","report","' + report['hash'] + '"]' }, 'summarize_by': 'certname', } eventcount_list = puppetdb.api_get( path='event-counts', api_url=source_url, api_version='v4', params=puppetdb.mk_puppetdb_query(events_params, request), ) # Make list of the results for event in eventcount_list: if event['subject']['title'] == report['certname']: found_report = True report_status.append({ 'hash': report['hash'], 'certname': report['certname'], 'environment': report['environment'], 'start_time': filters.date(localtime(json_to_datetime(report['start_time'])), 'Y-m-d H:i:s'), 'end_time': filters.date(localtime(json_to_datetime(report['end_time'])), 'Y-m-d H:i:s'), 'events_successes': event['successes'], 'events_noops': event['noops'], 'events_failures': event['failures'], 'events_skipped': event['skips'], 'report_status': report['status'], 'config_version': report['configuration_version'], 'run_duration': "{0:.0f}".format( (json_to_datetime(report['end_time']) - json_to_datetime(report['start_time'])).total_seconds()) }) break if found_report is False: report_status.append({ 'hash': report['hash'], 'certname': report['certname'], 'environment': report['environment'], 'start_time': filters.date(localtime(json_to_datetime(report['start_time'])), 'Y-m-d H:i:s'), 'end_time': filters.date(localtime(json_to_datetime(report['end_time'])), 'Y-m-d H:i:s'), 'events_successes': 0, 'events_noops': 0, 'events_failures': 0, 'events_skipped': 0, 'report_status': report['status'], 'config_version': report['configuration_version'], 'run_duration': "{0:.0f}".format( (json_to_datetime(report['end_time']) - json_to_datetime(report['start_time'])).total_seconds()) }) context['certname'] = certname context['reports_list'] = report_status context['curr_page'] = request.session['report_page'] context['tot_pages'] = "{:.0f}".format(num_pages) return HttpResponse(json.dumps(context), content_type="application/json")
def analytics(request): context = {'timezones': pytz.common_timezones, 'SOURCES': AVAILABLE_SOURCES} if request.method == 'GET': if 'source' in request.GET: source = request.GET.get('source') set_server(request, source) if request.method == 'POST': request.session['django_timezone'] = request.POST['timezone'] return redirect(request.POST['return_url']) source_url, source_certs, source_verify = get_server(request) events_class_params = { 'query': { 1: '["and",["=","latest_report?",true],["in","certname",["extract","certname",["select_nodes",["null?","deactivated",true]]]]]' }, 'summarize_by': 'containing_class', } events_resource_params = { 'query': { 1: '["and",["=","latest_report?",true],["in","certname",["extract","certname",["select_nodes",["null?","deactivated",true]]]]]' }, 'summarize_by': 'resource', } events_status_params = { 'query': { 1: '["and",["=","latest_report?",true],["in","certname",["extract","certname",["select_nodes",["null?","deactivated",true]]]]]' }, 'summarize_by': 'resource', } reports_runavg_params = { 'limit': 100, 'order_by': { 'order_field': { 'field': 'receive_time', 'order': 'desc', }, 'query_field': {'field': 'certname'}, }, } jobs = { 'events_class_list': { 'url': source_url, 'certs': source_certs, 'verify': source_verify, 'id': 'events_class_list', 'path': '/event-counts', 'api_version': 'v4', 'params': events_class_params, 'request': request }, 'events_resource_list': { 'url': source_url, 'certs': source_certs, 'verify': source_verify, 'id': 'events_resource_list', 'path': '/event-counts', 'api_version': 'v4', 'params': events_resource_params, 'request': request }, 'events_status_list': { 'url': source_url, 'certs': source_certs, 'verify': source_verify, 'id': 'events_status_list', 'path': '/aggregate-event-counts', 'api_version': 'v4', 'params': events_status_params, 'request': request }, 'reports_run_avg': { 'url': source_url, 'certs': source_certs, 'verify': source_verify, 'id': 'reports_run_avg', 'path': '/reports', 'api_version': 'v4', 'params': reports_runavg_params, 'request': request }, } job_results = run_puppetdb_jobs(jobs, 4) reports_run_avg = job_results['reports_run_avg'] events_class_list = job_results['events_class_list'] events_resource_list = job_results['events_resource_list'] events_status_list = job_results['events_status_list'] num_runs_avg = len(reports_run_avg) run_avg_times = [] avg_run_time = 0 for report in reports_run_avg: run_time = "{0:.0f}".format( (json_to_datetime(report['end_time']) - json_to_datetime(report['start_time'])).total_seconds()) avg_run_time += int(run_time) run_avg_times.append(run_time) if num_runs_avg != 0: avg_run_time = "{0:.0f}".format(avg_run_time / num_runs_avg) else: avg_run_time = 0 class_event_results = [] class_resource_results = [] class_status_results = [] for item in events_class_list: class_name = item['subject']['title'] class_total = item['skips'] + item['failures'] + item['noops'] + item['successes'] class_event_results.append((class_name, class_total)) for item in events_resource_list: class_name = item['subject']['type'] class_total = item['skips'] + item['failures'] + item['noops'] + item['successes'] class_resource_results.append((class_name, class_total)) if events_status_list: for status, value in events_status_list[0].items(): if value is 0 or status == 'total' or status == 'summarize_by': continue class_status_results.append((status, value)) context['class_events'] = class_event_results context['class_status'] = class_status_results context['resource_events'] = class_resource_results context['run_times'] = run_avg_times context['run_num'] = num_runs_avg context['run_avg'] = avg_run_time return render(request, 'pano/analytics/analytics.html', context)
def analytics(request): context = { 'timezones': pytz.common_timezones, 'SOURCES': AVAILABLE_SOURCES } if request.method == 'GET': if 'source' in request.GET: source = request.GET.get('source') set_server(request, source) if request.method == 'POST': request.session['django_timezone'] = request.POST['timezone'] return redirect(request.POST['return_url']) source_url, source_certs, source_verify = get_server(request) events_class_params = { 'query': { 1: '["and",["=","latest_report?",true],["in","certname",["extract","certname",["select_nodes",["null?","deactivated",true]]]]]' }, 'summarize_by': 'containing_class', } events_resource_params = { 'query': { 1: '["and",["=","latest_report?",true],["in","certname",["extract","certname",["select_nodes",["null?","deactivated",true]]]]]' }, 'summarize_by': 'resource', } events_status_params = { 'query': { 1: '["and",["=","latest_report?",true],["in","certname",["extract","certname",["select_nodes",["null?","deactivated",true]]]]]' }, 'summarize_by': 'resource', } reports_runavg_params = { 'limit': 100, 'order_by': { 'order_field': { 'field': 'receive_time', 'order': 'desc', }, 'query_field': { 'field': 'certname' }, }, } jobs = { 'events_class_list': { 'url': source_url, 'certs': source_certs, 'verify': source_verify, 'id': 'events_class_list', 'path': '/event-counts', 'api_version': 'v4', 'params': events_class_params, 'request': request }, 'events_resource_list': { 'url': source_url, 'certs': source_certs, 'verify': source_verify, 'id': 'events_resource_list', 'path': '/event-counts', 'api_version': 'v4', 'params': events_resource_params, 'request': request }, 'events_status_list': { 'url': source_url, 'certs': source_certs, 'verify': source_verify, 'id': 'events_status_list', 'path': '/aggregate-event-counts', 'api_version': 'v4', 'params': events_status_params, 'request': request }, 'reports_run_avg': { 'url': source_url, 'certs': source_certs, 'verify': source_verify, 'id': 'reports_run_avg', 'path': '/reports', 'api_version': 'v4', 'params': reports_runavg_params, 'request': request }, } job_results = run_puppetdb_jobs(jobs, 4) reports_run_avg = job_results['reports_run_avg'] events_class_list = job_results['events_class_list'] events_resource_list = job_results['events_resource_list'] events_status_list = job_results['events_status_list'] num_runs_avg = len(reports_run_avg) run_avg_times = [] avg_run_time = 0 for report in reports_run_avg: run_time = "{0:.0f}".format( (json_to_datetime(report['end_time']) - json_to_datetime(report['start_time'])).total_seconds()) avg_run_time += int(run_time) run_avg_times.append(run_time) if num_runs_avg != 0: avg_run_time = "{0:.0f}".format(avg_run_time / num_runs_avg) else: avg_run_time = 0 class_event_results = [] class_resource_results = [] class_status_results = [] for item in events_class_list: class_name = item['subject']['title'] class_total = item['skips'] + item['failures'] + item['noops'] + item[ 'successes'] class_event_results.append((class_name, class_total)) for item in events_resource_list: class_name = item['subject']['type'] class_total = item['skips'] + item['failures'] + item['noops'] + item[ 'successes'] class_resource_results.append((class_name, class_total)) print(events_status_list) if events_status_list: for status, value in events_status_list[0].items(): print(status, value) if value is 0 or status == 'total' or status == 'summarize_by': continue class_status_results.append((status, value)) context['class_events'] = class_event_results context['class_status'] = class_status_results context['resource_events'] = class_resource_results context['run_times'] = run_avg_times context['run_num'] = num_runs_avg context['run_avg'] = avg_run_time return render(request, 'pano/analytics/analytics.html', context)
def detailed_events(request, hashid=None): context = {'timezones': pytz.common_timezones, 'SOURCES': AVAILABLE_SOURCES} if request.method == 'GET': if 'source' in request.GET: source = request.GET.get('source') set_server(request, source) if request.method == 'POST': request.session['django_timezone'] = request.POST['timezone'] return redirect(request.POST['return_url']) source_url, source_certs, source_verify = get_server(request) report_timestamp = request.GET.get('report_timestamp') events_params = { 'query': { 1: '["=","report","' + hashid + '"]' }, 'order_by': { 'order_field': { 'field': 'timestamp', 'order': 'asc', }, 'query_field': {'field': 'certname'}, }, } events_list = puppetdb.api_get( api_url=source_url, cert=source_certs, verify=source_verify, path='/events', api_version='v4', params=puppetdb.mk_puppetdb_query(events_params), ) environment = '' certname = '' event_execution_times = [] sorted_events = None last_event_time = None last_event_title = None run_end_time = None if len(events_list) != 0: single_event = events_list[0] environment = single_event['environment'] certname = single_event['certname'] for event in events_list: event_title = event['resource_title'] event_start_time = json_to_datetime(event['timestamp']) if last_event_time is None and last_event_title is None: last_event_time = event_start_time last_event_title = event_title run_end_time = json_to_datetime(event['run_end_time']) continue else: event_exec_time = (event_start_time - last_event_time).total_seconds() add_event = (last_event_title, event_exec_time) event_execution_times.append(add_event) last_event_time = event_start_time last_event_title = event_title event_exec_time = (last_event_time - run_end_time).total_seconds() add_event = [last_event_title, event_exec_time] event_execution_times.append(add_event) sorted_events = sorted(event_execution_times, reverse=True, key=lambda field: field[1]) if len(sorted_events) > 10: sorted_events = sorted_events[:10] else: events_list = False context['certname'] = certname context['report_timestamp'] = report_timestamp context['hashid'] = hashid context['events_list'] = events_list context['event_durations'] = sorted_events context['environment'] = environment return render(request, 'pano/detailed_events.html', context)
def detailed_events(request, hashid=None): context = { 'timezones': pytz.common_timezones, 'SOURCES': AVAILABLE_SOURCES } if request.method == 'GET': if 'source' in request.GET: source = request.GET.get('source') set_server(request, source) if request.method == 'POST': request.session['django_timezone'] = request.POST['timezone'] return redirect(request.POST['return_url']) source_url, source_certs, source_verify = get_server(request) report_timestamp = request.GET.get('report_timestamp') events_params = { 'query': { 1: '["=","report","' + hashid + '"]' }, 'order_by': { 'order_field': { 'field': 'timestamp', 'order': 'asc', }, 'query_field': { 'field': 'certname' }, }, } events_list = puppetdb.api_get( api_url=source_url, cert=source_certs, verify=source_verify, path='/events', api_version='v4', params=puppetdb.mk_puppetdb_query(events_params), ) environment = '' certname = '' event_execution_times = [] sorted_events = None last_event_time = None last_event_title = None run_end_time = None if len(events_list) != 0: single_event = events_list[0] environment = single_event['environment'] certname = single_event['certname'] for event in events_list: event_title = event['resource_title'] event_start_time = json_to_datetime(event['timestamp']) if last_event_time is None and last_event_title is None: last_event_time = event_start_time last_event_title = event_title run_end_time = json_to_datetime(event['run_end_time']) continue else: event_exec_time = (event_start_time - last_event_time).total_seconds() add_event = (last_event_title, event_exec_time) event_execution_times.append(add_event) last_event_time = event_start_time last_event_title = event_title event_exec_time = (last_event_time - run_end_time).total_seconds() add_event = [last_event_title, event_exec_time] event_execution_times.append(add_event) sorted_events = sorted(event_execution_times, reverse=True, key=lambda field: field[1]) if len(sorted_events) > 10: sorted_events = sorted_events[:10] else: events_list = False context['certname'] = certname context['report_timestamp'] = report_timestamp context['hashid'] = hashid context['events_list'] = events_list context['event_durations'] = sorted_events context['environment'] = environment return render(request, 'pano/detailed_events.html', context)