def append_list(n_data, s_data, m_list, r_status):
if type(n_data) is not dict or type(s_data) is not dict and type(
m_list) is not list and not r_status:
raise ValueError(
'Incorrect type given as input. Expects n_data, s_data as dict and m_list as list.'
)
m_list.append((
n_data['certname'],
filters.date(
localtime(json_to_datetime(n_data['catalog_timestamp'])),
'Y-m-d H:i:s')
if n_data['catalog_timestamp'] is not None else '',
filters.date(
localtime(json_to_datetime(n_data['report_timestamp'])),
'Y-m-d H:i:s')
if n_data['report_timestamp'] is not None else '',
filters.date(
localtime(json_to_datetime(n_data['facts_timestamp'])),
'Y-m-d H:i:s')
if n_data['facts_timestamp'] is not None else '',
s_data.get('successes', 0),
s_data.get('noops', 0),
s_data.get('failures', 0),
s_data.get('skips', 0),
r_status,
))
return m_list
def append_list(n_data, s_data, m_list, r_status):
if type(n_data) is not dict or type(s_data) is not dict and type(m_list) is not list and not r_status:
raise ValueError('Incorrect type given as input. Expects n_data, s_data as dict and m_list as list.')
m_list.append((
n_data['certname'],
filters.date(localtime(json_to_datetime(n_data['catalog_timestamp'])), 'Y-m-d H:i:s') if n_data[
'catalog_timestamp'] is not None else '',
filters.date(localtime(json_to_datetime(n_data['report_timestamp'])), 'Y-m-d H:i:s') if n_data[
'report_timestamp'] is not None else '',
filters.date(localtime(json_to_datetime(n_data['facts_timestamp'])), 'Y-m-d H:i:s') if n_data[
'facts_timestamp'] is not None else '',
s_data.get('successes', 0),
s_data.get('noops', 0),
s_data.get('failures', 0),
s_data.get('skips', 0),
r_status,
))
return m_list
def check_failed_compile(report_timestamp,
fact_timestamp,
catalog_timestamp,
puppet_run_interval=puppet_run_time):
"""
:param report_timestamp: str
:param fact_timestamp: str
:param catalog_timestamp: str
:return: Bool
Returns False if the compiled run has not failed
Returns True if the compiled run has failed
"""
if report_timestamp is None or catalog_timestamp is None or fact_timestamp is None:
return True
# check if the fact report is older than puppet_run_time by double the run time
report_time = json_to_datetime(report_timestamp)
fact_time = json_to_datetime(fact_timestamp)
catalog_time = json_to_datetime(catalog_timestamp)
# Report time, fact time and catalog time should all be run within (PUPPET_RUN_INTERVAL / 2)
# minutes of each other
diffs = dict()
# Time elapsed between fact time and catalog time
diffs['catalog_fact'] = catalog_time - fact_time
diffs['fact_catalog'] = fact_time - catalog_time
# Time elapsed between fact time and report time
diffs['report_fact'] = report_time - fact_time
diffs['fact_report'] = fact_time - report_time
# Time elapsed between report and catalog
diffs['report_catalog'] = report_time - catalog_time
diffs['catalog_report'] = catalog_time - report_time
for key, value in diffs.items():
if value > timedelta(minutes=puppet_run_interval / 2):
return True
return False
def append_list(n_data, s_data, m_list, r_status, format_time=True):
if type(n_data) is not dict or type(s_data) is not dict and type(
m_list) is not list and not r_status:
raise ValueError(
'Incorrect type given as input. Expects n_data, s_data as dict and m_list as list.'
)
catalog_timestamp = n_data['catalog_timestamp'] if n_data[
'catalog_timestamp'] is not None else ''
report_timestamp = n_data['report_timestamp'] if n_data[
'report_timestamp'] is not None else ''
facts_timestamp = n_data['facts_timestamp'] if n_data[
'facts_timestamp'] is not None else ''
if format_time:
if catalog_timestamp is not '':
catalog_timestamp = filters.date(
localtime(json_to_datetime(catalog_timestamp)),
'Y-m-d H:i:s')
if report_timestamp is not '':
report_timestamp = filters.date(
localtime(json_to_datetime(report_timestamp)),
'Y-m-d H:i:s')
if facts_timestamp is not '':
facts_timestamp = filters.date(
localtime(json_to_datetime(facts_timestamp)),
'Y-m-d H:i:s')
m_list.append((
n_data['certname'],
catalog_timestamp,
report_timestamp,
facts_timestamp,
s_data.get('successes', 0),
s_data.get('noops', 0),
s_data.get('failures', 0),
s_data.get('skips', 0),
r_status,
))
def check_failed_compile(report_timestamp,
fact_timestamp,
catalog_timestamp,
puppet_run_interval=puppet_run_time):
"""
:param report_timestamp: str
:param fact_timestamp: str
:param catalog_timestamp: str
:return: Bool
Returns False if the compiled run has not failed
Returns True if the compiled run has failed
"""
if report_timestamp is None or catalog_timestamp is None or fact_timestamp is None:
return True
# check if the fact report is older than puppet_run_time by double the run time
report_time = json_to_datetime(report_timestamp)
fact_time = json_to_datetime(fact_timestamp)
catalog_time = json_to_datetime(catalog_timestamp)
# Report time, fact time and catalog time should all be run within (PUPPET_RUN_INTERVAL / 2)
# minutes of each other
diffs = dict()
# Time elapsed between fact time and catalog time
diffs['catalog_fact'] = catalog_time - fact_time
diffs['fact_catalog'] = fact_time - catalog_time
# Time elapsed between fact time and report time
diffs['report_fact'] = report_time - fact_time
diffs['fact_report'] = fact_time - report_time
# Time elapsed between report and catalog
diffs['report_catalog'] = report_time - catalog_time
diffs['catalog_report'] = catalog_time - report_time
for key, value in diffs.items():
if value > timedelta(minutes=puppet_run_interval / 2):
return True
return False
def reports_json(request, certname=None):
source_url, source_certs, source_verify = get_server(request)
# Redirects to the events page if GET param latest is true..
context = {}
# Cur Page Number
if request.GET.get('page', False):
if request.session['report_page'] != int(request.GET.get('page', 1)):
request.session['report_page'] = int(request.GET.get('page', 1))
if request.session['report_page'] <= 0:
request.session['report_page'] = 1
else:
if 'report_page' not in request.session:
request.session['report_page'] = 1
if request.session['report_page'] <= 0:
offset = 0
else:
offset = (25 * request.session['report_page']) - 25
reports_params = {
'query': {
1: '["=","certname","' + certname + '"]'
},
'order_by': {
'order_field': {
'field': 'start_time',
'order': 'desc',
},
},
'limit': 25,
'include_total': 'true',
'offset': offset,
}
reports_list, headers = puppetdb.api_get(
api_url=source_url,
cert=source_certs,
verify=source_verify,
path='/reports',
api_version='v4',
params=puppetdb.mk_puppetdb_query(reports_params, request),
)
# Work out the number of pages from the xrecords response
xrecords = headers['X-Records']
num_pages_wdec = float(xrecords) / 25
num_pages_wodec = float("{:.0f}".format(num_pages_wdec))
if num_pages_wdec > num_pages_wodec:
num_pages = num_pages_wodec + 1
else:
num_pages = num_pages_wodec
report_status = []
for report in reports_list:
found_report = False
events_params = {
'query': {
1: '["=","report","' + report['hash'] + '"]'
},
'summarize_by': 'certname',
}
eventcount_list = puppetdb.api_get(
path='event-counts',
api_url=source_url,
api_version='v4',
verify=source_verify,
cert=source_certs,
params=puppetdb.mk_puppetdb_query(events_params, request),
)
# Make list of the results
for event in eventcount_list:
if event['subject']['title'] == report['certname']:
found_report = True
report_status.append({
'hash':
report['hash'],
'certname':
report['certname'],
'environment':
report['environment'],
'is_noop':
report['noop'],
'start_time':
filters.date(
localtime(json_to_datetime(report['start_time'])),
'Y-m-d H:i:s'),
'end_time':
filters.date(
localtime(json_to_datetime(report['end_time'])),
'Y-m-d H:i:s'),
'events_successes':
event['successes'],
'events_noops':
event['noops'],
'events_failures':
event['failures'],
'events_skipped':
event['skips'],
'report_status':
report['status'],
'config_version':
report['configuration_version'],
'run_duration':
"{0:.0f}".format(
(json_to_datetime(report['end_time']) -
json_to_datetime(
report['start_time'])).total_seconds())
})
break
if found_report is False:
report_status.append({
'hash':
report['hash'],
'certname':
report['certname'],
'environment':
report['environment'],
'is_noop':
report['noop'],
'start_time':
filters.date(localtime(json_to_datetime(report['start_time'])),
'Y-m-d H:i:s'),
'end_time':
filters.date(localtime(json_to_datetime(report['end_time'])),
'Y-m-d H:i:s'),
'events_successes':
0,
'events_noops':
0,
'events_failures':
0,
'events_skipped':
0,
'report_status':
report['status'],
'config_version':
report['configuration_version'],
'run_duration':
"{0:.0f}".format(
(json_to_datetime(report['end_time']) -
json_to_datetime(report['start_time'])).total_seconds())
})
context['certname'] = certname
context['reports_list'] = report_status
context['curr_page'] = request.session['report_page']
context['tot_pages'] = "{:.0f}".format(num_pages)
return HttpResponse(json.dumps(context, indent=2),
content_type="application/json")
def reports_json(request, certname=None):
source_url, source_certs, source_verify = get_server(request)
# Redirects to the events page if GET param latest is true..
context = {}
# Cur Page Number
if request.GET.get('page', False):
if request.session['report_page'] != int(request.GET.get('page', 1)):
request.session['report_page'] = int(request.GET.get('page', 1))
if request.session['report_page'] <= 0:
request.session['report_page'] = 1
else:
if 'report_page' not in request.session:
request.session['report_page'] = 1
if request.session['report_page'] <= 0:
offset = 0
else:
offset = (25 * request.session['report_page']) - 25
reports_params = {
'query':
{
1: '["=","certname","' + certname + '"]'
},
'order_by':
{
'order_field':
{
'field': 'start_time',
'order': 'desc',
},
},
'limit': 25,
'include_total': 'true',
'offset': offset,
}
reports_list, headers = puppetdb.api_get(
api_url=source_url,
cert=source_certs,
verify=source_verify,
path='/reports',
api_version='v4',
params=puppetdb.mk_puppetdb_query(
reports_params, request),
)
# Work out the number of pages from the xrecords response
xrecords = headers['X-Records']
num_pages_wdec = float(xrecords) / 25
num_pages_wodec = float("{:.0f}".format(num_pages_wdec))
if num_pages_wdec > num_pages_wodec:
num_pages = num_pages_wodec + 1
else:
num_pages = num_pages_wodec
report_status = []
for report in reports_list:
found_report = False
events_params = {
'query':
{
1: '["=","report","' + report['hash'] + '"]'
},
'summarize_by': 'certname',
}
eventcount_list = puppetdb.api_get(
path='event-counts',
api_url=source_url,
api_version='v4',
params=puppetdb.mk_puppetdb_query(events_params, request),
)
# Make list of the results
for event in eventcount_list:
if event['subject']['title'] == report['certname']:
found_report = True
report_status.append({
'hash': report['hash'],
'certname': report['certname'],
'environment': report['environment'],
'start_time': filters.date(localtime(json_to_datetime(report['start_time'])), 'Y-m-d H:i:s'),
'end_time': filters.date(localtime(json_to_datetime(report['end_time'])), 'Y-m-d H:i:s'),
'events_successes': event['successes'],
'events_noops': event['noops'],
'events_failures': event['failures'],
'events_skipped': event['skips'],
'report_status': report['status'],
'config_version': report['configuration_version'],
'run_duration': "{0:.0f}".format(
(json_to_datetime(report['end_time']) - json_to_datetime(report['start_time'])).total_seconds())
})
break
if found_report is False:
report_status.append({
'hash': report['hash'],
'certname': report['certname'],
'environment': report['environment'],
'start_time': filters.date(localtime(json_to_datetime(report['start_time'])), 'Y-m-d H:i:s'),
'end_time': filters.date(localtime(json_to_datetime(report['end_time'])), 'Y-m-d H:i:s'),
'events_successes': 0,
'events_noops': 0,
'events_failures': 0,
'events_skipped': 0,
'report_status': report['status'],
'config_version': report['configuration_version'],
'run_duration': "{0:.0f}".format(
(json_to_datetime(report['end_time']) - json_to_datetime(report['start_time'])).total_seconds())
})
context['certname'] = certname
context['reports_list'] = report_status
context['curr_page'] = request.session['report_page']
context['tot_pages'] = "{:.0f}".format(num_pages)
return HttpResponse(json.dumps(context), content_type="application/json")
def analytics(request):
context = {'timezones': pytz.common_timezones,
'SOURCES': AVAILABLE_SOURCES}
if request.method == 'GET':
if 'source' in request.GET:
source = request.GET.get('source')
set_server(request, source)
if request.method == 'POST':
request.session['django_timezone'] = request.POST['timezone']
return redirect(request.POST['return_url'])
source_url, source_certs, source_verify = get_server(request)
events_class_params = {
'query':
{
1: '["and",["=","latest_report?",true],["in","certname",["extract","certname",["select_nodes",["null?","deactivated",true]]]]]'
},
'summarize_by': 'containing_class',
}
events_resource_params = {
'query':
{
1: '["and",["=","latest_report?",true],["in","certname",["extract","certname",["select_nodes",["null?","deactivated",true]]]]]'
},
'summarize_by': 'resource',
}
events_status_params = {
'query':
{
1: '["and",["=","latest_report?",true],["in","certname",["extract","certname",["select_nodes",["null?","deactivated",true]]]]]'
},
'summarize_by': 'resource',
}
reports_runavg_params = {
'limit': 100,
'order_by': {
'order_field': {
'field': 'receive_time',
'order': 'desc',
},
'query_field': {'field': 'certname'},
},
}
jobs = {
'events_class_list': {
'url': source_url,
'certs': source_certs,
'verify': source_verify,
'id': 'events_class_list',
'path': '/event-counts',
'api_version': 'v4',
'params': events_class_params,
'request': request
},
'events_resource_list': {
'url': source_url,
'certs': source_certs,
'verify': source_verify,
'id': 'events_resource_list',
'path': '/event-counts',
'api_version': 'v4',
'params': events_resource_params,
'request': request
},
'events_status_list': {
'url': source_url,
'certs': source_certs,
'verify': source_verify,
'id': 'events_status_list',
'path': '/aggregate-event-counts',
'api_version': 'v4',
'params': events_status_params,
'request': request
},
'reports_run_avg': {
'url': source_url,
'certs': source_certs,
'verify': source_verify,
'id': 'reports_run_avg',
'path': '/reports',
'api_version': 'v4',
'params': reports_runavg_params,
'request': request
},
}
job_results = run_puppetdb_jobs(jobs, 4)
reports_run_avg = job_results['reports_run_avg']
events_class_list = job_results['events_class_list']
events_resource_list = job_results['events_resource_list']
events_status_list = job_results['events_status_list']
num_runs_avg = len(reports_run_avg)
run_avg_times = []
avg_run_time = 0
for report in reports_run_avg:
run_time = "{0:.0f}".format(
(json_to_datetime(report['end_time']) - json_to_datetime(report['start_time'])).total_seconds())
avg_run_time += int(run_time)
run_avg_times.append(run_time)
if num_runs_avg != 0:
avg_run_time = "{0:.0f}".format(avg_run_time / num_runs_avg)
else:
avg_run_time = 0
class_event_results = []
class_resource_results = []
class_status_results = []
for item in events_class_list:
class_name = item['subject']['title']
class_total = item['skips'] + item['failures'] + item['noops'] + item['successes']
class_event_results.append((class_name, class_total))
for item in events_resource_list:
class_name = item['subject']['type']
class_total = item['skips'] + item['failures'] + item['noops'] + item['successes']
class_resource_results.append((class_name, class_total))
if events_status_list:
for status, value in events_status_list[0].items():
if value is 0 or status == 'total' or status == 'summarize_by':
continue
class_status_results.append((status, value))
context['class_events'] = class_event_results
context['class_status'] = class_status_results
context['resource_events'] = class_resource_results
context['run_times'] = run_avg_times
context['run_num'] = num_runs_avg
context['run_avg'] = avg_run_time
return render(request, 'pano/analytics/analytics.html', context)
def analytics(request):
context = {
'timezones': pytz.common_timezones,
'SOURCES': AVAILABLE_SOURCES
}
if request.method == 'GET':
if 'source' in request.GET:
source = request.GET.get('source')
set_server(request, source)
if request.method == 'POST':
request.session['django_timezone'] = request.POST['timezone']
return redirect(request.POST['return_url'])
source_url, source_certs, source_verify = get_server(request)
events_class_params = {
'query': {
1:
'["and",["=","latest_report?",true],["in","certname",["extract","certname",["select_nodes",["null?","deactivated",true]]]]]'
},
'summarize_by': 'containing_class',
}
events_resource_params = {
'query': {
1:
'["and",["=","latest_report?",true],["in","certname",["extract","certname",["select_nodes",["null?","deactivated",true]]]]]'
},
'summarize_by': 'resource',
}
events_status_params = {
'query': {
1:
'["and",["=","latest_report?",true],["in","certname",["extract","certname",["select_nodes",["null?","deactivated",true]]]]]'
},
'summarize_by': 'resource',
}
reports_runavg_params = {
'limit': 100,
'order_by': {
'order_field': {
'field': 'receive_time',
'order': 'desc',
},
'query_field': {
'field': 'certname'
},
},
}
jobs = {
'events_class_list': {
'url': source_url,
'certs': source_certs,
'verify': source_verify,
'id': 'events_class_list',
'path': '/event-counts',
'api_version': 'v4',
'params': events_class_params,
'request': request
},
'events_resource_list': {
'url': source_url,
'certs': source_certs,
'verify': source_verify,
'id': 'events_resource_list',
'path': '/event-counts',
'api_version': 'v4',
'params': events_resource_params,
'request': request
},
'events_status_list': {
'url': source_url,
'certs': source_certs,
'verify': source_verify,
'id': 'events_status_list',
'path': '/aggregate-event-counts',
'api_version': 'v4',
'params': events_status_params,
'request': request
},
'reports_run_avg': {
'url': source_url,
'certs': source_certs,
'verify': source_verify,
'id': 'reports_run_avg',
'path': '/reports',
'api_version': 'v4',
'params': reports_runavg_params,
'request': request
},
}
job_results = run_puppetdb_jobs(jobs, 4)
reports_run_avg = job_results['reports_run_avg']
events_class_list = job_results['events_class_list']
events_resource_list = job_results['events_resource_list']
events_status_list = job_results['events_status_list']
num_runs_avg = len(reports_run_avg)
run_avg_times = []
avg_run_time = 0
for report in reports_run_avg:
run_time = "{0:.0f}".format(
(json_to_datetime(report['end_time']) -
json_to_datetime(report['start_time'])).total_seconds())
avg_run_time += int(run_time)
run_avg_times.append(run_time)
if num_runs_avg != 0:
avg_run_time = "{0:.0f}".format(avg_run_time / num_runs_avg)
else:
avg_run_time = 0
class_event_results = []
class_resource_results = []
class_status_results = []
for item in events_class_list:
class_name = item['subject']['title']
class_total = item['skips'] + item['failures'] + item['noops'] + item[
'successes']
class_event_results.append((class_name, class_total))
for item in events_resource_list:
class_name = item['subject']['type']
class_total = item['skips'] + item['failures'] + item['noops'] + item[
'successes']
class_resource_results.append((class_name, class_total))
print(events_status_list)
if events_status_list:
for status, value in events_status_list[0].items():
print(status, value)
if value is 0 or status == 'total' or status == 'summarize_by':
continue
class_status_results.append((status, value))
context['class_events'] = class_event_results
context['class_status'] = class_status_results
context['resource_events'] = class_resource_results
context['run_times'] = run_avg_times
context['run_num'] = num_runs_avg
context['run_avg'] = avg_run_time
return render(request, 'pano/analytics/analytics.html', context)
def detailed_events(request, hashid=None):
context = {'timezones': pytz.common_timezones,
'SOURCES': AVAILABLE_SOURCES}
if request.method == 'GET':
if 'source' in request.GET:
source = request.GET.get('source')
set_server(request, source)
if request.method == 'POST':
request.session['django_timezone'] = request.POST['timezone']
return redirect(request.POST['return_url'])
source_url, source_certs, source_verify = get_server(request)
report_timestamp = request.GET.get('report_timestamp')
events_params = {
'query':
{
1: '["=","report","' + hashid + '"]'
},
'order_by':
{
'order_field':
{
'field': 'timestamp',
'order': 'asc',
},
'query_field': {'field': 'certname'},
},
}
events_list = puppetdb.api_get(
api_url=source_url,
cert=source_certs,
verify=source_verify,
path='/events',
api_version='v4',
params=puppetdb.mk_puppetdb_query(events_params),
)
environment = ''
certname = ''
event_execution_times = []
sorted_events = None
last_event_time = None
last_event_title = None
run_end_time = None
if len(events_list) != 0:
single_event = events_list[0]
environment = single_event['environment']
certname = single_event['certname']
for event in events_list:
event_title = event['resource_title']
event_start_time = json_to_datetime(event['timestamp'])
if last_event_time is None and last_event_title is None:
last_event_time = event_start_time
last_event_title = event_title
run_end_time = json_to_datetime(event['run_end_time'])
continue
else:
event_exec_time = (event_start_time - last_event_time).total_seconds()
add_event = (last_event_title, event_exec_time)
event_execution_times.append(add_event)
last_event_time = event_start_time
last_event_title = event_title
event_exec_time = (last_event_time - run_end_time).total_seconds()
add_event = [last_event_title, event_exec_time]
event_execution_times.append(add_event)
sorted_events = sorted(event_execution_times, reverse=True, key=lambda field: field[1])
if len(sorted_events) > 10:
sorted_events = sorted_events[:10]
else:
events_list = False
context['certname'] = certname
context['report_timestamp'] = report_timestamp
context['hashid'] = hashid
context['events_list'] = events_list
context['event_durations'] = sorted_events
context['environment'] = environment
return render(request, 'pano/detailed_events.html', context)
def detailed_events(request, hashid=None):
context = {
'timezones': pytz.common_timezones,
'SOURCES': AVAILABLE_SOURCES
}
if request.method == 'GET':
if 'source' in request.GET:
source = request.GET.get('source')
set_server(request, source)
if request.method == 'POST':
request.session['django_timezone'] = request.POST['timezone']
return redirect(request.POST['return_url'])
source_url, source_certs, source_verify = get_server(request)
report_timestamp = request.GET.get('report_timestamp')
events_params = {
'query': {
1: '["=","report","' + hashid + '"]'
},
'order_by': {
'order_field': {
'field': 'timestamp',
'order': 'asc',
},
'query_field': {
'field': 'certname'
},
},
}
events_list = puppetdb.api_get(
api_url=source_url,
cert=source_certs,
verify=source_verify,
path='/events',
api_version='v4',
params=puppetdb.mk_puppetdb_query(events_params),
)
environment = ''
certname = ''
event_execution_times = []
sorted_events = None
last_event_time = None
last_event_title = None
run_end_time = None
if len(events_list) != 0:
single_event = events_list[0]
environment = single_event['environment']
certname = single_event['certname']
for event in events_list:
event_title = event['resource_title']
event_start_time = json_to_datetime(event['timestamp'])
if last_event_time is None and last_event_title is None:
last_event_time = event_start_time
last_event_title = event_title
run_end_time = json_to_datetime(event['run_end_time'])
continue
else:
event_exec_time = (event_start_time -
last_event_time).total_seconds()
add_event = (last_event_title, event_exec_time)
event_execution_times.append(add_event)
last_event_time = event_start_time
last_event_title = event_title
event_exec_time = (last_event_time - run_end_time).total_seconds()
add_event = [last_event_title, event_exec_time]
event_execution_times.append(add_event)
sorted_events = sorted(event_execution_times,
reverse=True,
key=lambda field: field[1])
if len(sorted_events) > 10:
sorted_events = sorted_events[:10]
else:
events_list = False
context['certname'] = certname
context['report_timestamp'] = report_timestamp
context['hashid'] = hashid
context['events_list'] = events_list
context['event_durations'] = sorted_events
context['environment'] = environment
return render(request, 'pano/detailed_events.html', context)
