def test_date_unreported_within_two_hours(self): """ Should return True since the node has not reported within the default value of 2 hours. """ date = (datetime.utcnow() - timedelta(hours=3)).strftime("%Y-%m-%dT%H:%M:%S.%fZ") results = is_unreported(date) self.assertEquals(results, True)
def test_date_unreported_within_two_hours(self): """ Should return True since the node has not reported within the default value of 2 hours. """ date = (datetime.utcnow() - timedelta(hours=3)).strftime('%Y-%m-%dT%H:%M:%S.%fZ') results = is_unreported(date) self.assertEquals(results, True)
def test_unreported_date_with_hours_set_to_24_hours(self): """ Test timestamp set to 25 hours ago, it should count as a unreported timestamp since the unreported time is set to 24 hours. """ date = (datetime.utcnow() - timedelta(hours=25)).strftime("%Y-%m-%dT%H:%M:%S.%fZ") results = is_unreported(date, unreported=24 * 60) self.assertEquals(results, True)
def test_unreported_date_with_hours_set_to_24_hours(self): """ Test timestamp set to 25 hours ago, it should count as a unreported timestamp since the unreported time is set to 24 hours. """ date = (datetime.utcnow() - timedelta(hours=25)).strftime('%Y-%m-%dT%H:%M:%S.%fZ') results = is_unreported(date, unreported=24) self.assertEquals(results, True)
def test_reported_date_with_hours_set_to_30_minutes_using_float_value(self): """ Test unreported parameter to is_unreported accepts float value. It is set to .5 hours which is effectively 30 minutes. With a time set to 15 minutes ago it should return that the node is not unreported. """ date = (datetime.utcnow() - timedelta(minutes=15)).strftime('%Y-%m-%dT%H:%M:%S.%fZ') results = is_unreported(date, unreported=.5) self.assertEquals(results, False)
def test_none_date(self): """ Should fail because if there is no report timestamp the node has not managed to complete a puppet run. """ """ :return: """ date = None results = is_unreported(date) self.assertEquals(results, True)
def test_reported_date_with_hours_set_to_30_minutes_using_float_value( self): """ Test unreported parameter to is_unreported accepts float value. It is set to .5 hours which is effectively 30 minutes. With a time set to 15 minutes ago it should return that the node is not unreported. """ date = (datetime.utcnow() - timedelta(minutes=15)).strftime('%Y-%m-%dT%H:%M:%S.%fZ') results = is_unreported(date, unreported=.5) self.assertEquals(results, False)
def dictstatus(node_dict, reports_dict, status_dict, sort=True, sortby=None, asc=False, get_status="all", puppet_run_time=PUPPET_RUN_INTERVAL): """ :param node_dict: dict :param status_dict: dict :param sortby: Takes a field name to sort by 'certname', 'latestCatalog', 'latestReport', 'latestFacts', 'success', 'noop', 'failure', 'skipped' :param get_status: Status type to return. all, changed, failed, unreported, noops :return: tuple(tuple,tuple) node_dict input: { 'certname': { "name": <string>, "deactivated": <timestamp>, "catalog_timestamp": <timestamp>, "facts_timestamp": <timestamp>, "report_timestamp": <timestamp> }, } -------------------------------- status_dict input: { 'certname': { "subject-type": "certname", "subject": { "title": "foo.local" }, "failures": 0, "successes": 2, "noops": 0, "skips": 1 }, } """ # The merged_list tuple should look like this. # ( # ('certname', 'latestCatalog', 'latestReport', 'latestFacts', 'success', 'noop', 'failure', 'skipped'), # ) def check_failed_compile(report_timestamp, fact_timestamp, catalog_timestamp, puppet_run_interval=puppet_run_time): """ :param report_timestamp: str :param fact_timestamp: str :param catalog_timestamp: str :return: Bool Returns False if the compiled run has not failed Returns True if the compiled run has failed """ if report_timestamp is None or catalog_timestamp is None or fact_timestamp is None: return True # check if the fact report is older than puppet_run_time by double the run time report_time = json_to_datetime(report_timestamp) fact_time = json_to_datetime(fact_timestamp) catalog_time = json_to_datetime(catalog_timestamp) # Report time, fact time and catalog time should all be run within (PUPPET_RUN_INTERVAL / 2) # minutes of each other diffs = dict() # Time elapsed between fact time and catalog time diffs['catalog_fact'] = catalog_time - fact_time diffs['fact_catalog'] = fact_time - catalog_time # Time elapsed between fact time and report time diffs['report_fact'] = report_time - fact_time diffs['fact_report'] = fact_time - report_time # Time elapsed between report and catalog diffs['report_catalog'] = report_time - catalog_time diffs['catalog_report'] = catalog_time - report_time for key, value in diffs.items(): if value > timedelta(minutes=puppet_run_interval / 2): return True return False def append_list(n_data, s_data, m_list, r_status): if type(n_data) is not dict or type(s_data) is not dict and type( m_list) is not list and not r_status: raise ValueError( 'Incorrect type given as input. Expects n_data, s_data as dict and m_list as list.' ) m_list.append(( n_data['certname'], filters.date( localtime(json_to_datetime(n_data['catalog_timestamp'])), 'Y-m-d H:i:s') if n_data['catalog_timestamp'] is not None else '', filters.date( localtime(json_to_datetime(n_data['report_timestamp'])), 'Y-m-d H:i:s') if n_data['report_timestamp'] is not None else '', filters.date( localtime(json_to_datetime(n_data['facts_timestamp'])), 'Y-m-d H:i:s') if n_data['facts_timestamp'] is not None else '', s_data.get('successes', 0), s_data.get('noops', 0), s_data.get('failures', 0), s_data.get('skips', 0), r_status, )) return m_list sortables = { 'certname': 0, 'catalog_timestamp': 1, 'report_timestamp': 2, 'facts_timestamp': 3, 'successes': 4, 'noops': 5, 'failures': 6, 'skips': 7, } if sortby: # Sort by the field recieved, if valid field was not supplied, fallback # to report sortbycol = sortables.get(sortby, 2) else: sortbycol = 2 # if sortbycol is 4, 5, 6 or 7 ( a different list creation method must be used. merged_list = [] failed_list = [] unreported_list = [] changed_list = [] pending_list = [] mismatch_list = [] # if sort field is certname or catalog/report/facts_timestamp then we will sort this way # or if the get_status is set to "not_all" indicating that the dashboard wants info. if get_status != 'all': for node_name, data in node_dict.items(): node_is_unreported = False node_has_mismatching_timestamps = False # Check if its unreported. if is_unreported(data['report_timestamp']): node_is_unreported = True if check_failed_compile( report_timestamp=data.get('report_timestamp', None), fact_timestamp=data.get('facts_timestamp', None), catalog_timestamp=data.get('catalog_timestamp', None)): node_has_mismatching_timestamps = True # Check for the latest report. if node_name in reports_dict: # Check which status the run is. report_status = reports_dict[node_name]['status'] """ Can be used later but right now we just utilize the event-counts response. # Dictify the metrics for the report. metrics_data = {item['category'] + '-' + item['name']: item for item in reports_dict[node_name]['metrics']['data']} """ # Collect the number of events for each node in its latest report. if node_name in status_dict: # If the puppet status is changed but there are noop events set to pending. if report_status == 'unchanged' and status_dict[node_name][ 'noops'] > 0: report_status = 'pending' # If there is no status events for the latest report then send an empty list to append_list function else: # Add an empty status_dict for the node. status_dict[node_name] = {} # If theres no report for this node ... panic no idea how to handle this yet. If it can even happen? # Check if its an unreported longer than the unreported time. if node_is_unreported is True: # Append to the unreported list. unreported_list = append_list(data, status_dict[node_name], unreported_list, report_status) # If its got mismatching timestamps put it in the mismatching list if node_has_mismatching_timestamps is True: mismatch_list = append_list(data, status_dict[node_name], mismatch_list, report_status) # If the node is not unreported or has mismatching timestamps.. proceed to put in the correct lists. if report_status == 'changed': changed_list = append_list(data, status_dict[node_name], changed_list, report_status) elif report_status == 'failed': failed_list = append_list(data, status_dict[node_name], failed_list, report_status) elif report_status == 'pending': pending_list = append_list(data, status_dict[node_name], pending_list, report_status) elif sortbycol <= 3 and get_status == 'all': for node_name, data in node_dict.items(): # Check for the latest report. if node_name in reports_dict: # Check which status the run is. report_status = reports_dict[node_name]['status'] """ Can be used later but right now we just utilize the event-counts response. # Dictify the metrics for the report. metrics_data = {item['category'] + '-' + item['name']: item for item in reports_dict[node_name]['metrics']['data']} """ # Collect the number of events for each node in its latest report. if node_name in status_dict: # If the puppet status is changed but there are noop events set to pending. if status_dict[node_name]: if report_status == 'unchanged' and status_dict[node_name][ 'noops'] > 0: report_status = 'pending' # If there is no status events for the latest report then send an empty list to append_list function else: # Add an empty status_dict for the node. status_dict[node_name] = {} merged_list = append_list(data, status_dict[node_name], merged_list, report_status) # Only used when orderby is a status field. elif sortbycol >= 4 and get_status == 'all': for status in status_dict: if status['subject']['title'] in reports_dict: # Check which status the run is. report_status = reports_dict[status['subject'] ['title']]['status'] if status['subject']['title'] in node_dict: merged_list = append_list( node_dict[status['subject']['title']], status, merged_list, report_status) # Sort the lists if sort is True if sort and get_status == 'all': return sort_table(merged_list, order=asc, col=sortbycol) elif sort and get_status != 'all': sorted_unreported_list = sort_table(unreported_list, order=asc, col=sortbycol) sorted_changed_list = sort_table(changed_list, order=asc, col=sortbycol) sorted_failed_list = sort_table(failed_list, order=asc, col=sortbycol) sorted_mismatch_list = sort_table(mismatch_list, order=asc, col=sortbycol) sorted_pending_list = sort_table(pending_list, order=asc, col=sortbycol) return sorted_failed_list, \ sorted_changed_list, \ sorted_unreported_list, \ sorted_mismatch_list, \ sorted_pending_list if get_status == 'all': return merged_list else: return failed_list, changed_list, unreported_list, mismatch_list, pending_list
def dictstatus(node_list, reports_dict, status_dict, sort=True, sortby=None, asc=False, get_status="all", puppet_run_time=PUPPET_RUN_INTERVAL): """ :param node_list: dict :param status_dict: dict :param sortby: Takes a field name to sort by 'certname', 'latestCatalog', 'latestReport', 'latestFacts', 'success', 'noop', 'failure', 'skipped' :param get_status: Status type to return. all, changed, failed, unreported, noops :return: tuple(tuple,tuple) node_dict input: { 'certname': { "name": <string>, "deactivated": <timestamp>, "catalog_timestamp": <timestamp>, "facts_timestamp": <timestamp>, "report_timestamp": <timestamp> }, } -------------------------------- status_dict input: { 'certname': { "subject-type": "certname", "subject": { "title": "foo.local" }, "failures": 0, "successes": 2, "noops": 0, "skips": 1 }, } """ # The merged_list tuple should look like this. # ( # ('certname', 'latestCatalog', 'latestReport', 'latestFacts', 'success', 'noop', 'failure', 'skipped'), # ) def check_failed_compile(report_timestamp, fact_timestamp, catalog_timestamp, puppet_run_interval=puppet_run_time): """ :param report_timestamp: str :param fact_timestamp: str :param catalog_timestamp: str :return: Bool Returns False if the compiled run has not failed Returns True if the compiled run has failed """ if report_timestamp is None or catalog_timestamp is None or fact_timestamp is None: return True # check if the fact report is older than puppet_run_time by double the run time report_time = json_to_datetime(report_timestamp) fact_time = json_to_datetime(fact_timestamp) catalog_time = json_to_datetime(catalog_timestamp) # Report time, fact time and catalog time should all be run within (PUPPET_RUN_INTERVAL / 2) # minutes of each other diffs = dict() # Time elapsed between fact time and catalog time diffs['catalog_fact'] = catalog_time - fact_time diffs['fact_catalog'] = fact_time - catalog_time # Time elapsed between fact time and report time diffs['report_fact'] = report_time - fact_time diffs['fact_report'] = fact_time - report_time # Time elapsed between report and catalog diffs['report_catalog'] = report_time - catalog_time diffs['catalog_report'] = catalog_time - report_time for key, value in diffs.items(): if value > timedelta(minutes=puppet_run_interval / 2): return True return False def append_list(n_data, s_data, m_list, r_status): if type(n_data) is not dict or type(s_data) is not dict and type(m_list) is not list and not r_status: raise ValueError('Incorrect type given as input. Expects n_data, s_data as dict and m_list as list.') m_list.append(( n_data['certname'], filters.date(localtime(json_to_datetime(n_data['catalog_timestamp'])), 'Y-m-d H:i:s') if n_data[ 'catalog_timestamp'] is not None else '', filters.date(localtime(json_to_datetime(n_data['report_timestamp'])), 'Y-m-d H:i:s') if n_data[ 'report_timestamp'] is not None else '', filters.date(localtime(json_to_datetime(n_data['facts_timestamp'])), 'Y-m-d H:i:s') if n_data[ 'facts_timestamp'] is not None else '', s_data.get('successes', 0), s_data.get('noops', 0), s_data.get('failures', 0), s_data.get('skips', 0), r_status, )) return m_list sortables = { 'certname': 0, 'catalog_timestamp': 1, 'report_timestamp': 2, 'facts_timestamp': 3, 'successes': 4, 'noops': 5, 'failures': 6, 'skips': 7, } if sortby: # Sort by the field recieved, if valid field was not supplied, fallback # to report sortbycol = sortables.get(sortby, 2) else: sortbycol = 2 # if sortbycol is 4, 5, 6 or 7 ( a different list creation method must be used. merged_list = [] failed_list = [] unreported_list = [] changed_list = [] pending_list = [] mismatch_list = [] # if sort field is certname or catalog/report/facts_timestamp then we will sort this way # or if the get_status is set to "not_all" indicating that the dashboard wants info. if get_status != 'all': for node in node_list: node_is_unreported = False node_has_mismatching_timestamps = False # Check if its unreported. if is_unreported(node['report_timestamp']): node_is_unreported = True if check_failed_compile(report_timestamp=node.get('report_timestamp', None), fact_timestamp=node.get('facts_timestamp', None), catalog_timestamp=node.get('catalog_timestamp', None)): node_has_mismatching_timestamps = True # Check for the latest report. if node['certname'] in reports_dict: # Check which status the run is. report_status = reports_dict[node['certname']]['status'] """ Can be used later but right now we just utilize the event-counts response. # Dictify the metrics for the report. metrics_data = {item['category'] + '-' + item['name']: item for item in reports_dict[node_name]['metrics']['data']} """ # Collect the number of events for each node in its latest report. if node['certname'] in status_dict: # If the puppet status is changed but there are noop events set to pending. if report_status == 'unchanged' and status_dict[node['certname']]['noops'] > 0: report_status = 'pending' # If there is no status events for the latest report then send an empty list to append_list function else: # Add an empty status_dict for the node. status_dict[node['certname']] = {} # If theres no report for this node ... panic no idea how to handle this yet. If it can even happen? # Check if its an unreported longer than the unreported time. if node_is_unreported is True: # Append to the unreported list. unreported_list = append_list(node, status_dict[node['certname']], unreported_list, report_status) # If its got mismatching timestamps put it in the mismatching list if node_has_mismatching_timestamps is True: mismatch_list = append_list(node, status_dict[node['certname']], mismatch_list, report_status) # If the node is not unreported or has mismatching timestamps.. proceed to put in the correct lists. if report_status == 'changed': changed_list = append_list(node, status_dict[node['certname']], changed_list, report_status) elif report_status == 'failed': failed_list = append_list(node, status_dict[node['certname']], failed_list, report_status) elif report_status == 'pending': pending_list = append_list(node, status_dict[node['certname']], pending_list, report_status) elif sortbycol <= 3 and get_status == 'all': for node in node_list: # Check for the latest report. if node['certname'] in reports_dict: # Check which status the run is. report_status = reports_dict[node['certname']]['status'] """ Can be used later but right now we just utilize the event-counts response. # Dictify the metrics for the report. metrics_data = {item['category'] + '-' + item['name']: item for item in reports_dict[node_name]['metrics']['data']} """ # Collect the number of events for each node in its latest report. if node['certname'] in status_dict: # If the puppet status is changed but there are noop events set to pending. if status_dict[node['certname']]: if report_status == 'unchanged' and status_dict[node['certname']]['noops'] > 0: report_status = 'pending' # If there is no status events for the latest report then send an empty list to append_list function else: # Add an empty status_dict for the node. status_dict[node['certname']] = {} merged_list = append_list(node, status_dict[node['certname']], merged_list, report_status) # Only used when orderby is a status field. elif sortbycol >= 4 and get_status == 'all': sort = True node_dict = {item['certname']: item for item in node_list} for status, value in status_dict.items(): if value['subject']['title'] in reports_dict: # Check which status the run is. report_status = reports_dict[value['subject']['title']]['status'] if value['subject']['title'] in node_dict and report_status: merged_list = append_list(node_dict[value['subject']['title']], value, merged_list, report_status) # Sort the lists if sort is True if sort and get_status == 'all': return sort_table(merged_list, order=asc, col=sortbycol) elif sort and get_status != 'all': sorted_unreported_list = sort_table(unreported_list, order=asc, col=sortbycol) sorted_changed_list = sort_table(changed_list, order=asc, col=sortbycol) sorted_failed_list = sort_table(failed_list, order=asc, col=sortbycol) sorted_mismatch_list = sort_table(mismatch_list, order=asc, col=sortbycol) sorted_pending_list = sort_table(pending_list, order=asc, col=sortbycol) return sorted_failed_list, \ sorted_changed_list, \ sorted_unreported_list, \ sorted_mismatch_list, \ sorted_pending_list if get_status == 'all': return merged_list else: return failed_list, changed_list, unreported_list, mismatch_list, pending_list
def dictstatus(node_dict, status_dict, sort=True, sortby=None, asc=False, get_status="all", puppet_run_time=PUPPET_RUN_INTERVAL): """ :param node_dict: dict :param status_dict: dict :param sortby: Takes a field name to sort by 'certname', 'latestCatalog', 'latestReport', 'latestFacts', 'success', 'noop', 'failure', 'skipped' :param get_status: Status type to return. all, changed, failed, unreported, noops :return: tuple(tuple,tuple) node_dict input: [ { "name": <string>, "deactivated": <timestamp>, "catalog_timestamp": <timestamp>, "facts_timestamp": <timestamp>, "report_timestamp": <timestamp> }, ] -------------------------------- status_dict input: [ { "subject-type": "certname", "subject": { "title": "foo.local" }, "failures": 0, "successes": 2, "noops": 0, "skips": 1 }, ] """ # The merged_list tuple should look like this. # ( # ('certname', 'latestCatalog', 'latestReport', 'latestFacts', 'success', 'noop', 'failure', 'skipped'), # ) def check_failed_compile(report_timestamp, fact_timestamp, catalog_timestamp, puppet_run_interval=puppet_run_time): """ :param report_timestamp: str :param fact_timestamp: str :param catalog_timestamp: str :return: Bool Returns False if the compiled run has not failed Returns True if the compiled run has failed """ if report_timestamp is None or catalog_timestamp is None or fact_timestamp is None: return True # check if the fact report is older than puppet_run_time by double the run time report_time = json_to_datetime(report_timestamp) fact_time = json_to_datetime(fact_timestamp) catalog_time = json_to_datetime(catalog_timestamp) # Report time, fact time and catalog time should all be run within (PUPPET_RUN_INTERVAL / 2) # minutes of each other diffs = dict() # Time elapsed between fact time and catalog time diffs['catalog_fact'] = catalog_time - fact_time diffs['fact_catalog'] = fact_time - catalog_time # Time elapsed between fact time and report time diffs['report_fact'] = report_time - fact_time diffs['fact_report'] = fact_time - report_time # Time elapsed between report and catalog diffs['report_catalog'] = report_time - catalog_time diffs['catalog_report'] = catalog_time - report_time for key, value in diffs.items(): if value > timedelta(minutes=puppet_run_interval / 2): return True return False sortables = { 'certname': 0, 'catalog-timestamp': 1, 'report-timestamp': 2, 'facts-timestamp': 3, 'successes': 4, 'noops': 5, 'failures': 6, 'skips': 7, } if sortby: # Sort by the field recieved, if valid field was not supplied, fallback # to report sortbycol = sortables.get(sortby, 2) else: sortbycol = 2 # if sortbycol is 4, 5, 6 or 7 ( a different list creation method must be used. merged_list = [] failed_list = [] unreported_list = [] changed_list = [] pending_list = [] mismatch_list = [] def append_list(n_data, s_data, m_list): if type(n_data) is not dict or type(s_data) is not dict and type(m_list) is not list: raise ValueError('Incorrect type given as input. Expects n_data, s_data as dict and m_list as list.') m_list.append(( n_data['certname'], filters.date(localtime(json_to_datetime(n_data['catalog-timestamp'])), 'Y-m-d H:i:s') if n_data['catalog-timestamp'] is not None else '', filters.date(localtime(json_to_datetime(n_data['report-timestamp'])), 'Y-m-d H:i:s') if n_data['report-timestamp'] is not None else '', filters.date(localtime(json_to_datetime(n_data['facts-timestamp'])), 'Y-m-d H:i:s') if n_data['facts-timestamp'] is not None else '', s_data.get('successes', 0), s_data.get('noops', 0), s_data.get('failures', 0), s_data.get('skips', 0), )) return m_list # if sort field is certname or catalog/report/facts-timestamp then we will sort this way # or if the get_status is set to "not_all" indicating that the dashboard wants info. if get_status != 'all': for node in node_dict: found_node = False for status in status_dict: if node['certname'] == status['subject']['title']: found_node = True # If the node has failures if status['failures'] > 0: failed_list = append_list(node, status, failed_list) if check_failed_compile(report_timestamp=node.get('report-timestamp', None), fact_timestamp=node.get('facts-timestamp', None), catalog_timestamp=node.get('catalog-timestamp', None)): mismatch_list = append_list(node, status, mismatch_list) # If the node is unreported if is_unreported(node['report-timestamp']): unreported_list = append_list(node, status, unreported_list) # If the node has noops if status['noops'] > 0 \ and status['successes'] == 0 \ and status['failures'] == 0 \ and status['skips'] == 0: pending_list = append_list(node, status, pending_list) # The node was found in the events list so it has to have changed changed_list = append_list(node, status, changed_list) # Found the node in events list so we can break this loop break if found_node is False: # If the node is unreported if is_unreported(node['report-timestamp']): unreported_list = append_list(node, dict(), unreported_list) if check_failed_compile(report_timestamp=node.get('report-timestamp', None), fact_timestamp=node.get('facts-timestamp', None), catalog_timestamp=node.get('catalog-timestamp', None)): mismatch_list = append_list(node, dict(), mismatch_list) elif sortbycol <= 3 and get_status == 'all': for node in node_dict: found_node = False for status in status_dict: if node['certname'] == status['subject']['title']: found_node = True merged_list = append_list(node, status, merged_list) # Found the node in events list so we can break this loop break if found_node is False: merged_list = append_list(node, dict(), merged_list) # Only used when orderby is a status field. elif sortbycol >= 4 and get_status == 'all': for status in status_dict: found_node = False for node in node_dict: if node['certname'] == status['subject']['title']: found_node = True merged_list = append_list(node, status, merged_list) break # Sort the lists if sort is True if sort and get_status == 'all': return sort_table(merged_list, order=asc, col=sortbycol) elif sort and get_status != 'all': sorted_unreported_list = sort_table(unreported_list, order=asc, col=sortbycol) sorted_changed_list = sort_table(changed_list, order=asc, col=sortbycol) sorted_failed_list = sort_table(failed_list, order=asc, col=sortbycol) sorted_mismatch_list = sort_table(mismatch_list, order=asc, col=sortbycol) sorted_pending_list = sort_table(pending_list, order=asc, col=sortbycol) return sorted_failed_list, \ sorted_changed_list, \ sorted_unreported_list, \ sorted_mismatch_list, \ sorted_pending_list if get_status == 'all': return merged_list else: return failed_list, changed_list, unreported_list, mismatch_list, pending_list