def test_fromoperator(self): op = InOperator('certname') ex = ExtractOperator() ex.add_field(["certname", "facts"]) fr = FromOperator("facts") fr.add_query(ex) fr.add_offset(10) op.add_query(fr) assert repr(op) == 'Query: ["in", "certname", ' \ '["from", "facts", ["extract", ' \ '["certname", "facts"]], ["offset", 10]]]' # last example on page # https://puppet.com/docs/puppetdb/5.1/api/query/v4/ast.html op = InOperator('certname') ex = ExtractOperator() ex.add_field('certname') fr = FromOperator('fact_contents') nd = AndOperator() nd.add( EqualsOperator("path", ["networking", "eth0", "macaddresses", 0])) nd.add(EqualsOperator("value", "aa:bb:cc:dd:ee:00")) ex.add_query(nd) fr.add_query(ex) op.add_query(fr) assert str(op) == '["in", "certname", ' \ '["from", "fact_contents", ' \ '["extract", ["certname"], ["and", ["=", "path", ' \ '["networking", "eth0", "macaddresses", 0]], ' \ '["=", "value", "aa:bb:cc:dd:ee:00"]]]]]'
def main(): """main entry point""" args = get_args() logging.basicConfig(level=get_log_level(args.verbose)) failed_nodes = [] pdb = connect() nodes = defaultdict(dict) max_age = datetime.utcnow() - timedelta(hours=args.max_age) extract = ExtractOperator() extract.add_field(['certname', FunctionOperator('count'), 'status']) extract.add_group_by(['certname', 'status']) extract.add_query(GreaterOperator('receive_time', max_age.isoformat())) # pypuppetdb does have a `reports` method which wraps `_query`. however it yields # results of type pypuppetdb.Report which expects a number of parameters e.g. hash # to be present in the result payload. however we don't extract theses values and # therefore have to resort to the more powerful private method reports = pdb._query('reports', query=extract) # pylint: disable=protected-access for report in reports: nodes[report['certname']][report['status']] = report['count'] if args.dev: failed_nodes = [hostname for hostname, node in nodes.items() if not node.get('unchanged', 0)] else: for fqdn, node in nodes.items(): # skip hosts with no unchanged reports: if node.get('unchanged', 0): continue # skip staging servers: # - hostname starting labstest* # - hostname ending dev or dev\d{4} # - hostname ending test or test\d{4} if (fqdn.startswith('labtest') or search(r'(:?dev|test)(:?\d{4})?$', fqdn.split('.')[0]) is not None): logger.debug('%s: Skipping staging host', fqdn) continue failed_nodes.append(fqdn) if len(failed_nodes) >= args.critical: print('CRITICAL: the following ({}) node(s) change every puppet run: {}'.format( len(failed_nodes), ', '.join(failed_nodes))) return 2 if len(failed_nodes) >= args.warning: print('WARNING: the following ({}) node(s) change every puppet run: {}'.format( len(failed_nodes), ', '.join(failed_nodes))) return 1 print('OK: all nodes running as expected') return 0
def _build_query(env, start, end, certname=None): """Build a extract query with optional certname and environment.""" query = ExtractOperator() query.add_field(FunctionOperator('count')) query.add_field('status') subquery = AndOperator() subquery.add(GreaterEqualOperator('producer_timestamp', start)) subquery.add(LessOperator('producer_timestamp', end)) if certname is not None: subquery.add(EqualsOperator('certname', certname)) if env != '*': subquery.add(EqualsOperator('environment', env)) query.add_query(subquery) query.add_group_by("status") return query
def _build_query(env, start, end, certname=None): """Build a extract query with optional certname and environment.""" query = ExtractOperator() query.add_field(FunctionOperator('count')) query.add_field('status') subquery = AndOperator() subquery.add(GreaterEqualOperator('start_time', start)) subquery.add(LessOperator('start_time', end)) if certname is not None: subquery.add(EqualsOperator('certname', certname)) if env != '*': subquery.add(EqualsOperator('environment', env)) query.add_query(subquery) query.add_group_by("status") return query
def main(): """main entry point""" args = get_args() logging.basicConfig(level=get_log_level(args.verbose)) pdb = connect() nodes = defaultdict(dict) max_age = datetime.utcnow() - timedelta(hours=args.max_age) extract = ExtractOperator() extract.add_field(['certname', FunctionOperator('count'), 'status']) extract.add_group_by(['certname', 'status']) extract.add_query(GreaterOperator('receive_time', max_age.isoformat())) # pypuppetdb does have a `reports` method which wraps `_query`. however it yields # results of type pypuppetdb.Report which expects a number of parameters e.g. hash # to be present in the result payload. however we don't extract theses values and # therefore have to resort to the more powerful private method reports = pdb._query('reports', query=extract) # pylint: disable=protected-access for report in reports: nodes[report['certname']][report['status']] = report['count'] failed_nodes = [ hostname for hostname, node in nodes.items() if not node.get('unchanged', 0) ] if len(failed_nodes) >= args.critical: print( 'CRITICAL: the following ({}) node(s) change every puppet run: {}'. format(len(failed_nodes), ', '.join(failed_nodes))) return 2 if len(failed_nodes) >= args.warning: print( 'WARNING: the following ({}) node(s) change every puppet run: {}'. format(len(failed_nodes), ', '.join(failed_nodes))) return 1 print('OK: all nodes running as expected') return 0
def test_with_add_group_by(self): op = ExtractOperator() op.add_field(['certname', 'fact_environment', 'catalog_environment']) op.add_query(EqualsOperator('domain', 'example.com')) op.add_group_by(["fact_environment", "catalog_environment"]) with pytest.raises(APIError): op.add_group_by({"deactivated": False}) assert repr(op) == 'Query: ["extract", '\ '["certname", "fact_environment", "catalog_environment"], '\ '["=", "domain", "example.com"], '\ '["group_by", "fact_environment", "catalog_environment"]]' assert str(op) == '["extract", '\ '["certname", "fact_environment", "catalog_environment"], '\ '["=", "domain", "example.com"], '\ '["group_by", "fact_environment", "catalog_environment"]]' assert str(op) == '["extract", ' \ '["certname", "fact_environment", "catalog_environment"], '\ '["=", "domain", "example.com"], '\ '["group_by", "fact_environment", "catalog_environment"]]'
def test_with_add_query(self): op = ExtractOperator() op.add_field(['certname', 'fact_environment', 'catalog_environment']) with pytest.raises(APIError): op.add_query({'less': 42, 'greater': 50}) op.add_query(EqualsOperator('domain', 'example.com')) assert repr(op) == 'Query: ["extract", '\ '["certname", "fact_environment", "catalog_environment"], '\ '["=", "domain", "example.com"]]' assert str(op) == '["extract", '\ '["certname", "fact_environment", "catalog_environment"], '\ '["=", "domain", "example.com"]]' assert str(op) == '["extract", ' \ '["certname", "fact_environment", "catalog_environment"], '\ '["=", "domain", "example.com"]]' with pytest.raises(APIError): op.add_query(GreaterOperator("processorcount", 1))
def index(env): """This view generates the index page and displays a set of metrics and latest reports on nodes fetched from PuppetDB. :param env: Search for nodes in this (Catalog and Fact) environment :type env: :obj:`string` """ envs = environments() metrics = {'num_nodes': 0, 'num_resources': 0, 'avg_resources_node': 0} check_env(env, envs) if env == '*': query = app.config['OVERVIEW_FILTER'] prefix = 'puppetlabs.puppetdb.population' db_version = get_db_version(puppetdb) query_type, metric_version = metric_params(db_version) num_nodes = get_or_abort(puppetdb.metric, "{0}{1}".format( prefix, ':%sname=num-nodes' % query_type), version=metric_version) num_resources = get_or_abort(puppetdb.metric, "{0}{1}".format( prefix, ':%sname=num-resources' % query_type), version=metric_version) metrics['num_nodes'] = num_nodes['Value'] metrics['num_resources'] = num_resources['Value'] try: # Compute our own average because avg_resources_node['Value'] # returns a string of the format "num_resources/num_nodes" # example: "1234/9" instead of doing the division itself. metrics['avg_resources_node'] = "{0:10.0f}".format( (num_resources['Value'] / num_nodes['Value'])) except ZeroDivisionError: metrics['avg_resources_node'] = 0 else: query = AndOperator() query.add(EqualsOperator('catalog_environment', env)) num_nodes_query = ExtractOperator() num_nodes_query.add_field(FunctionOperator('count')) num_nodes_query.add_query(query) if app.config['OVERVIEW_FILTER'] is not None: query.add(app.config['OVERVIEW_FILTER']) num_resources_query = ExtractOperator() num_resources_query.add_field(FunctionOperator('count')) num_resources_query.add_query(EqualsOperator("environment", env)) num_nodes = get_or_abort(puppetdb._query, 'nodes', query=num_nodes_query) num_resources = get_or_abort(puppetdb._query, 'resources', query=num_resources_query) metrics['num_nodes'] = num_nodes[0]['count'] metrics['num_resources'] = num_resources[0]['count'] try: metrics['avg_resources_node'] = "{0:10.0f}".format( (num_resources[0]['count'] / num_nodes[0]['count'])) except ZeroDivisionError: metrics['avg_resources_node'] = 0 nodes = get_or_abort(puppetdb.nodes, query=query, unreported=app.config['UNRESPONSIVE_HOURS'], with_status=True, with_event_numbers=app.config['WITH_EVENT_NUMBERS']) nodes_overview = [] stats = { 'changed': 0, 'unchanged': 0, 'failed': 0, 'unreported': 0, 'noop': 0 } for node in nodes: if node.status == 'unreported': stats['unreported'] += 1 elif node.status == 'changed': stats['changed'] += 1 elif node.status == 'failed': stats['failed'] += 1 elif node.status == 'noop': stats['noop'] += 1 else: stats['unchanged'] += 1 if node.status != 'unchanged': nodes_overview.append(node) return render_template('index.html', metrics=metrics, nodes=nodes_overview, stats=stats, envs=envs, current_env=env)
def radiator(env): """This view generates a simplified monitoring page akin to the radiator view in puppet dashboard """ envs = environments() check_env(env, envs) if env == '*': db_version = get_db_version(puppetdb) query_type, metric_version = metric_params(db_version) query = None metrics = get_or_abort( puppetdb.metric, 'puppetlabs.puppetdb.population:%sname=num-nodes' % query_type, version=metric_version) num_nodes = metrics['Value'] else: query = AndOperator() metric_query = ExtractOperator() query.add(EqualsOperator("catalog_environment", env)) metric_query.add_field(FunctionOperator('count')) metric_query.add_query(query) metrics = get_or_abort(puppetdb._query, 'nodes', query=metric_query) num_nodes = metrics[0]['count'] nodes = puppetdb.nodes(query=query, unreported=app.config['UNRESPONSIVE_HOURS'], with_status=True) stats = { 'changed_percent': 0, 'changed': 0, 'failed_percent': 0, 'failed': 0, 'noop_percent': 0, 'noop': 0, 'skipped_percent': 0, 'skipped': 0, 'unchanged_percent': 0, 'unchanged': 0, 'unreported_percent': 0, 'unreported': 0, } for node in nodes: if node.status == 'unreported': stats['unreported'] += 1 elif node.status == 'changed': stats['changed'] += 1 elif node.status == 'failed': stats['failed'] += 1 elif node.status == 'noop': stats['noop'] += 1 elif node.status == 'skipped': stats['skipped'] += 1 else: stats['unchanged'] += 1 try: stats['changed_percent'] = int(100 * (stats['changed'] / float(num_nodes))) stats['failed_percent'] = int(100 * stats['failed'] / float(num_nodes)) stats['noop_percent'] = int(100 * stats['noop'] / float(num_nodes)) stats['skipped_percent'] = int(100 * (stats['skipped'] / float(num_nodes))) stats['unchanged_percent'] = int( 100 * (stats['unchanged'] / float(num_nodes))) stats['unreported_percent'] = int( 100 * (stats['unreported'] / float(num_nodes))) except ZeroDivisionError: stats['changed_percent'] = 0 stats['failed_percent'] = 0 stats['noop_percent'] = 0 stats['skipped_percent'] = 0 stats['unchanged_percent'] = 0 stats['unreported_percent'] = 0 if ('Accept' in request.headers and request.headers["Accept"] == 'application/json'): return jsonify(**stats) return render_template('radiator.html', stats=stats, total=num_nodes)