Example #1
0
def reports_node(env, node_name, page):
    """Fetches all reports for a node and processes them eventually rendering
    a table displaying those reports.

    :param env: Search for reports in this environment
    :type env: :obj:`string`
    :param node_name: Find the reports whose certname match this value
    :type node_name: :obj:`string`
    :param page: Calculates the offset of the query based on the report count
        and this value
    :type page: :obj:`int`
    """
    envs = environments()
    check_env(env, envs)

    if env == '*':
        query = '["=", "certname", "{0}"]]'.format(node_name)
    else:
        query='["and",' \
            '["=", "environment", "{0}"],' \
            '["=", "certname", "{1}"]]'.format(env, node_name),

    reports = get_or_abort(puppetdb.reports,
        query=query,
        limit=app.config['REPORTS_COUNT'],
        offset=(page-1) * app.config['REPORTS_COUNT'],
        order_by='[{"field": "start_time", "order": "desc"}]')
    total = get_or_abort(puppetdb._query,
        'reports',
        query='["extract", [["function", "count"]],' \
            '["and", ["=", "environment", "{0}"], ["=", "certname", "{1}"]]]'.format(
            env,
            node_name))
    total = total[0]['count']
    reports, reports_events = tee(reports)
    report_event_counts = {}

    if total == 0 and page != 1:
        abort(404)

    for report in reports_events:
        counts = get_or_abort(puppetdb.event_counts,
            query='["and",' \
                '["=", "environment", "{0}"],' \
                '["=", "certname", "{1}"],' \
                '["=", "report", "{2}"]]'.format(env, report.node, report.hash_),
            summarize_by="certname")
        try:
            report_event_counts[report.hash_] = counts[0]
        except IndexError:
            report_event_counts[report.hash_] = {}
    return render_template(
        'reports.html',
        reports=reports,
        reports_count=app.config['REPORTS_COUNT'],
        report_event_counts=report_event_counts,
        pagination=Pagination(page, app.config['REPORTS_COUNT'], total),
        envs=envs,
        current_env=env)
Example #2
0
def node(env, node_name):
    """Display a dashboard for a node showing as much data as we have on that
    node. This includes facts and reports but not Resources as that is too
    heavy to do within a single request.

    :param env: Ensure that the node, facts and reports are in this environment
    :type env: :obj:`string`
    """
    envs = environments()
    check_env(env, envs)
    query = AndOperator()

    if env != '*':
        query.add(EqualsOperator("environment", env))

    query.add(EqualsOperator("certname", node_name))

    node = get_or_abort(puppetdb.node, node_name)
    facts = node.facts()
    reports = get_or_abort(puppetdb.reports,
                           query=query,
                           limit=app.config['REPORTS_COUNT'],
                           order_by=DEFAULT_ORDER_BY)
    reports, reports_events = tee(reports)
    report_event_counts = {}

    for report in reports_events:
        report_event_counts[report.hash_] = {}

        for event in report.events():
            if event.status == 'success':
                try:
                    report_event_counts[report.hash_]['successes'] += 1
                except KeyError:
                    report_event_counts[report.hash_]['successes'] = 1
            elif event.status == 'failure':
                try:
                    report_event_counts[report.hash_]['failures'] += 1
                except KeyError:
                    report_event_counts[report.hash_]['failures'] = 1
            elif event.status == 'noop':
                try:
                    report_event_counts[report.hash_]['noops'] += 1
                except KeyError:
                    report_event_counts[report.hash_]['noops'] = 1
            elif event.status == 'skipped':
                try:
                    report_event_counts[report.hash_]['skips'] += 1
                except KeyError:
                    report_event_counts[report.hash_]['skips'] = 1
    return render_template(
        'node.html',
        node=node,
        facts=yield_or_stop(facts),
        reports=yield_or_stop(reports),
        reports_count=app.config['REPORTS_COUNT'],
        report_event_counts=report_event_counts,
        envs=envs,
        current_env=env)
Example #3
0
def node(env, node_name):
    """Display a dashboard for a node showing as much data as we have on that
    node. This includes facts and reports but not Resources as that is too
    heavy to do within a single request.

    :param env: Ensure that the node, facts and reports are in this environment
    :type env: :obj:`string`
    """
    envs = environments()
    check_env(env, envs)

    if env == '*':
        query = '["=", "certname", "{0}"]]'.format(node_name)
    else:
        query='["and", ["=", "environment", "{0}"],' \
            '["=", "certname", "{1}"]]'.format(env, node_name),

    node = get_or_abort(puppetdb.node, node_name)
    facts = node.facts()
    reports = get_or_abort(puppetdb.reports,
        query=query,
        limit=app.config['REPORTS_COUNT'],
        order_by='[{"field": "start_time", "order": "desc"}]')
    reports, reports_events = tee(reports)
    report_event_counts = {}

    for report in reports_events:
        report_event_counts[report.hash_] = {}

        for event in report.events():
            if event.status == 'success':
                try:
                    report_event_counts[report.hash_]['successes'] += 1
                except KeyError:
                    report_event_counts[report.hash_]['successes'] = 1
            elif event.status == 'failure':
                try:
                    report_event_counts[report.hash_]['failures'] += 1
                except KeyError:
                    report_event_counts[report.hash_]['failures'] = 1
            elif event.status == 'noop':
                try:
                    report_event_counts[report.hash_]['noops'] += 1
                except KeyError:
                    report_event_counts[report.hash_]['noops'] = 1
            elif event.status == 'skipped':
                try:
                    report_event_counts[report.hash_]['skips'] += 1
                except KeyError:
                    report_event_counts[report.hash_]['skips'] = 1
    return render_template(
        'node.html',
        node=node,
        facts=yield_or_stop(facts),
        reports=yield_or_stop(reports),
        reports_count=app.config['REPORTS_COUNT'],
        report_event_counts=report_event_counts,
        envs=envs,
        current_env=env)
Example #4
0
def reports(env, page):
    """Displays a list of reports and status from all nodes, retreived using the
    reports endpoint, sorted by start_time.

    :param env: Search for all reports in this environment
    :type env: :obj:`string`
    :param page: Calculates the offset of the query based on the report count
        and this value
    :type page: :obj:`int`
    """
    envs = environments()
    check_env(env, envs)

    if env == '*':
        reports_query = None
        total_query = '["extract", [["function", "count"]], ["~", "certname", ""]]'
    else:
        reports_query = '["=", "environment", "{0}"]'.format(env)
        total_query = '["extract", [["function", "count"]],'\
            '["and", ["=", "environment", "{0}"]]]'.format(env)

    reports = get_or_abort(puppetdb.reports,
        query=reports_query,
        limit=app.config['REPORTS_COUNT'],
        offset=(page-1) * app.config['REPORTS_COUNT'],
        order_by='[{"field": "start_time", "order": "desc"}]')
    total = get_or_abort(puppetdb._query,
        'reports',
        query=total_query)
    total = total[0]['count']
    reports, reports_events = tee(reports)
    report_event_counts = {}

    if total == 0 and page != 1:
        abort(404)

    for report in reports_events:
        counts = get_or_abort(puppetdb.event_counts,
            query='["and",' \
                '["=", "environment", "{0}"],' \
                '["=", "certname", "{1}"],' \
                '["=", "report", "{2}"]]'.format(
                    env,
                    report.node,
                    report.hash_),
            summarize_by="certname")
        try:
            report_event_counts[report.hash_] = counts[0]
        except IndexError:
            report_event_counts[report.hash_] = {}
    return Response(stream_with_context(stream_template(
        'reports.html',
        reports=yield_or_stop(reports),
        reports_count=app.config['REPORTS_COUNT'],
        report_event_counts=report_event_counts,
        pagination=Pagination(page, app.config['REPORTS_COUNT'], total),
        envs=envs,
        current_env=env)))
Example #5
0
def index():
    """This view generates the index page and displays a set of metrics and
    latest reports on nodes fetched from PuppetDB.
    """
    # TODO: Would be great if we could parallelize this somehow, doing these
    # requests in sequence is rather pointless.
    prefix = 'com.puppetlabs.puppetdb.query.population'
    num_nodes = get_or_abort(
        puppetdb.metric,
        "{0}{1}".format(prefix, ':type=default,name=num-nodes'))
    num_resources = get_or_abort(
        puppetdb.metric,
        "{0}{1}".format(prefix, ':type=default,name=num-resources'))
    avg_resources_node = get_or_abort(
        puppetdb.metric,
        "{0}{1}".format(prefix, ':type=default,name=avg-resources-per-node'))
    metrics = {
        'num_nodes': num_nodes['Value'],
        'num_resources': num_resources['Value'],
        'avg_resources_node': "{0:10.0f}".format(avg_resources_node['Value']),
        }

    nodes = puppetdb.nodes(
        unreported=app.config['UNRESPONSIVE_HOURS'],
        with_status=True)

    nodes_overview = []
    stats = {
        'changed': 0,
        'unchanged': 0,
        'failed': 0,
        'unreported': 0,
        'noop': 0
        }

    for node in nodes:
        if node.status == 'unreported':
            stats['unreported'] += 1
        elif node.status == 'changed':
            stats['changed'] += 1
        elif node.status == 'failed':
            stats['failed'] += 1
        elif node.status == 'noop':
            stats['noop'] += 1
        else:
            stats['unchanged'] += 1

        if node.status != 'unchanged':
            nodes_overview.append(node)

    return render_template(
        'index.html',
        metrics=metrics,
        nodes=nodes_overview,
        stats=stats
        )
Example #6
0
    def test_http_empty(self, mock_log, flask_abort):
        err = "Empty Response"

        def connection_error():
            raise EmptyResponseError(err)

        with self.assertRaises(NoContent) as error:
            utils.get_or_abort(connection_error)
            mock_log.error.assert_called_with(err)
            flask_abort.assert_called_with('204')
Example #7
0
def index(env):
    """This view generates the index page and displays a set of metrics and
    latest reports on nodes fetched from PuppetDB.

    :param env: Search for nodes in this (Catalog and Fact) environment
    :type env: :obj:`string`
    """
    envs = environments()
    check_env(env, envs)

    stats = status_count(env)
    if stats['total'] < 1000:
        stats['num_resources'] = get_count(
            'resources', EqualsOperator("environment", env))

        # average resource / node
        try:
            stats['avg_resources_node'] = "{0:10.0f}".format(
                (stats['num_resources'] / stats['total']))
        except ZeroDivisionError:
            stats['avg_resources_node'] = 0

    else:
        prefix = 'puppetlabs.puppetdb.population'
        query_type = ''

        # Puppet DB version changed the query format from 3.2.0
        # to 4.0 when querying mbeans
        if get_db_version(puppetdb) < (4, 0, 0):
            query_type = 'type=default,'

        stats['num_resources'] = get_or_abort(
            puppetdb.metric, "{0}{1}".format(
                prefix, ':%sname=num-resources' % query_type))['Value']
        stats['avg_resources_node'] = int(get_or_abort(
            puppetdb.metric, "{0}{1}".format(
                prefix,
                ':%sname=avg-resources-per-node' % query_type))['Value'])

    paging_args = {'limit': app.config['NORMAL_TABLE_COUNT'], 'offset': 0}
    order_arg = '[{"field": "catalog_timestamp", "order": "desc"}]'
    nodes = get_or_abort(puppetdb.nodes,
                         query=get_node_env_query(env),
                         unreported=app.config['UNRESPONSIVE_HOURS'],
                         with_status=True,
                         order_by=order_arg,
                         **paging_args)

    return render_template(
        'index.html',
        nodes=nodes,
        stats=stats,
        envs=envs,
        current_env=env
    )
Example #8
0
def catalog_compare(compare, against):
    """Compares the catalog of one node, parameter compare, with that of
       with that of another node, parameter against.
    """
    if app.config["ENABLE_CATALOG"]:
        compare_cat = get_or_abort(puppetdb.catalog, node=compare)
        against_cat = get_or_abort(puppetdb.catalog, node=against)

        return render_template("catalog_compare.html", compare=compare_cat, against=against_cat)
    else:
        log.warn("Access to catalog interface disabled by administrator")
        abort(403)
Example #9
0
def test_http_error(mock_log):
    err = "NotFound"

    def raise_http_error():
        x = Response()
        x.status_code = 404
        x.reason = err
        raise HTTPError(err, response=x)

    with pytest.raises(NotFound):
        utils.get_or_abort(raise_http_error)
        mock_log.error.assert_called_once_with(err)
Example #10
0
def test_http_connection_error(mock_log):
    err = "ConnectionError"

    def connection_error():
        x = Response()
        x.status_code = 500
        x.reason = err
        raise ConnectionError(err, response=x)

    with pytest.raises(InternalServerError):
        utils.get_or_abort(connection_error)
        mock_log.error.assert_called_with(err)
Example #11
0
def report_latest(node_name):
    """Redirect to the latest report of a given node. This is a workaround
    as long as PuppetDB can't filter reports for latest-report? field. This
    feature has been requested: http://projects.puppetlabs.com/issues/21554
    """
    node = get_or_abort(puppetdb.node, node_name)
    reports = get_or_abort(puppetdb._query, "reports", query='["=","certname","{0}"]'.format(node_name), limit=1)
    if len(reports) > 0:
        report = reports[0]["hash"]
        return redirect(url_for("report", node=node_name, report_id=report))
    else:
        abort(404)
Example #12
0
def report_latest(node_name):
    """Redirect to the latest report of a given node. This is a workaround
    as long as PuppetDB can't filter reports for latest-report? field. This
    feature has been requested: https://tickets.puppetlabs.com/browse/PDB-203
    """
    node = get_or_abort(puppetdb.node, node_name)
    reports = get_or_abort(puppetdb._query, 'reports',
                           query='["=","certname","{0}"]'.format(node_name),
                           limit=1)
    if len(reports) > 0:
        report = reports[0]['hash']
        return redirect(url_for('report', node=node_name, report_id=report))
    else:
        abort(404)
Example #13
0
def report_latest(env, node_name):
    """Redirect to the latest report of a given node.

    :param env: Search for reports in this environment
    :type env: :obj:`string`
    :param node_name: Find the reports whose certname match this value
    :type node_name: :obj:`string`
    """
    envs = environments()
    check_env(env, envs)

    if env == '*':
        node_query = '["=", "certname", "{0}"]'.format(node_name)
    else:
        node_query = '["and",' \
            '["=", "report_environment", "{0}"],' \
            '["=", "certname", "{1}"]]'.format(env, node_name)

    try:
        node = next(get_or_abort(puppetdb.nodes,
            query=node_query,
            with_status=True))
    except StopIteration:
        abort(404)

    if node.latest_report_hash is not None:
        hash_ = node.latest_report_hash
    else:
        if env == '*':
            query='["and",' \
                '["=", "certname", "{0}"],' \
                '["=", "latest_report?", true]]'.format(node.name)
        else:
            query='["and",' \
                '["=", "environment", "{0}"],' \
                '["=", "certname", "{1}"],' \
                '["=", "latest_report?", true]]'.format(
                    env,
                    node.name)

        reports = get_or_abort(puppetdb.reports, query=query)
        try:
            report = next(reports)
            hash_ = report.hash_
        except StopIteration:
            abort(404)

    return redirect(
        url_for('report', env=env, node_name=node_name, report_id=hash_))
Example #14
0
def catalogs():
    if app.config["ENABLE_CATALOG"]:
        nodenames = []
        catalog_list = []
        nodes = get_or_abort(
            puppetdb.nodes,
            query='["null?", "catalog_timestamp", false]',
            with_status=False,
            order_by='[{"field": "certname", "order": "asc"}]',
        )
        nodes, temp = tee(nodes)

        for node in temp:
            nodenames.append(node.name)

        for node in nodes:
            table_row = {"name": node.name, "catalog_timestamp": node.catalog_timestamp}

            if len(nodenames) > 1:
                form = CatalogForm()

                form.compare.data = node.name
                form.against.choices = [(x, x) for x in nodenames if x != node.name]
                table_row["form"] = form
            else:
                table_row["form"] = None

            catalog_list.append(table_row)

        return render_template("catalogs.html", nodes=catalog_list)
    else:
        log.warn("Access to catalogs endpoint disabled by administrator")
        abort(403)
Example #15
0
    def test_get(self, mock_log):
        x = "hello world"

        def test_get_or_abort():
            return x

        self.assertEqual(x, utils.get_or_abort(test_get_or_abort))
Example #16
0
def node(env, node_name):
    """Display a dashboard for a node showing as much data as we have on that
    node. This includes facts and reports but not Resources as that is too
    heavy to do within a single request.

    :param env: Ensure that the node, facts and reports are in this environment
    :type env: :obj:`string`
    """
    envs = environments()
    check_env(env, envs)
    query = AndOperator()

    if env != '*':
        query.add(EqualsOperator("environment", env))

    query.add(EqualsOperator("certname", node_name))

    node = get_or_abort(puppetdb.node, node_name)

    return render_template(
        'node.html',
        node=node,
        envs=envs,
        current_env=env,
        columns=REPORTS_COLUMNS[:2])
Example #17
0
def facts(env):
    """Displays an alphabetical list of all facts currently known to
    PuppetDB.

    :param env: Serves no purpose for this function, only for consistency's
        sake
    :type env: :obj:`string`
    """
    envs = environments()
    check_env(env, envs)

    facts_dict = collections.defaultdict(list)
    facts = get_or_abort(puppetdb.fact_names)
    for fact in facts:
        letter = fact[0].upper()
        letter_list = facts_dict[letter]
        letter_list.append(fact)
        facts_dict[letter] = letter_list

    sorted_facts_dict = sorted(facts_dict.items())
    return render_template(
        "facts.html",
        facts_dict=sorted_facts_dict,
        facts_len=(sum(map(len, facts_dict.values())) + len(facts_dict) * 5),
        envs=envs,
        current_env=env,
    )
Example #18
0
def test_get():
    x = "hello world"

    def test_get_or_abort():
        return x

    assert x == utils.get_or_abort(test_get_or_abort)
Example #19
0
def query(env):
    """Allows to execute raw, user created querries against PuppetDB. This is
    currently highly experimental and explodes in interesting ways since none
    of the possible exceptions are being handled just yet. This will return
    the JSON of the response or a message telling you what whent wrong /
    why nothing was returned.

    :param env: Serves no purpose for the query data but is required for the
        select field in the environment block
    :type env: :obj:`string`
    """
    if app.config["ENABLE_QUERY"]:
        envs = environments()
        check_env(env, envs)

        form = QueryForm(meta={"csrf_secret": app.config["SECRET_KEY"], "csrf_context": session})
        if form.validate_on_submit():
            if form.endpoints.data == "pql":
                query = form.query.data
            elif form.query.data[0] == "[":
                query = form.query.data
            else:
                query = "[{0}]".format(form.query.data)
            result = get_or_abort(puppetdb._query, form.endpoints.data, query=query)
            return render_template("query.html", form=form, result=result, envs=envs, current_env=env)
        return render_template("query.html", form=form, envs=envs, current_env=env)
    else:
        log.warn("Access to query interface disabled by administrator..")
        abort(403)
Example #20
0
def fact_value(env, fact, value):
    """On asking for fact/value get all nodes with that fact.

    :param env: Searches for facts in this environment
    :type env: :obj:`string`
    :param fact: Find all facts with this name
    :type fact: :obj:`string`
    :param value: Filter facts whose value is equal to this
    :type value: :obj:`string`
    """
    check_env(env)

    if env == '*':
        query = None
    else:
        query = '["=", "environment", "{0}"]'.format(env)
    facts = get_or_abort(puppetdb.facts,
        name=fact,
        value=value,
        query=query)
    localfacts = [f for f in yield_or_stop(facts)]
    return render_template(
        'fact.html',
        name=fact,
        value=value,
        facts=localfacts,
        envs=envs,
        current_env=env)
Example #21
0
def environments():
    envs = get_or_abort(puppetdb.environments)
    x = []

    for env in envs:
        x.append(env["name"])

    return x
Example #22
0
def node(node_name):
    """Display a dashboard for a node showing as much data as we have on that
    node. This includes facts and reports but not Resources as that is too
    heavy to do within a single request.
    """
    node = get_or_abort(puppetdb.node, node_name)
    facts = node.facts()
    reports = ten_reports(node.reports())
    return render_template("node.html", node=node, facts=yield_or_stop(facts), reports=yield_or_stop(reports))
Example #23
0
def get_count(endpoint, query):
    c_query = ExtractOperator()
    c_query.add_field(FunctionOperator('count'))
    if query:
        c_query.add_query(query)
    res = get_or_abort(
        puppetdb._query, endpoint,
        query=c_query)
    return res[0]['count']
Example #24
0
def environments():
    puppetdb = get_puppetdb()
    envs = get_or_abort(puppetdb.environments)
    x = []

    for env in envs:
        x.append(env['name'])

    return x
Example #25
0
def fact_value(fact, value):
    """On asking for fact/value get all nodes with that fact."""
    facts = get_or_abort(puppetdb.facts, fact, value)
    localfacts = [f for f in yield_or_stop(facts)]
    return render_template(
        'fact.html',
        name=fact,
        value=value,
        facts=localfacts)
Example #26
0
def report_latest(node_name):
    """Redirect to the latest report of a given node. This is a workaround
    as long as PuppetDB can't filter reports for latest-report? field. This
    feature has been requested: http://projects.puppetlabs.com/issues/21554
    """
    # TODO: use limit parameter in _query to get just one report
    node = get_or_abort(puppetdb.node, node_name)
    reports = ten_reports(node.reports())
    report = list(yield_or_stop(reports))[0]
    return redirect(url_for('report', node=node_name, report_id=report))
Example #27
0
def catalogs(env):
    """Lists all nodes with a compiled catalog.

    :param env: Find the nodes with this catalog_environment value
    :type env: :obj:`string`
    """
    envs = environments()
    check_env(env, envs)

    if app.config['ENABLE_CATALOG']:
        nodenames = []
        catalog_list = []
        query = AndOperator()

        if env != '*':
            query.add(EqualsOperator("catalog_environment", env))

        query.add(NullOperator("catalog_timestamp", False))

        order_by_str = '[{"field": "certname", "order": "asc"}]'
        nodes = get_or_abort(puppetdb.nodes,
                             query=query,
                             with_status=False,
                             order_by=order_by_str)
        nodes, temp = tee(nodes)

        for node in temp:
            nodenames.append(node.name)

        for node in nodes:
            table_row = {
                'name': node.name,
                'catalog_timestamp': node.catalog_timestamp
            }

            if len(nodenames) > 1:
                form = CatalogForm()

                form.compare.data = node.name
                form.against.choices = [(x, x) for x in nodenames
                                        if x != node.name]
                table_row['form'] = form
            else:
                table_row['form'] = None

            catalog_list.append(table_row)

        return render_template(
            'catalogs.html',
            nodes=catalog_list,
            envs=envs,
            current_env=env)
    else:
        log.warn('Access to catalog interface disabled by administrator')
        abort(403)
Example #28
0
def query():
    """Allows to execute raw, user created querries against PuppetDB. This is
    currently highly experimental and explodes in interesting ways since none
    of the possible exceptions are being handled just yet. This will return
    the JSON of the response or a message telling you what whent wrong /
    why nothing was returned."""
    form = QueryForm()
    if form.validate_on_submit():
        result = get_or_abort(puppetdb._query, form.endpoints.data, query="[{0}]".format(form.query.data))
        return render_template("query.html", form=form, result=result)
    return render_template("query.html", form=form)
Example #29
0
def catalogs_ajax(env, compare):
    """Server data to catalogs as JSON to Jquery datatables
    """
    draw = int(request.args.get('draw', 0))
    start = int(request.args.get('start', 0))
    length = int(request.args.get('length', app.config['NORMAL_TABLE_COUNT']))
    paging_args = {'limit': length, 'offset': start}
    search_arg = request.args.get('search[value]')
    order_column = int(request.args.get('order[0][column]', 0))
    order_filter = CATALOGS_COLUMNS[order_column].get(
        'filter', CATALOGS_COLUMNS[order_column]['attr'])
    order_dir = request.args.get('order[0][dir]', 'asc')
    order_args = '[{"field": "%s", "order": "%s"}]' % (order_filter, order_dir)

    envs = environments()
    check_env(env, envs)

    query = AndOperator()
    if env != '*':
        query.add(EqualsOperator("catalog_environment", env))
    if search_arg:
        query.add(RegexOperator("certname", r"%s" % search_arg))
    query.add(NullOperator("catalog_timestamp", False))

    nodes = get_or_abort(puppetdb.nodes,
                         query=query,
                         include_total=True,
                         order_by=order_args,
                         **paging_args)

    catalog_list = []
    total = None
    for node in nodes:
        if total is None:
            total = puppetdb.total

        catalog_list.append({
            'certname': node.name,
            'catalog_timestamp': node.catalog_timestamp,
            'form': compare,
        })

    if total is None:
        total = 0

    return render_template(
        'catalogs.json.tpl',
        total=total,
        total_filtered=total,
        draw=draw,
        columns=CATALOGS_COLUMNS,
        catalogs=catalog_list,
        envs=envs,
        current_env=env)
Example #30
0
def catalog_compare(env, compare, against):
    """Compares the catalog of one node, parameter compare, with that of
       with that of another node, parameter against.

    :param env: Ensure that the 2 catalogs are in the same environment
    :type env: :obj:`string`
    """
    envs = environments()
    check_env(env, envs)

    if app.config["ENABLE_CATALOG"]:
        compare_cat = get_or_abort(puppetdb.catalog, node=compare)
        against_cat = get_or_abort(puppetdb.catalog, node=against)

        return render_template(
            "catalog_compare.html", compare=compare_cat, against=against_cat, envs=envs, current_env=env
        )
    else:
        log.warn("Access to catalog interface disabled by administrator")
        abort(403)
Example #31
0
def index(env):
    """This view generates the index page and displays a set of metrics and
    latest reports on nodes fetched from PuppetDB.

    :param env: Search for nodes in this (Catalog and Fact) environment
    :type env: :obj:`string`
    """
    envs = environments()
    metrics = {'num_nodes': 0, 'num_resources': 0, 'avg_resources_node': 0}
    check_env(env, envs)

    if env == '*':
        query = None

        prefix = 'puppetlabs.puppetdb.population'
        num_nodes = get_or_abort(puppetdb.metric,
                                 "{0}{1}".format(prefix, ':name=num-nodes'))
        num_resources = get_or_abort(
            puppetdb.metric, "{0}{1}".format(prefix, ':name=num-resources'))
        avg_resources_node = get_or_abort(
            puppetdb.metric, "{0}{1}".format(prefix,
                                             ':name=avg-resources-per-node'))
        metrics['num_nodes'] = num_nodes['Value']
        metrics['num_resources'] = num_resources['Value']
        metrics['avg_resources_node'] = "{0:10.0f}".format(
            avg_resources_node['Value'])
    else:
        query = '["and", {0}]'.format(", ".join(
            '["=", "{0}", "{1}"]'.format(field, env)
            for field in ['catalog_environment', 'facts_environment']))
        num_nodes = get_or_abort(
            puppetdb._query,
            'nodes',
            query='["extract", [["function", "count"]],["and", {0}]]'.format(
                ",".join('["=", "{0}", "{1}"]'.format(field, env) for field in
                         ['catalog_environment', 'facts_environment'])))
        num_resources = get_or_abort(
            puppetdb._query,
            'resources',
            query='["extract", [["function", "count"]],' \
                '["=", "environment", "{0}"]]'.format(
                    env))
        metrics['num_nodes'] = num_nodes[0]['count']
        metrics['num_resources'] = num_resources[0]['count']
        metrics['avg_resources_node'] = "{0:10.0f}".format(
            (num_resources[0]['count'] / num_nodes[0]['count']))

    nodes = get_or_abort(puppetdb.nodes,
                         query=query,
                         unreported=app.config['UNRESPONSIVE_HOURS'],
                         with_status=True)

    nodes_overview = []
    stats = {
        'changed': 0,
        'unchanged': 0,
        'failed': 0,
        'unreported': 0,
        'noop': 0
    }

    for node in nodes:
        if node.status == 'unreported':
            stats['unreported'] += 1
        elif node.status == 'changed':
            stats['changed'] += 1
        elif node.status == 'failed':
            stats['failed'] += 1
        elif node.status == 'noop':
            stats['noop'] += 1
        else:
            stats['unchanged'] += 1

        if node.status != 'unchanged':
            nodes_overview.append(node)

    return render_template('index.html',
                           metrics=metrics,
                           nodes=nodes_overview,
                           stats=stats,
                           envs=envs,
                           current_env=env)
Example #32
0
def index(env):
    """This view generates the index page and displays a set of metrics and
    latest reports on nodes fetched from PuppetDB.

    :param env: Search for nodes in this (Catalog and Fact) environment
    :type env: :obj:`string`
    """
    envs = environments()
    metrics = {'num_nodes': 0, 'num_resources': 0, 'avg_resources_node': 0}
    check_env(env, envs)

    if env == '*':
        query = app.config['OVERVIEW_FILTER']

        prefix = 'puppetlabs.puppetdb.population'
        db_version = get_db_version(puppetdb)
        query_type, metric_version = metric_params(db_version)

        num_nodes = get_or_abort(puppetdb.metric,
                                 "{0}{1}".format(
                                     prefix, ':%sname=num-nodes' % query_type),
                                 version=metric_version)
        num_resources = get_or_abort(puppetdb.metric,
                                     "{0}{1}".format(
                                         prefix,
                                         ':%sname=num-resources' % query_type),
                                     version=metric_version)
        avg_resources_node = get_or_abort(
            puppetdb.metric,
            "{0}{1}".format(prefix,
                            ':%sname=avg-resources-per-node' % query_type),
            version=metric_version)
        metrics['num_nodes'] = num_nodes['Value']
        metrics['num_resources'] = num_resources['Value']
        try:
            # Compute our own average because avg_resources_node['Value']
            # returns a string of the format "num_resources/num_nodes"
            # example: "1234/9" instead of doing the division itself.
            metrics['avg_resources_node'] = "{0:10.0f}".format(
                (num_resources['Value'] / num_nodes['Value']))
        except ZeroDivisionError:
            metrics['avg_resources_node'] = 0
    else:
        query = AndOperator()
        query.add(EqualsOperator('catalog_environment', env))

        num_nodes_query = ExtractOperator()
        num_nodes_query.add_field(FunctionOperator('count'))
        num_nodes_query.add_query(query)

        if app.config['OVERVIEW_FILTER'] is not None:
            query.add(app.config['OVERVIEW_FILTER'])

        num_resources_query = ExtractOperator()
        num_resources_query.add_field(FunctionOperator('count'))
        num_resources_query.add_query(EqualsOperator("environment", env))

        num_nodes = get_or_abort(puppetdb._query,
                                 'nodes',
                                 query=num_nodes_query)
        num_resources = get_or_abort(puppetdb._query,
                                     'resources',
                                     query=num_resources_query)
        metrics['num_nodes'] = num_nodes[0]['count']
        metrics['num_resources'] = num_resources[0]['count']
        try:
            metrics['avg_resources_node'] = "{0:10.0f}".format(
                (num_resources[0]['count'] / num_nodes[0]['count']))
        except ZeroDivisionError:
            metrics['avg_resources_node'] = 0

    nodes = get_or_abort(puppetdb.nodes,
                         query=query,
                         unreported=app.config['UNRESPONSIVE_HOURS'],
                         with_status=True,
                         with_event_numbers=app.config['WITH_EVENT_NUMBERS'])

    nodes_overview = []
    stats = {
        'changed': 0,
        'unchanged': 0,
        'failed': 0,
        'unreported': 0,
        'noop': 0
    }

    for node in nodes:
        if node.status == 'unreported':
            stats['unreported'] += 1
        elif node.status == 'changed':
            stats['changed'] += 1
        elif node.status == 'failed':
            stats['failed'] += 1
        elif node.status == 'noop':
            stats['noop'] += 1
        else:
            stats['unchanged'] += 1

        if node.status != 'unchanged':
            nodes_overview.append(node)

    return render_template('index.html',
                           metrics=metrics,
                           nodes=nodes_overview,
                           stats=stats,
                           envs=envs,
                           current_env=env)
Example #33
0
def metrics():
    metrics = get_or_abort(puppetdb._query, 'metrics', path='mbeans')
    for key, value in metrics.items():
        metrics[key] = value.split('/')[3]
    return render_template('metrics.html', metrics=sorted(metrics.items()))
Example #34
0
def index(env):
    """This view generates the index page and displays a set of metrics and
    latest reports on nodes fetched from PuppetDB.

    :param env: Search for nodes in this (Catalog and Fact) environment
    :type env: :obj:`string`
    """
    check_env(env)

    # TODO: Would be great if we could parallelize this somehow, doing these
    # requests in sequence is rather pointless.
    prefix = 'puppetlabs.puppetdb.query.population'
    num_nodes = get_or_abort(
        puppetdb.metric, "{0}{1}".format(prefix,
                                         ':type=default,name=num-nodes'))
    num_resources = get_or_abort(
        puppetdb.metric, "{0}{1}".format(prefix,
                                         ':type=default,name=num-resources'))
    avg_resources_node = get_or_abort(
        puppetdb.metric,
        "{0}{1}".format(prefix, ':type=default,name=avg-resources-per-node'))
    metrics = {
        'num_nodes': num_nodes['Value'],
        'num_resources': num_resources['Value'],
        'avg_resources_node': "{0:10.0f}".format(avg_resources_node['Value']),
    }

    if env == '*':
        query = None
    else:
        query = '["and", {0}]'.format(", ".join(
            '["=", "{0}", "{1}"]'.format(field, env)
            for field in ['catalog_environment', 'facts_environment']))

    nodes = get_or_abort(puppetdb.nodes,
                         query=query,
                         unreported=app.config['UNRESPONSIVE_HOURS'],
                         with_status=True)

    nodes_overview = []
    stats = {
        'changed': 0,
        'unchanged': 0,
        'failed': 0,
        'unreported': 0,
        'noop': 0
    }

    for node in nodes:
        if node.status == 'unreported':
            stats['unreported'] += 1
        elif node.status == 'changed':
            stats['changed'] += 1
        elif node.status == 'failed':
            stats['failed'] += 1
        elif node.status == 'noop':
            stats['noop'] += 1
        else:
            stats['unchanged'] += 1

        if node.status != 'unchanged':
            nodes_overview.append(node)

    return render_template('index.html',
                           metrics=metrics,
                           nodes=nodes_overview,
                           stats=stats,
                           envs=envs,
                           current_env=env)
Example #35
0
def fact_ajax(env, node, fact, value):
    """Fetches the specific facts matching (node/fact/value) from PuppetDB and
    return a JSON table

    :param env: Searches for facts in this environment
    :type env: :obj:`string`
    :param node: Find all facts for this node
    :type node: :obj:`string`
    :param fact: Find all facts with this name
    :type fact: :obj:`string`
    :param value: Filter facts whose value is equal to this
    :type value: :obj:`string`
    """
    draw = int(request.args.get('draw', 0))

    envs = environments()
    check_env(env, envs)

    render_graph = False
    if fact in graph_facts and not value and not node:
        render_graph = True

    query = AndOperator()
    if node:
        query.add(EqualsOperator("certname", node))

    if env != '*':
        query.add(EqualsOperator("environment", env))

    if len(query.operations) == 0:
        query = None

    # Generator needs to be converted (graph / total)
    try:
        value = int(value)
    except ValueError:
        if value is not None and query is not None:
            query.add(EqualsOperator('value', unquote_plus(value)))
    except TypeError:
        pass

    facts = [f for f in get_or_abort(puppetdb.facts, name=fact, query=query)]

    total = len(facts)

    counts = {}
    json = {
        'draw': draw,
        'recordsTotal': total,
        'recordsFiltered': total,
        'data': []
    }

    for fact_h in facts:
        line = []
        if not fact:
            line.append(fact_h.name)
        if not node:
            line.append('<a href="{0}">{1}</a>'.format(
                url_for('node', env=env, node_name=fact_h.node), fact_h.node))
        if not value:
            fact_value = fact_h.value
            if isinstance(fact_value, str):
                fact_value = quote_plus(fact_h.value)

            line.append('<a href="{0}">{1}</a>'.format(
                url_for('fact', env=env, fact=fact_h.name, value=fact_value),
                fact_h.value))

        json['data'].append(line)

        if render_graph:
            if fact_h.value not in counts:
                counts[fact_h.value] = 0
            counts[fact_h.value] += 1

    if render_graph:
        json['chart'] = [{
            "label": "{0}".format(k).replace('\n', ' '),
            "value": counts[k]
        } for k in sorted(counts, key=lambda k: counts[k], reverse=True)]

    return jsonify(json)
Example #36
0
def reports_ajax(env, node_name):
    """Query and Return JSON data to reports Jquery datatable

    :param env: Search for all reports in this environment
    :type env: :obj:`string`
    """
    draw = int(request.args.get('draw', 0))
    start = int(request.args.get('start', 0))
    length = int(request.args.get('length', app.config['NORMAL_TABLE_COUNT']))
    paging_args = {'limit': length, 'offset': start}
    search_arg = request.args.get('search[value]')
    order_column = int(request.args.get('order[0][column]', 0))
    order_filter = REPORTS_COLUMNS[order_column].get(
        'filter', REPORTS_COLUMNS[order_column]['attr'])
    order_dir = request.args.get('order[0][dir]', 'desc')
    order_args = '[{"field": "%s", "order": "%s"}]' % (order_filter, order_dir)
    status_args = request.args.get('columns[1][search][value]', '').split('|')
    max_col = len(REPORTS_COLUMNS)
    for i in range(len(REPORTS_COLUMNS)):
        if request.args.get("columns[%s][data]" % i, None):
            max_col = i + 1

    envs = environments()
    check_env(env, envs)
    reports_query = AndOperator()

    if env != '*':
        reports_query.add(EqualsOperator("environment", env))

    if node_name:
        reports_query.add(EqualsOperator("certname", node_name))

    if search_arg:
        search_query = OrOperator()
        search_query.add(RegexOperator("certname", r"%s" % search_arg))
        search_query.add(RegexOperator("puppet_version", r"%s" % search_arg))
        search_query.add(
            RegexOperator("configuration_version", r"%s" % search_arg))
        reports_query.add(search_query)

    status_query = OrOperator()
    for status_arg in status_args:
        if status_arg in ['failed', 'changed', 'unchanged']:
            arg_query = AndOperator()
            arg_query.add(EqualsOperator('status', status_arg))
            arg_query.add(EqualsOperator('noop', False))
            status_query.add(arg_query)
            if status_arg == 'unchanged':
                arg_query = AndOperator()
                arg_query.add(EqualsOperator('noop', True))
                arg_query.add(EqualsOperator('noop_pending', False))
                status_query.add(arg_query)
        elif status_arg == 'noop':
            arg_query = AndOperator()
            arg_query.add(EqualsOperator('noop', True))
            arg_query.add(EqualsOperator('noop_pending', True))
            status_query.add(arg_query)

    if len(status_query.operations) == 0:
        if len(reports_query.operations) == 0:
            reports_query = None
    else:
        reports_query.add(status_query)

    if status_args[0] != 'none':
        reports = get_or_abort(puppetdb.reports,
                               query=reports_query,
                               order_by=order_args,
                               include_total=True,
                               **paging_args)
        reports, reports_events = tee(reports)
        total = None
    else:
        reports = []
        reports_events = []
        total = 0

    # Convert metrics to relational dict
    metrics = {}
    for report in reports_events:
        if total is None:
            total = puppetdb.total

        metrics[report.hash_] = {}
        for m in report.metrics:
            if m['category'] not in metrics[report.hash_]:
                metrics[report.hash_][m['category']] = {}
            metrics[report.hash_][m['category']][m['name']] = m['value']

    if total is None:
        total = 0

    return render_template('reports.json.tpl',
                           draw=draw,
                           total=total,
                           total_filtered=total,
                           reports=reports,
                           metrics=metrics,
                           envs=envs,
                           current_env=env,
                           columns=REPORTS_COLUMNS[:max_col])
Example #37
0
def facts(env):
    """Displays an alphabetical list of all facts currently known to
    PuppetDB.

    :param env: Serves no purpose for this function, only for consistency's
        sake
    :type env: :obj:`string`
    """
    envs = environments()
    check_env(env, envs)
    facts = []
    order_by = '[{"field": "name", "order": "asc"}]'
    facts = get_or_abort(puppetdb.fact_names)

    # we consider a column label to count for ~5 lines
    column_label_height = 5

    # 1 label per different letter and up to 3 more labels for letters spanning
    # multiple columns.
    column_label_count = 3 + len(set(map(lambda fact: fact[0].upper(), facts)))

    break_size = (len(facts) + column_label_count * column_label_height) / 4.0
    next_break = break_size

    facts_columns = []
    facts_current_column = []
    facts_current_letter = []
    letter = None
    count = 0

    for fact in facts:
        count += 1

        if count > next_break:
            next_break += break_size
            if facts_current_letter:
                facts_current_column.append(facts_current_letter)
            if facts_current_column:
                facts_columns.append(facts_current_column)
            facts_current_column = []
            facts_current_letter = []
            letter = None

        if letter != fact[0].upper():
            if facts_current_letter:
                facts_current_column.append(facts_current_letter)
                facts_current_letter = []
            letter = fact[0].upper()
            count += column_label_height

        facts_current_letter.append(fact)

    if facts_current_letter:
        facts_current_column.append(facts_current_letter)
    if facts_current_column:
        facts_columns.append(facts_current_column)

    return render_template('facts.html',
                           facts_columns=facts_columns,
                           envs=envs,
                           current_env=env)
Example #38
0
def radiator(env):
    """This view generates a simplified monitoring page
    akin to the radiator view in puppet dashboard
    """
    envs = environments()
    check_env(env, envs)

    if env == '*':
        db_version = get_db_version(puppetdb)
        query_type, metric_version = metric_params(db_version)

        query = None
        metrics = get_or_abort(
            puppetdb.metric,
            'puppetlabs.puppetdb.population:%sname=num-nodes' % query_type,
            version=metric_version)
        num_nodes = metrics['Value']
    else:
        query = AndOperator()
        metric_query = ExtractOperator()

        query.add(EqualsOperator("catalog_environment", env))
        metric_query.add_field(FunctionOperator('count'))
        metric_query.add_query(query)

        metrics = get_or_abort(puppetdb._query, 'nodes', query=metric_query)
        num_nodes = metrics[0]['count']

    nodes = puppetdb.nodes(query=query,
                           unreported=app.config['UNRESPONSIVE_HOURS'],
                           with_status=True)

    stats = {
        'changed_percent': 0,
        'changed': 0,
        'failed_percent': 0,
        'failed': 0,
        'noop_percent': 0,
        'noop': 0,
        'skipped_percent': 0,
        'skipped': 0,
        'unchanged_percent': 0,
        'unchanged': 0,
        'unreported_percent': 0,
        'unreported': 0,
    }

    for node in nodes:
        if node.status == 'unreported':
            stats['unreported'] += 1
        elif node.status == 'changed':
            stats['changed'] += 1
        elif node.status == 'failed':
            stats['failed'] += 1
        elif node.status == 'noop':
            stats['noop'] += 1
        elif node.status == 'skipped':
            stats['skipped'] += 1
        else:
            stats['unchanged'] += 1

    try:
        stats['changed_percent'] = int(100 *
                                       (stats['changed'] / float(num_nodes)))
        stats['failed_percent'] = int(100 * stats['failed'] / float(num_nodes))
        stats['noop_percent'] = int(100 * stats['noop'] / float(num_nodes))
        stats['skipped_percent'] = int(100 *
                                       (stats['skipped'] / float(num_nodes)))
        stats['unchanged_percent'] = int(
            100 * (stats['unchanged'] / float(num_nodes)))
        stats['unreported_percent'] = int(
            100 * (stats['unreported'] / float(num_nodes)))
    except ZeroDivisionError:
        stats['changed_percent'] = 0
        stats['failed_percent'] = 0
        stats['noop_percent'] = 0
        stats['skipped_percent'] = 0
        stats['unchanged_percent'] = 0
        stats['unreported_percent'] = 0

    if ('Accept' in request.headers
            and request.headers["Accept"] == 'application/json'):
        return jsonify(**stats)

    return render_template('radiator.html', stats=stats, total=num_nodes)
Example #39
0
def index(env):
    """This view generates the index page and displays a set of metrics and
    latest reports on nodes fetched from PuppetDB.

    :param env: Search for nodes in this (Catalog and Fact) environment
    :type env: :obj:`string`
    """
    envs = environments()
    metrics = {'num_nodes': 0, 'num_resources': 0, 'avg_resources_node': 0}
    check_env(env, envs)

    if env == '*':
        query = app.config['OVERVIEW_FILTER']

        prefix = 'puppetlabs.puppetdb.population'
        query_type = ''

        # Puppet DB version changed the query format from 3.2.0
        # to 4.0 when querying mbeans
        if get_db_version(puppetdb) < (4, 0, 0):
            query_type = 'type=default,'

        num_nodes = get_or_abort(
            puppetdb.metric, "{0}{1}".format(prefix,
                                             ':%sname=num-nodes' % query_type))
        num_resources = get_or_abort(
            puppetdb.metric,
            "{0}{1}".format(prefix, ':%sname=num-resources' % query_type))
        avg_resources_node = get_or_abort(
            puppetdb.metric,
            "{0}{1}".format(prefix,
                            ':%sname=avg-resources-per-node' % query_type))
        metrics['num_nodes'] = num_nodes['Value']
        metrics['num_resources'] = num_resources['Value']
        metrics['avg_resources_node'] = "{0:10.0f}".format(
            avg_resources_node['Value'])
    else:
        query = AndOperator()
        query.add(EqualsOperator('catalog_environment', env))
        query.add(EqualsOperator('facts_environment', env))

        num_nodes_query = ExtractOperator()
        num_nodes_query.add_field(FunctionOperator('count'))
        num_nodes_query.add_query(query)

        if app.config['OVERVIEW_FILTER'] is not None:
            query.add(app.config['OVERVIEW_FILTER'])

        num_resources_query = ExtractOperator()
        num_resources_query.add_field(FunctionOperator('count'))
        num_resources_query.add_query(EqualsOperator("environment", env))

        num_nodes = get_or_abort(puppetdb._query,
                                 'nodes',
                                 query=num_nodes_query)
        num_resources = get_or_abort(puppetdb._query,
                                     'resources',
                                     query=num_resources_query)
        metrics['num_nodes'] = num_nodes[0]['count']
        metrics['num_resources'] = num_resources[0]['count']
        try:
            metrics['avg_resources_node'] = "{0:10.0f}".format(
                (num_resources[0]['count'] / num_nodes[0]['count']))
        except ZeroDivisionError:
            metrics['avg_resources_node'] = 0

    nodes = get_or_abort(puppetdb.nodes,
                         query=query,
                         unreported=app.config['UNRESPONSIVE_HOURS'],
                         with_status=True)

    nodes_overview = []
    stats = {
        'changed': 0,
        'unchanged': 0,
        'failed': 0,
        'unreported': 0,
        'noop': 0
    }

    for node in nodes:
        if node.status == 'unreported':
            stats['unreported'] += 1
        elif node.status == 'changed':
            stats['changed'] += 1
        elif node.status == 'failed':
            stats['failed'] += 1
        elif node.status == 'noop':
            stats['noop'] += 1
        else:
            stats['unchanged'] += 1

        if node.status != 'unchanged':
            nodes_overview.append(node)

    return render_template('index.html',
                           metrics=metrics,
                           nodes=nodes_overview,
                           stats=stats,
                           envs=envs,
                           current_env=env)