def _get_puppetdb_connexion(configuration: Configuration) -> BaseAPI:
return connect(host=configuration.puppetdb_host,
port=configuration.puppetdb_port,
ssl_verify=configuration.puppetdb_ssl_verify,
ssl_key=configuration.puppetdb_ssl_key,
ssl_cert=configuration.puppetdb_ssl_cert,
protocol=configuration.puppetdb_proto)
def puppet(config):
"""Query puppetdb for details about hosts.
:param config: configuration dictionary
:type config: dictionary
"""
puppetdb_config = {}
for key, value in config.items():
if key.startswith("puppetdb"):
puppetdb_config[key.replace("puppetdb-", "")] = value
logging.debug("PuppetDB configuration: %s", puppetdb_config)
puppetdb = pypuppetdb.connect(**puppetdb_config)
nodes = {}
keywords = {
'sshfp_rsa': 'sshfp_rsa',
'sshfp_dsa': 'sshfp_dsa',
'sshfp_ecdsa': 'sshfp_ecdsa',
'sshfp_ed25519': 'sshfp_ed25519',
}
for key, value in keywords.items():
facts = puppetdb.facts(key)
for fact in facts:
node_name = fact.node
if config['pdns-zone'] not in node_name:
continue
ip_address = fact.value
if node_name in nodes:
nodes[node_name][value] = ip_address
else:
nodes[node_name] = {value: ip_address}
return nodes
def main(hostname, to=None, num_days=7, cache_dir=None, dry_run=False):
"""
main entry point
:param hostname: PuppetDB hostname
:type hostname: string
:param to: list of addresses to send mail to
:type to: list
:param num_days: the number of days to report on, default 7
:type num_days: int
:param cache_dir: absolute path to where to cache data from PuppetDB
:type cache_dir: string
:param dry_run: whether to actually send, or just print what would be sent
:type dry_run: boolean
"""
pdb = connect(host=hostname)
# essentially figure out all these for yesterday, build the tables, serialize the result as JSON somewhere. then just keep the last ~7 days json files
date_data = {}
dates = [] # ordered
date_list = get_date_list(num_days)
localtz = tzlocal.get_localzone()
start_date = date_list[0]
end_date = date_list[-1] - datetime.timedelta(hours=23, minutes=59, seconds=59)
for query_date in date_list:
end = query_date
start = query_date - datetime.timedelta(days=1) + datetime.timedelta(seconds=1)
date_s = (query_date - datetime.timedelta(hours=1)).astimezone(localtz).strftime('%a %m/%d')
date_data[date_s] = get_data_for_timespan(hostname, pdb, start, end, cache_dir=cache_dir)
dates.append(date_s)
html = format_html(hostname, dates, date_data, start_date, end_date)
subject = 'daily puppet(db) run summary for {host}'.format(host=hostname)
send_mail(to, subject, html, dry_run=dry_run)
return True
def get_puppetdb():
global PUPPETDB
if PUPPETDB is None:
app = get_app()
puppetdb = connect(
host=app.config['PUPPETDB_HOST'],
port=app.config['PUPPETDB_PORT'],
ssl_verify=app.config['PUPPETDB_SSL_VERIFY'],
ssl_key=app.config['PUPPETDB_KEY'],
ssl_cert=app.config['PUPPETDB_CERT'],
timeout=app.config['PUPPETDB_TIMEOUT'],
protocol=app.config['PUPPETDB_PROTO'],
)
requests_version = pkg_resources.get_distribution("requests").version
user_agent_header = {
"user-agent": f"puppetboard/{own_version} (r/{requests_version})",
}
puppetdb.session.headers = {
**puppetdb.session.headers,
**user_agent_header
}
PUPPETDB = puppetdb
return PUPPETDB
def generate(host, *args):
db = pypuppetdb.connect()
ssh = db.facts('ssh')
servers = [{
'fqdn': x.node,
'ssh-key': '%s %s' % (x.value['ecdsa']['type'], x.value['ecdsa']['key'])
} for x in ssh]
return {'borgserver': {'servers': sorted(servers)}}
def connect_puppetdb(puppetdb_host):
db = connect(host=puppetdb_host,
port=443,
protocol='https',
ssl_key=None,
ssl_cert=None,
ssl_verify='/var/lib/puppet/ssl/certs/ca.pem')
return db
def get_host_list(self):
db = connect(api_version=3, host=self.puppetdb_server, port=self.puppetdb_server_port)
nodes = db.nodes()
inv = { 'all': []}
for node in nodes:
inv['all'].append(node.name)
return inv
def make_it_so():
args = argparser()
db = pypuppetdb.connect(host=PUPPETDB_HOST, port=PUPPETDB_PORT)
if args.section == 'nodes':
nodes_query(db, args.query)
if args.section == 'facts':
fact_query(db, args.query)
def puppetdb_connect():
"""Connects to PuppetDB using the parameters specified in the
application configuration."""
# Connect to PuppetDB
return pypuppetdb.connect(app.config['PUPPETDB_HOST'],
port=app.config['PUPPETDB_PORT'],
ssl_cert=app.config['PUPPETDB_SSL_CERT'],
ssl_key=app.config['PUPPETDB_SSL_KEY'],
ssl_verify=app.config['PUPPETDB_SSL_VERIFY'])
def __init__(self, config, debug):
self.debug = debug
self.config = config
self.puppetdb = connect(
host=config.get('host', 'localhost'),
port=config.get('port', '8080'),
ssl_verify=config.get('ssl_verify', False),
ssl_key=config.get('ssl_key', None),
ssl_cert=config.get('ssl_cert', None),
)
def main():
"""main entry point"""
args = get_args()
logging.basicConfig(level=get_log_level(args.verbose))
failed_nodes = []
pdb = connect()
nodes = defaultdict(dict)
max_age = datetime.utcnow() - timedelta(hours=args.max_age)
extract = ExtractOperator()
extract.add_field(['certname', FunctionOperator('count'), 'status'])
extract.add_group_by(['certname', 'status'])
extract.add_query(GreaterOperator('receive_time', max_age.isoformat()))
# pypuppetdb does have a `reports` method which wraps `_query`. however it yields
# results of type pypuppetdb.Report which expects a number of parameters e.g. hash
# to be present in the result payload. however we don't extract theses values and
# therefore have to resort to the more powerful private method
reports = pdb._query('reports', query=extract) # pylint: disable=protected-access
for report in reports:
nodes[report['certname']][report['status']] = report['count']
if args.dev:
failed_nodes = [hostname for hostname, node in nodes.items()
if not node.get('unchanged', 0)]
else:
for fqdn, node in nodes.items():
# skip hosts with no unchanged reports:
if node.get('unchanged', 0):
continue
# skip staging servers:
# - hostname starting labstest*
# - hostname ending dev or dev\d{4}
# - hostname ending test or test\d{4}
if (fqdn.startswith('labtest')
or search(r'(:?dev|test)(:?\d{4})?$', fqdn.split('.')[0]) is not None):
logger.debug('%s: Skipping staging host', fqdn)
continue
failed_nodes.append(fqdn)
if len(failed_nodes) >= args.critical:
print('CRITICAL: the following ({}) node(s) change every puppet run: {}'.format(
len(failed_nodes), ', '.join(failed_nodes)))
return 2
if len(failed_nodes) >= args.warning:
print('WARNING: the following ({}) node(s) change every puppet run: {}'.format(
len(failed_nodes), ', '.join(failed_nodes)))
return 1
print('OK: all nodes running as expected')
return 0
def get_host_list_based_on_environments(self):
"""Getting the nodes out of puppetdb using api version 4 based on environments"""
db = connect(host=self.puppetdb_server, port=self.puppetdb_server_port)
inv = {}
for env in self.puppetdb_environments:
inv.update({env: []})
facts = db.facts('fqdn', environment=env)
for fact in facts:
inv[env].append(fact.value)
return inv
def get_host_list_based_on_environments(self):
db = connect(api_version=4, host=self.puppetdb_server, port=self.puppetdb_server_port)
json_data_toReturn = ''
inv = {}
for env in self.puppetdb_environments:
inv.update( { env: [] })
facts = db.facts('fqdn', environment=env)
for fact in facts:
inv[env].append(fact.value)
return inv
def read_callback():
puppetdb = connect(
api_version=3,
host=PUPPETDB_HOST,
port=PUPPETDB_PORT,
ssl_verify=PUPPETDB_SSL,
ssl_key=PUPPETDB_KEY,
ssl_cert=PUPPETDB_CERT,
timeout=PUPPETDB_TIMEOUT,
)
prefix = "com.puppetlabs.puppetdb.query.population"
num_nodes = puppetdb.metric("{0}{1}".format(prefix, ":type=default,name=num-nodes"))
num_resources = puppetdb.metric("{0}{1}".format(prefix, ":type=default,name=num-resources"))
avg_resources_node = puppetdb.metric("{0}{1}".format(prefix, ":type=default,name=avg-resources-per-node"))
# Ftech nodes
nodes = puppetdb.nodes(unreported=UNREPORTED_TIME, with_status=True)
# Init stats
stats = {"changed": 0, "unchanged": 0, "failed": 0, "unreported": 0, "noop": 0}
for node in nodes:
if node.status == "unreported":
stats["unreported"] += 1
elif node.status == "changed":
stats["changed"] += 1
elif node.status == "failed":
stats["failed"] += 1
elif node.status == "noop":
stats["noop"] += 1
else:
stats["unchanged"] += 1
log_verbose("population: %s\n" % num_nodes["Value"])
dispatch_value(num_nodes["Value"], "population", "gauge")
log_verbose("unreported: %s\n" % stats["unreported"])
dispatch_value(stats["unreported"], "unreported", "gauge")
log_verbose("changed: %s\n" % stats["changed"])
dispatch_value(stats["changed"], "changed", "gauge")
log_verbose("failed: %s\n" % stats["failed"])
dispatch_value(stats["failed"], "failed", "gauge")
log_verbose("noop: %s\n" % stats["noop"])
dispatch_value(stats["noop"], "noop", "gauge")
log_verbose("unchanged: %s\n" % stats["unchanged"])
dispatch_value(stats["unchanged"], "unchanged", "gauge")
def __init__(self, hostname, port, api_version, output_dir,
nodefacts=None, query=None, environment=None):
self.db = connect(host=hostname,
port=port,
api_version=api_version,
timeout=20)
self.db.resources = self.db.resources
self.output_dir = output_dir
self.environment = environment
if not nodefacts:
self.nodefacts = self.get_nodefacts()
else:
self.nodefacts = nodefacts
self.query = query
def get_puppetdb():
global PUPPETDB
if PUPPETDB is None:
app = get_app()
puppetdb = connect(host=app.config['PUPPETDB_HOST'],
port=app.config['PUPPETDB_PORT'],
ssl_verify=app.config['PUPPETDB_SSL_VERIFY'],
ssl_key=app.config['PUPPETDB_KEY'],
ssl_cert=app.config['PUPPETDB_CERT'],
timeout=app.config['PUPPETDB_TIMEOUT'],)
PUPPETDB = puppetdb
return PUPPETDB
def check_user(username):
puppet_db = pypuppetdb.connect(host=settings.PUPPETDB_HOST,
port=settings.PUPPETDB_PORT,
ssl_verify=settings.PUPPETDB_SSL_VERIFY,
ssl_key=settings.PUPPETDB_KEY,
ssl_cert=settings.PUPPETDB_CERT)
try:
node = puppet_db.node(settings.PUPPETDB_NODE)
return next(node.resources(type_='User', title=username))
except ConnectionError:
return False
except StopIteration:
return None
def get_puppetdb():
global PUPPETDB
if PUPPETDB is None:
app = get_app()
puppetdb = connect(
host=app.config['PUPPETDB_HOST'],
port=app.config['PUPPETDB_PORT'],
ssl_verify=app.config['PUPPETDB_SSL_VERIFY'],
ssl_key=app.config['PUPPETDB_KEY'],
ssl_cert=app.config['PUPPETDB_CERT'],
timeout=app.config['PUPPETDB_TIMEOUT'],
)
PUPPETDB = puppetdb
return PUPPETDB
def puppetdb_connect():
# Basic configuration
puppetdb_params = {
'host': settings.PUPPETDB_HOST,
'port': settings.PUPPETDB_PORT
}
# SSL specific configuration
if settings.PUPPETDB_SSL:
puppetdb_params.update({
'ssl_verify': settings.PUPPET_CA_CRT,
'ssl_cert': settings.PUPPET_CLIENT_CRT,
'ssl_key': settings.PUPPET_CLIENT_KEY
})
return pypuppetdb.connect(**puppetdb_params)
def __init__(self, hostname, port, api_version,
query=None, environment=None,
ssl_key=None, ssl_cert=None,
timeout=20):
from pypuppetdb import connect
self.db = connect(host=hostname,
port=port,
ssl_key=ssl_key,
ssl_cert=ssl_cert,
api_version=api_version,
timeout=timeout)
self.db.resources = self.db.resources
self.environment = environment
if query is None:
query = {}
self.query = query
def run(
self,
connection='',
server='',
port=8081,
transport='https',
ssldir=PUPPET_SSLDIR,
certname=PUPPET_CERTNAME,
ssl_verify=True,
# query options
query=None,
order_by=None,
limit=None,
offset=None,
include_total=None,
# invocation
endpoint='',
function='',
method='GET'):
if not ssl_verify:
ssl_verify = False
import requests
requests.packages.urllib3.disable_warnings()
# get proper SSL cert paths
ssl_key = '{}/private_keys/{}.pem'.format(ssldir, certname)
ssl_cert = '{}/certs/{}.pem'.format(ssldir, certname)
ssl_ca = '{}/certs/ca.pem'.format(ssldir)
os.environ['REQUESTS_CA_BUNDLE'] = ssl_ca
# create our client
client = pypuppetdb.connect(host=server,
port=port,
ssl_verify=ssl_verify,
ssl_key=ssl_key,
ssl_cert=ssl_cert,
protocol=transport)
if function:
raw_data = getattr(client, function)()
data = [d.__dict__ for d in raw_data]
else:
if not isinstance(query, six.string_types):
query = json.dumps(query)
data = client._query(endpoint, query=query)
return data
def main():
"""
Main function
"""
parser = argparse.ArgumentParser(prog="puppetdb_stencil")
parser.add_argument("resource_types", metavar="RESOURCE_TYPE", nargs="+")
parser.add_argument("--templates", "-t", metavar="TEMPLATE", nargs="*")
parser.add_argument("--debug", "-d", action="store_true")
parser.add_argument("--host", "-H", default="localhost")
parser.add_argument("--port", "-p", default="8080")
parser.add_argument("--localsite", "-l", default="true")
args = parser.parse_args()
logging.basicConfig(level=logging.DEBUG if args.debug else logging.WARN)
database = pypuppetdb.connect(host=args.host, port=args.port)
for resource_type in args.resource_types:
templates = ["{0}.jinja2".format(resource_type)]
if args.templates:
templates += args.templates
print(render_resources(database, resource_type, args.localsite, templates))
def main():
options = cmd()
if options.verify == 'yes':
ssl_verify = True
elif options.verify == 'no':
ssl_verify = False
else:
ssl_verify = options.verify
p = pypuppetdb.connect(host=options.server, port=options.port,
ssl_verify=ssl_verify, ssl_key=options.clientkey,
ssl_cert=options.clientcert,
timeout=options.timeout)
node = get_node(options.hostname, p)
if not node.report_timestamp:
print("UNKNOWN - This machine hasn't checked in with us yet")
sys.exit(3)
critical_seconds = 60 * 60 * options.critical
warning_seconds = 60 * 60 * options.warning
(delta, delta_seconds) = time_elapsed(node.report_timestamp)
status = node_status(options.hostname, p)
if status is not None and status['failures'] > 0:
print(('CRITICAL - We have {0} failure(s). Last report was '
'{1} ago'.format(status['failures'], delta)))
sys.exit(2)
elif delta_seconds < warning_seconds:
print('OK - Last run happened {0} ago'.format(delta))
sys.exit(0)
elif warning_seconds <= delta_seconds < critical_seconds:
print('WARNING - Last run happened {0} ago'.format(delta))
sys.exit(1)
elif delta_seconds >= critical_seconds:
print('CRITICAL - Last run happened {0} ago'.format(delta))
sys.exit(2)
else:
print("UNKNOWN - Something went wrong determining this node's state")
sys.exit(3)
def __init__(self, refresh):
self.config = load_config()
if not self.config:
sys.exit('Error: Could not load any config files: {0}'.format(
', '.join(CONFIG_FILES)))
puppetdb_config = {
'host': self.config.get('host'),
'port': self.config.get('port'),
'timeout': self.config.get('timeout'),
'ssl_verify': self.config.get('ssl_verify'),
'ssl_key': self.config.get('ssl_key') or None,
'ssl_cert': self.config.get('ssl_cert') or None
}
self.puppetdb = connect(**puppetdb_config)
self.cache_file = self.config.get('cache_file')
self.cache_duration = self.config.get('cache_duration')
self.refresh = refresh
def __init__(self,
hostname,
port,
api_version,
output_dir,
nodefacts=None,
query=None,
environment=None):
self.db = connect(host=hostname,
port=port,
api_version=api_version,
timeout=20)
self.db.resources = self.db.resources
self.output_dir = output_dir
self.environment = environment
if not nodefacts:
self.nodefacts = self.get_nodefacts()
else:
self.nodefacts = nodefacts
self.query = query
def __init__(self, refresh):
self.config = load_config()
if not self.config:
sys.exit('Error: Could not load any config files: {0}'
.format(', '.join(CONFIG_FILES)))
puppetdb_config = {
'host': self.config.get('host'),
'port': self.config.get('port'),
'timeout': self.config.get('timeout'),
'ssl_verify': self.config.get('ssl_verify'),
'ssl_key': self.config.get('ssl_key') or None,
'ssl_cert': self.config.get('ssl_cert') or None
}
self.puppetdb = connect(**puppetdb_config)
self.cache_file = self.config.get('cache_file')
self.cache_duration = self.config.get('cache_duration')
self.refresh = refresh
def main(hostname, to=None, num_days=7, cache_dir=None, dry_run=False):
"""
main entry point
:param hostname: PuppetDB hostname
:type hostname: string
:param to: list of addresses to send mail to
:type to: list
:param num_days: the number of days to report on, default 7
:type num_days: int
:param cache_dir: absolute path to where to cache data from PuppetDB
:type cache_dir: string
:param dry_run: whether to actually send, or just print what would be sent
:type dry_run: boolean
"""
pdb = connect(host=hostname)
# essentially figure out all these for yesterday, build the tables, serialize the result as JSON somewhere. then just keep the last ~7 days json files
date_data = {}
dates = [] # ordered
date_list = get_date_list(num_days)
localtz = tzlocal.get_localzone()
start_date = date_list[0]
end_date = date_list[-1] - datetime.timedelta(
hours=23, minutes=59, seconds=59)
for query_date in date_list:
end = query_date
start = query_date - datetime.timedelta(days=1) + datetime.timedelta(
seconds=1)
date_s = (query_date - datetime.timedelta(hours=1)
).astimezone(localtz).strftime('%a %m/%d')
date_data[date_s] = get_data_for_timespan(hostname,
pdb,
start,
end,
cache_dir=cache_dir)
dates.append(date_s)
html = format_html(hostname, dates, date_data, start_date, end_date)
subject = 'daily puppet(db) run summary for {host}'.format(host=hostname)
send_mail(to, subject, html, dry_run=dry_run)
return True
def main():
"""main entry point"""
args = get_args()
logging.basicConfig(level=get_log_level(args.verbose))
pdb = connect()
nodes = defaultdict(dict)
max_age = datetime.utcnow() - timedelta(hours=args.max_age)
extract = ExtractOperator()
extract.add_field(['certname', FunctionOperator('count'), 'status'])
extract.add_group_by(['certname', 'status'])
extract.add_query(GreaterOperator('receive_time', max_age.isoformat()))
# pypuppetdb does have a `reports` method which wraps `_query`. however it yields
# results of type pypuppetdb.Report which expects a number of parameters e.g. hash
# to be present in the result payload. however we don't extract theses values and
# therefore have to resort to the more powerful private method
reports = pdb._query('reports', query=extract) # pylint: disable=protected-access
for report in reports:
nodes[report['certname']][report['status']] = report['count']
failed_nodes = [
hostname for hostname, node in nodes.items()
if not node.get('unchanged', 0)
]
if len(failed_nodes) >= args.critical:
print(
'CRITICAL: the following ({}) node(s) change every puppet run: {}'.
format(len(failed_nodes), ', '.join(failed_nodes)))
return 2
if len(failed_nodes) >= args.warning:
print(
'WARNING: the following ({}) node(s) change every puppet run: {}'.
format(len(failed_nodes), ', '.join(failed_nodes)))
return 1
print('OK: all nodes running as expected')
return 0
def main():
"""
Main function
"""
parser = argparse.ArgumentParser(prog='puppetdb_stencil')
parser.add_argument('resource_types', metavar='RESOURCE_TYPE', nargs='+')
parser.add_argument('--templates', '-t', metavar='TEMPLATE', nargs='*')
parser.add_argument('--debug', '-d', action='store_true')
parser.add_argument('--host', '-H', default='localhost')
parser.add_argument('--port', '-p', default='8080')
args = parser.parse_args()
logging.basicConfig(level=logging.DEBUG if args.debug else logging.WARN)
database = pypuppetdb.connect(host=args.host, port=args.port)
for resource_type in args.resource_types:
templates = ['{0}.jinja2'.format(resource_type)]
if args.templates:
templates += args.templates
print(render_resources(database, resource_type, templates))
def main():
"""
Main function
"""
parser = argparse.ArgumentParser(prog='puppetdb_stencil')
parser.add_argument('resource_types', metavar='RESOURCE_TYPE', nargs='+')
parser.add_argument('--templates', '-t', metavar='TEMPLATE', nargs='*')
parser.add_argument('--debug', '-d', action='store_true')
parser.add_argument('--host', '-H', default='localhost')
parser.add_argument('--port', '-p', default='8080')
args = parser.parse_args()
logging.basicConfig(level=logging.DEBUG if args.debug else logging.WARN)
database = pypuppetdb.connect(host=args.host, port=args.port)
for resource_type in args.resource_types:
templates = ['{0}.jinja2'.format(resource_type)]
if args.templates:
templates += args.templates
print(render_resources(database, resource_type, templates))
def __init__(self,
hostname,
port,
api_version,
query=None,
environment=None,
ssl_key=None,
ssl_cert=None,
timeout=20):
from pypuppetdb import connect
self.db = connect(host=hostname,
port=port,
ssl_key=ssl_key,
ssl_cert=ssl_cert,
api_version=api_version,
timeout=timeout)
self.db.resources = self.db.resources
self.environment = environment
if query is None:
query = {}
self.query = query
def sign_host_key(host):
db = pypuppetdb.connect()
hosts = {x.node: x for x in db.facts('ssh')}
if host not in hosts:
return ''
ssh_keys = hosts[host].value
# TODO(bluecmd): Only ecdsa for now
ssh_key = ssh_keys['ecdsa']['key']
# See if this key is already signed and saved
login_data = lib.read_secret(login_path(host)) or {}
if (login_data.get('sshecdsakey', None) == ssh_key
and 'sshecdsakey_signed' in login_data):
return login_data['sshecdsakey_signed']
res = lib.save_secret('ssh/sign/server',
cert_type='host',
public_key=ssh_key)
# Save key
login_data['sshecdsakey'] = ssh_key
login_data['sshecdsakey_signed'] = res['signed_key']
lib.save_secret(login_path(host), **login_data)
return res['signed_key']
def __init__(self, refresh):
self.config = load_config()
self.mac = ':'.join(("%012X" % mac)[i:i + 2] for i in range(0, 12, 2))
if not self.config:
sys.exit('Error: Could not load any config files: {0}'.format(
', '.join(CONFIG_FILES)))
puppetdb_config = {
'host': self.config.get('host'),
'port': self.config.get('port'),
'timeout': self.config.get('timeout'),
'ssl_verify': self.config.get('ssl_verify'),
'ssl_key': self.config.get('ssl_key') or None,
'ssl_cert': self.config.get('ssl_cert') or None
}
self.puppetdb = connect(**puppetdb_config)
self.cache_file = os.path.expanduser(
'~') + '/ansible-inventory-puppetdb.cache'
self.cache_duration = self.config.get('cache_duration')
self.refresh = refresh
def get_puppetdb_resources():
"""Get a list of unique resource inuse by the puppetdb
This functions assumes one can connecto puppetdb:8080 i.e.
add the following to /etc/hosts
127.0.0.1 puppetdb
and:
ssh -L8080:localhost:8080 puppetdb1002.eqiad.wmnet
"""
unique_resources = set()
db = connect()
extract = ExtractOperator()
extract.add_field(['type', 'title'])
# TODO: this is pretty slow as it gets all resources then dose a unique
# would be better to do a select distinct via the api
resources = db._query('resources', query=extract)
for resource in resources:
# if we have a class we want the title to know which class
if resource['type'] == 'Class':
unique_resources.add(resource['title'].lower())
else:
unique_resources.add(resource['type'].lower())
return unique_resources
from pypuppetdb.errors import ExperimentalDisabledError
from puppetboard.forms import QueryForm
from puppetboard.utils import get_or_abort, yield_or_stop, ten_reports
app = Flask(__name__)
app.config.from_object("puppetboard.default_settings")
app.config.from_envvar("PUPPETBOARD_SETTINGS", silent=True)
app.secret_key = os.urandom(24)
puppetdb = connect(
host=app.config["PUPPETDB_HOST"],
port=app.config["PUPPETDB_PORT"],
ssl=app.config["PUPPETDB_SSL"],
ssl_key=app.config["PUPPETDB_KEY"],
ssl_cert=app.config["PUPPETDB_CERT"],
timeout=app.config["PUPPETDB_TIMEOUT"],
experimental=app.config["PUPPETDB_EXPERIMENTAL"],
)
numeric_level = getattr(logging, app.config["LOGLEVEL"].upper(), None)
if not isinstance(numeric_level, int):
raise ValueError("Invalid log level: %s" % loglevel)
logging.basicConfig(level=numeric_level)
log = logging.getLogger(__name__)
def stream_template(template_name, **context):
app.update_template_context(context)
t = app.jinja_env.get_template(template_name)
def test_connect_unknown_api_version():
with pytest.raises(UnsupportedVersionError):
connect(api_version=1)
# This file is part of pypuppetdbquery.
# Copyright © 2016 Chris Boot <*****@*****.**>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Query for selected facts on matching nodes using :mod:`pypuppetdb`.
"""
import pypuppetdb
import pypuppetdbquery
pdb = pypuppetdb.connect()
node_facts = pypuppetdbquery.query_facts(
pdb, '(processorcount=4 or processorcount=8) and kernel=Linux',
['/^lsb/', 'architecture'])
for node in node_facts:
facts = node_facts[node]
print(node, facts)
get_or_abort, yield_or_stop,
ten_reports, jsonprint
)
app = Flask(__name__)
app.config.from_object('puppetboard.default_settings')
app.config.from_envvar('PUPPETBOARD_SETTINGS', silent=True)
app.secret_key = os.urandom(24)
app.jinja_env.filters['jsonprint'] = jsonprint
puppetdb = connect(
api_version=3,
host=app.config['PUPPETDB_HOST'],
port=app.config['PUPPETDB_PORT'],
ssl=app.config['PUPPETDB_SSL'],
ssl_key=app.config['PUPPETDB_KEY'],
ssl_cert=app.config['PUPPETDB_CERT'],
timeout=app.config['PUPPETDB_TIMEOUT'],)
numeric_level = getattr(logging, app.config['LOGLEVEL'].upper(), None)
if not isinstance(numeric_level, int):
raise ValueError('Invalid log level: %s' % loglevel)
logging.basicConfig(level=numeric_level)
log = logging.getLogger(__name__)
def stream_template(template_name, **context):
app.update_template_context(context)
t = app.jinja_env.get_template(template_name)
rv = t.stream(context)
def test_connect_api_v3():
puppetdb = pypuppetdb.connect(api_version=3)
assert puppetdb.version == 'v3'
from pypuppetdb import connect
from puppetboard.forms import QueryForm
from puppetboard.utils import get_or_abort, yield_or_stop, ten_reports
app = Flask(__name__)
app.config.from_object("puppetboard.default_settings")
app.config.from_envvar("PUPPETBOARD_SETTINGS", silent=True)
app.secret_key = os.urandom(24)
puppetdb = connect(
api_version=app.config["PUPPETDB_API"],
host=app.config["PUPPETDB_HOST"],
port=app.config["PUPPETDB_PORT"],
ssl=app.config["PUPPETDB_SSL"],
ssl_key=app.config["PUPPETDB_KEY"],
ssl_cert=app.config["PUPPETDB_CERT"],
timeout=app.config["PUPPETDB_TIMEOUT"],
)
numeric_level = getattr(logging, app.config["LOGLEVEL"].upper(), None)
if not isinstance(numeric_level, int):
raise ValueError("Invalid log level: %s" % loglevel)
logging.basicConfig(level=numeric_level)
log = logging.getLogger(__name__)
def stream_template(template_name, **context):
app.update_template_context(context)
t = app.jinja_env.get_template(template_name)
def test_connect_unknown_api_version():
with pytest.raises(pypuppetdb.errors.UnsupportedVersionError):
pypuppetdb.connect(api_version=1)
def test_connect_api_v2():
puppetdb = pypuppetdb.connect(api_version=2)
assert puppetdb.version == 'v2'
def main():
parser = OptionParser(usage=__usage__, version=__version__)
parser.add_option("-l", "--list", action='store_true', dest='list', help="Get the list of all nodes")
parser.add_option("-f", type="string", action="store", dest="facts",
help="Get fact value from all nodes. Can be as many as you want, comma separated")
parser.add_option("-o", action="store", dest="outofsync", default="30",
help="Get list of out of sync nodes (30 min without sending a report). Number of mins can be passed as a parameter")
(options, args) = parser.parse_args()
if len(sys.argv)==1:
print __usage__
sys.exit(1)
db = connect()
nodes = db.nodes()
if options.list:
for node in nodes:
print node
sys.exit(0)
if options.facts:
facts = options.facts.split(',')
factsout = ""
for node in nodes:
matchfacts = 0
factlistindex = -1
for fact in facts:
factlistindex += 1
try:
# If the user gives a value to the fact...
if '=' in fact:
factarray= fact.split('=')
fact=factarray[0]
if node.fact(fact).value == factarray[1]:
matchfacts +=1
else:
factsout = factsout + " " + node.fact(fact).value
except:
print "Unexpected error:", sys.exc_info()[0]
raise
if matchfacts == len(facts):
# Good! all the given facts have the desired value on this node
print node
if factsout != "": # If there is an array of facts, it means that the user didn't pass a value for the fact,
# so we print out all nodes, and the value of the facts
try:
print '%s: %s' %(node,factsout)
factsout = ""
except:
print "Unexpected error:", sys.exc_info()[0]
raise
if options.outofsync:
# there are 2 hours difference because of the timezone. So instead of dealing with pytz we use this workaround
deltaminutes = 120
now = datetime.datetime.now()
delta = timedelta(minutes=int(deltaminutes))
for node in nodes:
try:
lastcatalog = now - node.catalog_timestamp.replace(tzinfo=None) - delta
minutes = lastcatalog.seconds / 60
except:
minutes = None
if minutes > int(options.outofsync):
print '%s has not sent a report within the last %s minutes' %(node.name,minutes)
sys.exit(0)
def puppetdb_connect(): """Connects to PuppetDB using the parameters specified in the application configuration.""" # Connect to PuppetDB return pypuppetdb.connect(app.config['PUPPETDB_HOST'], port=app.config['PUPPETDB_PORT'], ssl_cert=app.config['PUPPETDB_SSL_CERT'], ssl_key=app.config['PUPPETDB_SSL_KEY'], ssl_verify=app.config['PUPPETDB_SSL_VERIFY'])
from puppetboard.forms import QueryForm
from puppetboard.utils import (get_or_abort, yield_or_stop, ten_reports,
jsonprint)
app = Flask(__name__)
app.config.from_object('puppetboard.default_settings')
app.config.from_envvar('PUPPETBOARD_SETTINGS', silent=True)
app.secret_key = os.urandom(24)
app.jinja_env.filters['jsonprint'] = jsonprint
puppetdb = connect(
api_version=3,
host=app.config['PUPPETDB_HOST'],
port=app.config['PUPPETDB_PORT'],
ssl_verify=app.config['PUPPETDB_SSL_VERIFY'],
ssl_key=app.config['PUPPETDB_KEY'],
ssl_cert=app.config['PUPPETDB_CERT'],
timeout=app.config['PUPPETDB_TIMEOUT'],
)
numeric_level = getattr(logging, app.config['LOGLEVEL'].upper(), None)
if not isinstance(numeric_level, int):
raise ValueError('Invalid log level: %s' % loglevel)
logging.basicConfig(level=numeric_level)
log = logging.getLogger(__name__)
def stream_template(template_name, **context):
app.update_template_context(context)
t = app.jinja_env.get_template(template_name)
def __init__(self, hostname, puppetdb):
self.hostname = hostname
self.puppetdb_host = puppetdb
self.pdb = connect(host=puppetdb)
app = Flask(__name__)
app.config.from_object("puppetboard.default_settings")
graph_facts = app.config["GRAPH_FACTS"]
app.config.from_envvar("PUPPETBOARD_SETTINGS", silent=True)
graph_facts += app.config["GRAPH_FACTS"]
app.secret_key = app.config["SECRET_KEY"]
app.jinja_env.filters["jsonprint"] = jsonprint
app.jinja_env.filters["prettyprint"] = prettyprint
puppetdb = connect(
host=app.config["PUPPETDB_HOST"],
port=app.config["PUPPETDB_PORT"],
ssl_verify=app.config["PUPPETDB_SSL_VERIFY"],
ssl_key=app.config["PUPPETDB_KEY"],
ssl_cert=app.config["PUPPETDB_CERT"],
timeout=app.config["PUPPETDB_TIMEOUT"],
)
numeric_level = getattr(logging, app.config["LOGLEVEL"].upper(), None)
if not isinstance(numeric_level, int):
raise ValueError("Invalid log level: %s" % app.config["LOGLEVEL"])
logging.basicConfig(level=numeric_level)
log = logging.getLogger(__name__)
def stream_template(template_name, **context):
app.update_template_context(context)
t = app.jinja_env.get_template(template_name)
rv = t.stream(context)
def read_callback():
puppetdb = connect(
api_version=3,
host=PUPPETDB_HOST,
port=PUPPETDB_PORT,
ssl_verify=PUPPETDB_SSL,
ssl_key=PUPPETDB_KEY,
ssl_cert=PUPPETDB_CERT,
timeout=PUPPETDB_TIMEOUT,
)
prefix = 'com.puppetlabs.puppetdb.query.population'
num_nodes = puppetdb.metric(
"{0}{1}".format(prefix, ':type=default,name=num-nodes'))
num_resources = puppetdb.metric(
"{0}{1}".format(prefix, ':type=default,name=num-resources'))
avg_resources_node = puppetdb.metric(
"{0}{1}".format(prefix, ':type=default,name=avg-resources-per-node'))
# Fetch nodes
nodes = puppetdb.nodes(
unreported=UNREPORTED_TIME,
with_status=True)
# Init stats
stats = {
'changed': 0,
'unchanged': 0,
'failed': 0,
'unreported': 0,
'noop': 0,
}
for node in nodes:
if node.status == 'unreported':
stats['unreported'] += 1
elif node.status == 'changed':
stats['changed'] += 1
elif node.status == 'failed':
stats['failed'] += 1
elif node.status == 'noop':
stats['noop'] += 1
else:
stats['unchanged'] += 1
log_verbose('population: %s\n' % num_nodes['Value'])
dispatch_value(num_nodes['Value'], 'population', 'gauge')
log_verbose('resources: %s\n' % num_resources['Value'])
dispatch_value(num_resources['Value'], 'resources', 'gauge')
log_verbose('resources_per_node: %s\n' % avg_resources_node['Value'])
dispatch_value(avg_resources_node['Value'], 'resources_per_node', 'gauge')
log_verbose('unreported: %s\n' % stats['unreported'])
dispatch_value(stats['unreported'], 'unreported', 'gauge')
log_verbose('changed: %s\n' % stats['changed'])
dispatch_value(stats['changed'], 'changed', 'gauge')
log_verbose('failed: %s\n' % stats['failed'])
dispatch_value(stats['failed'], 'failed', 'gauge')
log_verbose('noop: %s\n' % stats['noop'])
dispatch_value(stats['noop'], 'noop', 'gauge')
log_verbose('unchanged: %s\n' % stats['unchanged'])
dispatch_value(stats['unchanged'], 'unchanged', 'gauge')
def render_resources(db, resource_type, template_names):
resources = db.resources(resource_type)
try:
template = environment.select_template(template_names)
except jinja2.TemplatesNotFound:
log.error('No template found for {0}'.format(resource_type))
else:
return template.render(resource_type=resource_type,
resources=resources, metaparams=METAPARAMS)
if __name__ == '__main__':
parser = argparse.ArgumentParser(prog='puppetdb_stencil')
parser.add_argument('resource_types', metavar='RESOURCE_TYPE', nargs='+')
parser.add_argument('--templates', '-t', metavar='TEMPLATE', nargs='*')
parser.add_argument('--debug', '-d', action='store_true')
parser.add_argument('--host', '-H', default='localhost')
parser.add_argument('--port', '-p', default='8080')
args = parser.parse_args()
logging.basicConfig(level=logging.DEBUG if args.debug else logging.WARN)
db = pypuppetdb.connect(host=args.host, port=args.port)
for resource_type in args.resource_types:
templates = ['{0}.jinja2'.format(resource_type)]
if args.templates:
templates += args.templates
print(render_resources(db, resource_type, templates))
def api3():
"""Set up a connection to PuppetDB with API version 3."""
puppetdb = connect(api_version=3)
return puppetdb
else:
data[event] = 1
return(data)
#####
parser = optparse.OptionParser()
parser.set_defaults(nodes=[], days=1)
parser.add_option('--days', type="int", dest='days')
(options, nodes) = parser.parse_args()
if not len(nodes):
sys.stderr.write("Syntax: %s node1 [node2 node3...]\n" % sys.argv[0])
sys.exit(1)
db = connect()
# get data and show report
print ' '*10, "*** Puppet node(s) report for last %i day(s) ***\n" % \
(options.days)
data = {}
for n in nodes:
print 'Node:', n
for event,count in get_events(db, n, options.days).iteritems():
if data.has_key(event):
data[event] += count
else:
data[event] = count
print
def api2():
"""Set up a connection to PuppetDB with API version 2."""
puppetdb = connect(api_version=2)
return puppetdb
# Copyright © 2016 Chris Boot <*****@*****.**>
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Query for selected facts on matching nodes using :mod:`pypuppetdb`.
"""
import pypuppetdb
import pypuppetdbquery
pdb = pypuppetdb.connect()
node_facts = pypuppetdbquery.query_facts(
pdb,
'(processorcount=4 or processorcount=8) and kernel=Linux',
['/^lsb/', 'architecture'])
for node in node_facts:
facts = node_facts[node]
print(node, facts)
def connect(self): self.db = pypuppetdb.connect(self.host, port=self.port, ssl_cert=self.ssl_cert, ssl_key=self.ssl_key, ssl_verify=self.ssl_verify)
#!/usr/bin/env python
import argparse
import sys
from pypuppetdb import connect
# Options
parser = argparse.ArgumentParser("For querying puppetdb for nodes of a particular status")
parser.add_argument("status", metavar="status", choices=['changed', 'failed', 'unreported'], help="status of nodes")
parser.add_argument( "--host", metavar="hostname", default="localhost", help="puppet db host to query")
args = parser.parse_args()
# Establish Connection to PuppetDB
puppetdb = connect( host = args.host )
# Get Nodes with status
nodes = puppetdb.nodes(
with_status=True)
# Get Status of Nodes
for node in nodes:
if node.status == args.status:
print (node)
