def _connect(self, ldapserver):
if self.session:
raise CheckError('LDAP Error: duplicate bind exception.')
self.logger.debug('connect to %s', ldapserver)
uri = ldapserver
if not uri.startswith("ldap://") and not uri.startswith("ldaps://"):
uri = 'ldap://{0}'.format(ldapserver)
try:
if self.use_krb5 and self.use_tls:
self.logger.debug('sasl bind')
self.session = ldapapi.Session(uri, tls=True)
elif self.use_tls:
self.logger.debug('simple bind')
self.session = ldapapi.Session(uri,
self.bind_dn,
self.__password,
tls=True)
elif uri.startswith("ldaps://"):
self.logger.debug('LDAPS + simple bind')
self.session = ldapapi.Session(uri,
self.bind_dn,
self.__password,
tls=False)
else:
raise CheckError('LDAP Error: unsupported method exception.')
except CheckError:
raise
except Exception, e:
raise CheckError('Error connecting to LDAP: {}'.format(
e)), None, sys.exc_info()[2]
def auth(self):
try:
start = time.time()
self._connect(self.host)
connect_elapsed = time.time() - start
self._search(self.user_base_dn, self.user_filter, self.user_attrib)
all_elapsed = time.time() - start
search_elapsed = all_elapsed - connect_elapsed
self._disconnect()
return {
'ok': True,
'connect_time': connect_elapsed,
'search_time': search_elapsed,
'elapsed': all_elapsed,
}
except ldap.LOCAL_ERROR:
bind_type = 'simple bind'
if self.use_krb5:
bind_type = 'sasl bind'
msg = 'Could not connect to {} via {}'.format(self.host, bind_type)
self.logger.exception(msg)
raise CheckError(msg)
except ldap.CONFIDENTIALITY_REQUIRED:
return {'ok': False}
except Exception, e:
raise CheckError('Error authenticating to LDAP: {}'.format(
e)), None, sys.exc_info()[2]
def power_query(self, query, minutes=5, end=0):
if not query or not query.strip():
raise CheckError(
'query "{}" is not allowed to be blank'.format(query))
value = {
'token': self.__read_key,
'query': query,
'startTime': parse_timestamp(minutes),
'priority': 'low',
}
if end is not None:
value['endTime'] = parse_timestamp(end)
response = requests.post(self.__power_query_url,
json=value,
headers={
'Content-Type': 'application/json',
'errorStatus': 'always200'
})
json_response = response.json()
if json_response.get('status', '').startswith('error'):
raise CheckError(
json_response.get(
'message',
'Unexpected error message was returned from scalyr'))
return json_response
def __init__(self,
shards,
user='******',
password='',
timeout=60000,
shard=None):
'''
Parameters
----------
shards: dict
A dict of shard definitions where key is the shard's name and value is the host/database string.
user: str
password: str
timeout: int
Statement timeout in milliseconds.
shard: str
Optional shard name. If provided, the check will be run on only one shard matching given name.
'''
if not shards:
raise CheckError('SqlWrapper: No shards defined')
if shard and not shards.get(shard):
raise CheckError(
'SqlWrapper: Shard {} not found in shards definition'.format(
shard))
self._cursors = []
self._stmt = None
mdb = _import_db_driver()
for shard_def in ([shards[shard]] if shard else shards.values()):
m = CONNECTION_RE.match(shard_def)
if not m:
raise CheckError(
'Invalid shard connection: {}'.format(shard_def))
try:
conn = mdb.connect(
host=m.group('host'),
user=user,
passwd=password,
db=m.group('dbname'),
port=(int(m.group('port'))
if int(m.group('port')) > 0 else DEFAULT_PORT),
connect_timeout=timeout)
except Exception, e:
raise DbError(str(e),
operation='Connect to {}'.format(
shard_def)), None, sys.exc_info()[2]
# TODO: Find a way to enforce readonly=True as it is done in postgres Wrapper
# TODO: Do we need to set charset="utf8" and use_unicode=True in connection?
conn.autocommit(True)
self._cursors.append(conn.cursor(mdb.cursors.DictCursor))
def logs(self,
query,
max_count=100,
minutes=5,
continuation_token=None,
columns=None,
end=0):
if not query or not query.strip():
raise CheckError(
'query "{}" is not allowed to be blank'.format(query))
val = {
'token': self.__read_key,
'queryType': 'log',
'maxCount': max_count,
'filter': query,
'startTime': parse_timestamp(minutes),
'priority': 'low'
}
if end is not None:
val['endTime'] = parse_timestamp(end)
if columns:
val['columns'] = ','.join(
columns) if type(columns) is list else str(columns)
if continuation_token:
val['continuationToken'] = continuation_token
r = requests.post(self.__query_url,
json=val,
headers={
'Content-Type': 'application/json',
'errorStatus': 'always200'
})
j = r.json()
if 'matches' in j:
new_continuation_token = j.get('continuationToken', None)
messages = j['matches'] if columns else [
match['message'] for match in j['matches']
]
return {
'messages': messages,
'continuation_token': new_continuation_token
}
if j.get('status', '').startswith('error'):
raise CheckError(j['message'])
else:
raise CheckError(
'No logs or error message was returned from scalyr')
def search(self, base, fltr, attrs, scope=ldap.SCOPE_SUBTREE):
try:
self._connect(self.host)
return self._search(base, fltr, attrs, scope)
except Exception, e:
raise CheckError('Error searching LDAP: {}'.format(e))
def certs(self):
parse = urlparse.urlparse(self.url)
if parse.scheme != 'https':
raise CheckError('Expected "https" connection!')
parsed_host = parse.netloc.split(':')
host = parsed_host[0]
port = parsed_host[1] if len(parsed_host) > 1 else 443
# TODO: The following could be moved to a diff wrapper with more TLS related features.
try:
sock = socket.socket()
sock.settimeout(self.timeout)
ssl_sock = ssl.wrap_socket(sock, cert_reqs=ssl.CERT_REQUIRED)
ssl_sock.connect((host, int(port)))
# returns a single certificate, not the entire certificate chain!
cert = ssl_sock.getpeercert()
if cert:
# Inject ZMON friendly time. This will make it useful with time() wrapper in checks and alerts.
cert['not_before'] = time.strftime(
ZMON_FORMAT, time.strptime(cert['notBefore'], CERT_FORMAT))
cert['not_after'] = time.strftime(
ZMON_FORMAT, time.strptime(cert['notAfter'], CERT_FORMAT))
return [cert]
finally:
ssl_sock.close()
return []
def jolokia(self, read_requests, raise_error=True):
'''
:param read_requests: see https://jolokia.org/reference/html/protocol.html#post-request
:type read_requests: list
:param raise_error:
:return: Jolokia response
'''
def set_read_type(x):
x['type'] = 'read'
# hack quick verify
parsed_url = urlparse.urlsplit(self.url)
if (not parsed_url.path.endswith('/jolokia/')) or (
'?' in self.url) or ('&' in self.url):
raise HttpError(
"URL needs to end in /jolokia/ and not contain ? and &",
self.url)
map(set_read_type, read_requests)
for rr in read_requests:
if 'mbean' not in rr:
raise CheckError('missing "mbean" key in read request')
r = self.__request(post_data=read_requests, raise_error=raise_error)
try:
return r.json()
except Exception, e:
raise HttpError(str(e), self.url), None, sys.exc_info()[2]
def deployments(self, name=None, replicas=None, ready=None, **kwargs):
"""
Return list of Deployments.
:param name: Deployment name.
:type name: str
:param replicas: Deployment replicas.
:type replicas: int
:param ready: Deployment ready status.
:type ready: bool
:param **kwargs: Deployment labelSelector filters.
:type **kwargs: dict
:return: List of Deployments. Typical Deployment has "metadata", "status" and "spec".
:rtype: list
"""
if ready is not None and type(ready) is not bool:
raise CheckError('Invalid ready value.')
filter_kwargs = self._get_filter_kwargs(name=name, **kwargs)
query = pykube.Deployment.objects(
self.__client).filter(**filter_kwargs)
deployments = self._get_resources(query)
return [
deployment.obj for deployment in deployments
if (replicas is None or deployment.replicas == replicas) and (
ready is None or deployment.ready == ready)
]
def _disconnect(self):
self.logger.debug('disconnect')
try:
self.session.disconnect()
except Exception, e:
raise CheckError('Error disconnecting to LDAP: {}'.format(
e)), None, sys.exc_info()[2]
def _sync(self, ldapservers):
'''Returns a list of dict, where 'diff' is the difference to the 'newest' of the full list,
'newest' is the newest timestamp for the given 'server',
'ok' means LDAP state for current 'server' and 'elapsed' the runtime of that ldap request.
Example dict:
{'diff': 0,
'elapsed': 0.2969970703125,
'newest': 20140228135148,
'ok': True,
'server': 'myserver'}
'''
if not ldapservers:
return
results = []
for ldapserver in ldapservers:
try:
start = time.time()
csn_list = self._sync_state(ldapserver)
rid2ts = {}
rid_ts = map(lambda csn: self._get_timestamp_rid(csn),
csn_list)
newest = rid_ts[0][1]
for rid, ts in rid_ts:
rid2ts[rid] = ts
if ts > newest:
newest = ts
elapsed = time.time() - start
results.append({
'server': ldapserver,
'ok': True,
'newest': newest,
'elapsed': elapsed,
})
except ldap.LOCAL_ERROR:
bind_type = 'simple bind'
if self.use_krb5:
bind_type = 'sasl bind'
msg = 'Could not connect to {} via {}'.format(
ldapserver, bind_type)
self.logger.exception(msg)
raise CheckError(msg)
except ldap.CONFIDENTIALITY_REQUIRED:
results.append({'ok': False, 'server': ldapserver})
newest = 0
for result in results:
if result['ok']:
if result['newest'] > newest:
newest = result['newest']
for result in results:
if result['ok']:
result['diff'] = newest - result['newest']
return results
def pods(self, name=None, phase=None, ready=None, **kwargs):
"""
Return list of Pods.
:param name: Pod name.
:type name: str
:param phase: Pod status phase. Valid values are: Pending, Running, Failed, Succeeded or Unknown.
:type phase: str
:param ready: Pod ready status. If None then all pods are returned.
:type ready: bool
:param **kwargs: Pod labelSelector filters. Example: application__in=['app-1', 'app-2'], version='v0.1'
Supported filter syntax:
- <label>
- <label>__in
- <label>__notin
- <label>__neq
:type **kwargs: dict
:return: List of pods. Typical pod has "metadata", "status" and "spec".
:rtype: list
"""
if ready is not None and type(ready) is not bool:
raise CheckError('Invalid ready value.')
if phase and phase not in VALID_PHASE:
raise CheckError(
'Invalid phase. Valid phase values are {}'.format(VALID_PHASE))
filter_kwargs = self._get_filter_kwargs(name, phase, **kwargs)
query = pykube.Pod.objects(self.__client).filter(**filter_kwargs)
pods = self._get_resources(query)
return [pod.obj for pod in pods if ready is None or pod.ready == ready]
def _get_resources(object_manager, name=None, field_selector=None, **kwargs):
if name is not None:
if object_manager.namespace == pykube.all:
raise CheckError("namespace is required for name= queries")
if field_selector is not None or kwargs:
raise CheckError("name= query doesn't support additional filters")
try:
return [object_manager.get_by_name(name)]
except pykube.exceptions.ObjectDoesNotExist:
return []
filter_kwargs = {}
if field_selector:
filter_kwargs['field_selector'] = field_selector
# labelSelector
if kwargs:
filter_kwargs['selector'] = kwargs
return list(object_manager.filter(**filter_kwargs))
def alarms(self,
alarm_names=None,
alarm_name_prefix=None,
state_value=STATE_ALARM,
action_prefix=None,
max_records=50):
"""
Retrieve cloudwatch alarms.
:param alarm_names: List of alarm names.
:type alarm_names: list
:param alarm_name_prfix: Prefix of alarms. Cannot be specified if ``alarm_names`` is specified.
:type alarm_name_prfix: str
:param state_value: State value used in alarm filtering. Available values are STATE_OK, STATE_ALARM(default) and STATE_DATA.
:type state_value: str
:param action_prefix: Action prefix.
:type action_prefix: str
:param max_records: Maximum records to be returned. Default is 50.
:type max_records: int
:return: List of MetricAlarms.
:rtype: list
""" # noqa
if alarm_names and alarm_name_prefix:
raise CheckError(
'"alarm_name_prefix" cannot be sprecified if "alarm_names" is specified!'
)
kwargs = dict(MaxRecords=max_records)
if state_value:
kwargs.update({'StateValue': state_value})
if alarm_names:
alarm_names = [alarm_names] if isinstance(
alarm_names, basestring) else alarm_names
kwargs['AlarmNames'] = alarm_names
elif alarm_name_prefix:
kwargs['AlarmNamePrefix'] = alarm_name_prefix
if action_prefix:
kwargs['ActionPrefix'] = action_prefix
return self.__client.describe_alarms(**kwargs)['MetricAlarms']
def sync(self):
'''Example:
checkldap().sync() => [
{'newest': 20140516151002, 'elapsed': 0.14442706108093262, 'ok': True, 'diff': 0,'server': 'myserv'},
{'newest': 20140516151002, 'elapsed': 0.19423580169677734, 'ok': True, 'diff': 0, 'server': 'myserver'},
{'newest': 20140516151002, 'elapsed': 0.2617530822753906, 'ok': True, 'diff': 0, 'server': 'z-auth123.example'},
{'newest': 20140516151002, 'elapsed': 0.15635299682617188, 'ok': True, 'diff': 0, 'server': 'myserver'},
{'newest': 20140516151002, 'elapsed': 0.20283913612365723, 'ok': True, 'diff': 0, 'server': 'myserver'}]
'''
try:
rid2url, url2rid = self._get_rid_to_url(self.host)
ldapservers = map(lambda url: url[7:], url2rid.keys())
return self._sync(ldapservers)
except CheckError:
raise
except Exception, e:
raise CheckError('{}'.format(e)), None, sys.exc_info()[2]
def __init__(
self,
url,
method='GET',
params=None,
base_url=None,
timeout=10,
max_retries=0,
allow_redirects=None,
verify=True,
oauth2=False,
oauth2_token_name='uid',
headers=None,
):
if method.lower() not in ('get', 'head'):
raise CheckError(
'Invalid method. Only GET and HEAD are supported!')
if not base_url and not absolute_http_url(url):
# More verbose error message!
raise ConfigurationError(
'HTTP wrapper improperly configured. Invalid base_url. Check entity["url"] or call with absolute url.'
)
self.url = (base_url + url if not absolute_http_url(url) else url)
self.clean_url = None
self.params = params
self.timeout = timeout
self.max_retries = max_retries
self.verify = verify
self._headers = headers or {}
self.oauth2 = oauth2
self.oauth2_token_name = oauth2_token_name
self.__method = method.lower()
self.allow_redirects = True if allow_redirects is None else allow_redirects
if self.__method == 'head' and allow_redirects is None:
self.allow_redirects = False
self.__r = None
def local(self, check, timeout=60, **kwargs):
config = self.__local_config[check]
parameters = {}
parameters.update(config.get('parameters', {}))
parameters.update(kwargs)
pre_run_hook_ok = config.get('pre_run_hook', self._check_ok)
if not pre_run_hook_ok(config.get('parameters', {})):
raise CheckError('Pre run hook does not accept your parameters')
cmd_args = config['args'].format(**parameters)
cmd = shlex.split('/usr/lib/nagios/plugins/{c} {a}'.format(c=check,
a=cmd_args))
try:
output = subprocess32.check_output(cmd,
shell=False,
timeout=timeout)
except subprocess32.CalledProcessError, e:
e = fix_sub32_exc(e)
if e.returncode < 3:
output = str(e.output)
else:
output = str(e.output)
return output
def get_details(self, pipeline_ids):
"""
Return a list of pipelines with their details.
:param name: Pipeline IDs as a String for a single item or list of Strings for multiple pipelines.
:param type: str
:param type: list
:return: Details from the requested pipelines
:rtype: map
"""
if isinstance(pipeline_ids, str):
pipeline_ids = [pipeline_ids]
else:
if not isinstance(pipeline_ids, list):
raise CheckError(
'Parameter \"pipeline_ids\" should be a string or a list of strings '
'denoting pipeline IDs')
response = self.__client.describe_pipelines(pipelineIds=pipeline_ids)
# parse the response and manipulate data to return the pipeline id and its description fields
pipelines_states = [
(str(pipeline['pipelineId']),
create_dict_from_list_of_fields(pipeline['fields']))
for pipeline in response['pipelineDescriptionList']
]
result = {}
if not pipelines_states:
return result
# create a dict of pipeline_id : details_map
for (pipeline_id, pipeline_details) in pipelines_states:
result[pipeline_id] = pipeline_details
# returns a map which has the pipeline IDs as keys and their details as values
return result
def _request(self, endpoint, q, method='get'):
try:
url = urlparse.urljoin(self.__service_url, endpoint)
request = getattr(self.__session, method.lower())
if method.lower() == 'post':
response = request(url, json=q)
else:
response = request(url, params={'query': json.dumps(q)})
if response.ok:
return response.json()
else:
raise CheckError(
'EntitiesWrapper query failed: {} with status {}:{}'.
format(q, response.status_code, response.text))
except requests.Timeout:
raise HttpError('timeout',
self.__service_url), None, sys.exc_info()[2]
except requests.ConnectionError:
raise HttpError('connection failed',
self.__service_url), None, sys.exc_info()[2]
'priority': 'low',
}, power_query_response, None),
(
{
'query': 'power-query'
}, # malformed powerquery without message
{
'token': SCALYR_READ_KEY,
'query': 'power-query',
'startTime': '5m',
'priority': 'low',
},
{
'status': 'error/something'
},
CheckError('Unexpected error message was returned from scalyr')),
(
{
'query': 'power-query'
}, # malformed powerquery with message
{
'token': SCALYR_READ_KEY,
'query': 'power-query',
'startTime': '5m',
'priority': 'low',
},
{
'status': 'error/something',
'message': 'Expected error message from Scalyr'
},
CheckError('Expected error message from Scalyr')),
def __init__(
self,
shards,
user='******',
password='',
timeout=60000,
shard=None,
created_by=None,
check_id=None,
):
'''
Parameters
----------
shards: dict
A dict of shard definitions where key is the shard's name and value is the host/database string.
user: str
password: str
timeout: int
Statement timeout in milliseconds.
shard: str
Optional shard name. If provided, the check will be run on only one shard matching given name.
created_by: str
Optional user name. If provided, the check will first make sure that the user has permissions to access the
requested database. It's optional because it's currently supported only in trial run.
check_id: int
The check definition ID in order to set PostgreSQL application name (easier tracking on server side).
'''
if not shards:
raise CheckError('SqlWrapper: No shards defined')
if shard and not shards.get(shard):
raise CheckError(
'SqlWrapper: Shard {} not found in shards definition'.format(
shard))
self._cursors = []
self._stmt = None
permissions = {}
for shard_def in ([shards[shard]] if shard else shards.values()):
m = CONNECTION_RE.match(shard_def)
if not m:
raise CheckError(
'Invalid shard connection: {}'.format(shard_def))
connection_str = (
"host='{host}' port='{port}' dbname='{dbname}' user='******' password='******' "
"connect_timeout=5 options='-c statement_timeout={timeout}' "
"application_name='ZMON Check {check_id} (created by {created_by})' "
).format(
host=m.group('host'),
port=int(m.group('port') or DEFAULT_PORT),
dbname=m.group('dbname'),
user=user,
password=password,
timeout=timeout,
check_id=check_id,
created_by=make_safe(created_by),
)
try:
conn = psycopg2.connect(connection_str)
conn.set_session(readonly=True, autocommit=True)
cursor = conn.cursor(cursor_factory=NamedTupleCursor)
self._cursors.append(cursor)
except Exception, e:
raise DbError(str(e),
operation='Connect to {}'.format(
shard_def)), None, sys.exc_info()[2]
try:
if created_by:
cursor.execute(PERMISSIONS_STMT, [created_by])
row = cursor.fetchone()
permissions[shard_def] = (
row.can_login and REQUIRED_GROUP in row.member_of
if row else False)
except Exception, e:
raise DbError(
str(e),
operation='Permission query'), None, sys.exc_info()[2]
def statistics_raw(self):
'''collect statistics from OpenLDAP "Monitor" DB as a dict
ldapsearch -b cn=Connections,cn=Monitor -H ldap://myserver '(monitorCounter=*)' '+'
Example result::
{
"connections_current": "51",
"connections_max_file_descriptors": "65536",
"connections_total": "26291",
"operations_add_completed": "0",
"operations_add_initiated": "0",
"operations_bind_completed": "25423",
"operations_bind_initiated": "25423",
"operations_delete_completed": "0",
"operations_delete_initiated": "0",
"operations_extended_completed": "293",
"operations_extended_initiated": "293",
"operations_modify_completed": "91",
"operations_modify_initiated": "91",
"operations_search_completed": "22865",
"operations_search_initiated": "22866",
"operations_unbind_completed": "25233",
"operations_unbind_initiated": "25233",
"statistics_bytes": "122581936",
"statistics_entries": "64039",
"statistics_pdu": "112707",
"statistics_referrals": "0",
"waiters_read": "51",
"waiters_write": "0"
}
'''
try:
self._connect(self.host)
data = {}
# we need to use the internal "conn" attribute as the default _search is using paging which does not work
# for the "cn=Monitor" tree!
result = self.session.conn.search_s(
'cn=Monitor', ldap.SCOPE_SUBTREE,
'(objectClass=monitorCounterObject)', ['monitorCounter'])
for dn, attr in result:
category, counter = self._split_monitor_dn(dn)
data['{}_{}'.format(category,
counter)] = int(attr['monitorCounter'][0])
result = self.session.conn.search_s(
'cn=Threads,cn=Monitor', ldap.SCOPE_SUBTREE,
'(&(objectClass=monitoredObject)(monitoredInfo=*))',
['monitoredInfo'])
for dn, attr in result:
category, key = self._split_monitor_dn(dn)
if key in ('active', 'max'):
data['{}_{}'.format(category,
key)] = int(attr['monitoredInfo'][0])
result = self.session.conn.search_s(
'cn=Operations,cn=Monitor', ldap.SCOPE_SUBTREE,
'(objectClass=monitorOperation)',
['monitorOpInitiated', 'monitorOpCompleted'])
for dn, attr in result:
category, op = self._split_monitor_dn(dn)
if op in STATISTICS_OPERATIONS_TO_MONITOR:
data['{}_{}_initiated'.format(category, op)] = int(
attr['monitorOpInitiated'][0])
data['{}_{}_completed'.format(category, op)] = int(
attr['monitorOpCompleted'][0])
self._disconnect()
except CheckError:
raise
except Exception, e:
raise CheckError('{}'.format(e)), None, sys.exc_info()[2]
