def check_times(self, start_date, end_date, max_age, max_runtime):
try:
start_datetime = datetime.strptime(start_date, '%m/%d/%Y %H:%M:%S')
end_datetime = datetime.strptime(end_date, '%m/%d/%Y %H:%M:%S')
except ValueError as _:
qquit('UNKNOWN', 'error parsing date time format: {0}'.format(_))
runtime_delta = end_datetime - start_datetime
self.msg += ' in {0}'.format(sec2human(runtime_delta.seconds))
if max_runtime is not None and max_runtime > (runtime_delta.seconds / 3600.0):
self.warning()
self.msg += ' (greater than {0} min{1}!)'.format('{0}'.format(max_runtime).rstrip('.0'),
plural(max_runtime))
age_timedelta = datetime.now() - start_datetime
if self.verbose:
self.msg += ", start date = '{startdate}', end date = '{enddate}'".\
format(startdate=start_date, enddate=end_date)
self.msg += ', started {0} ago'.format(sec2human(age_timedelta.seconds))
if max_age is not None and age_timedelta.seconds > (max_age * 60.0):
self.warning()
self.msg += ' (last run started more than {0} min{1} ago!)'.format('{0}'.format(max_age).rstrip('.0'),
plural(max_age))
self.msg += ' |'
self.msg += ' runtime={0}s;{1}'.format(runtime_delta.seconds, max_runtime * 3600 if max_runtime else '')
self.msg += ' age={0}s;{1}'.format(age_timedelta.seconds, max_age * 3600 if max_age else '')
self.msg += ' auth_time={auth_time}s query_time={query_time}s'.format(auth_time=self.auth_time,
query_time=self.query_time)
def calculate_human_age(timestr):
#started_datetime = time.strptime(started, '%Y-%m-%dT%H:%M:%S.%fZ')
parsed_datetime = datetime.strptime(timestr.split('.')[0], '%Y-%m-%dT%H:%M:%S')
time_delta = datetime.now() - parsed_datetime
secs_ago = time_delta.total_seconds()
human_time = sec2human(secs_ago)
return human_time
def process_build_info(self, build_info):
displayname = build_info['displayName']
duration = build_info['duration']
if not isInt(duration):
raise UnknownError('duration field returned non-integer! {0}'.format(support_msg_api()))
duration = int(duration) / 1000
result = build_info['result']
timestamp = build_info['timestamp']
if not isInt(timestamp):
raise UnknownError('timestamp field returned non-integer! {0}'.format(support_msg_api()))
timestamp = int(timestamp)
building = build_info['building']
self.msg += "build {build} status: ".format(build=displayname)
if building:
self.unknown()
self.msg += 'STILL BUILDING!'
return
self.msg += result
if result != 'SUCCESS':
self.critical()
self.msg += ', duration={duration} secs'.format(duration=duration)
self.check_thresholds(duration)
age = time.time() - (timestamp/1000)
self.msg += ', age={age} secs'.format(age=sec2human(age))
if age < 0:
self.warning()
self.msg += ' (< 0!)'
if self.age and age > self.age:
self.critical()
self.msg += ' (> {0:d})'.format(self.age)
self.msg += ' | build_duration={duration}s{perf_thresholds}'.format(duration=duration, \
perf_thresholds=self.get_perf_thresholds())
def check_times(self, start_date, end_date):
start_date = str(start_date).strip()
end_date = str(end_date).strip()
invalid_dates = ('', 'null', 'None', None)
age_timedelta = None
runtime_delta = None
if start_date not in invalid_dates and \
end_date not in invalid_dates:
try:
start_datetime = datetime.strptime(start_date, '%m/%d/%Y %H:%M:%S')
end_datetime = datetime.strptime(end_date, '%m/%d/%Y %H:%M:%S')
except ValueError as _:
qquit('UNKNOWN', 'error parsing date time format: {0}'.format(_))
runtime_delta = end_datetime - start_datetime
runtime_delta_secs = self.timedelta_seconds(runtime_delta)
self.msg += ' in {0}'.format(sec2human(runtime_delta_secs))
if self.max_runtime is not None and (runtime_delta_secs / 60.0) > self.max_runtime:
self.warning()
self.msg += ' (greater than {0} min{1}!)'.format(str(self.max_runtime).rstrip('0').rstrip('.'),
plural(self.max_runtime))
if self.min_runtime is not None and (runtime_delta_secs / 60.0) < self.min_runtime:
self.warning()
self.msg += ' (less than {0} min{1}!)'.format(str(self.min_runtime).rstrip('0').rstrip('.'),
plural(self.min_runtime))
age_timedelta = datetime.now() - start_datetime
age_timedelta_secs = self.timedelta_seconds(age_timedelta)
if self.verbose:
self.msg += ", start date = '{startdate}', end date = '{enddate}'".\
format(startdate=start_date, enddate=end_date)
if age_timedelta is not None:
self.msg += ', started {0} ago'.format(sec2human(age_timedelta_secs))
if self.max_age is not None and age_timedelta is not None \
and age_timedelta_secs > (self.max_age * 60.0):
self.warning()
self.msg += ' (last run started more than {0} min{1} ago!)'.format(str(self.max_age)
.rstrip('0')
.rstrip('.'),
plural(self.max_age))
# Do not output variable number of fields at all if agedelta is not available as that breaks PNP4Nagios graphing
if age_timedelta is not None and runtime_delta:
self.msg += ' |'
self.msg += ' runtime={0}s;{1}'.format(runtime_delta_secs, self.max_runtime * 60 \
if self.max_runtime else '')
self.msg += ' age={0}s;{1}'.format(age_timedelta_secs, self.max_age * 60 if self.max_age else '')
self.msg += ' auth_time={auth_time}s query_time={query_time}s'.format(auth_time=self.auth_time,
query_time=self.query_time)
def calculate_human_age(timestr):
#started_datetime = time.strptime(started, '%Y-%m-%dT%H:%M:%S.%fZ')
parsed_datetime = datetime.strptime(
timestr.split('.')[0], '%Y-%m-%dT%H:%M:%S')
time_delta = datetime.now() - parsed_datetime
secs_ago = time_delta.total_seconds()
human_time = sec2human(secs_ago)
return (human_time, secs_ago)
def process_result(self, result):
_id = result['id']
log.info('latest build id: %s', _id)
status = result['status']
log.info('status: %s', status)
if not isInt(status, allow_negative=True):
raise UnknownError(
'non-integer status returned by DockerHub API. {0}'.format(
support_msg_api()))
tag = result['dockertag_name']
log.info('tag: %s', tag)
trigger = result['cause']
log.info('trigger: %s', trigger)
created_date = result['created_date']
log.info('created date: %s', created_date)
last_updated = result['last_updated']
log.info('last updated: %s', last_updated)
created_datetime = datetime.datetime.strptime(
created_date.split('.')[0], '%Y-%m-%dT%H:%M:%S')
updated_datetime = datetime.datetime.strptime(
last_updated.split('.')[0], '%Y-%m-%dT%H:%M:%S')
build_latency_timedelta = updated_datetime - created_datetime
build_latency = build_latency_timedelta.total_seconds()
log.info('build latency (creation to last updated): %s', build_latency)
# results in .0 floats anyway
build_latency = int(build_latency)
build_code = result['build_code']
build_url = 'https://hub.docker.com/r/{0}/builds/{1}'.format(
self.repo, build_code)
log.info('latest build URL: %s', build_url)
if str(status) in self.statuses:
status = self.statuses[str(status)]
else:
log.warning("status code '%s' not recognized! %s", status,
support_msg_api())
log.warning('defaulting to assume status is an Error')
status = 'Error'
if status != 'Success':
self.critical()
self.msg += "'{repo}' last completed build status: '{status}', tag: '{tag}', build code: {build_code}"\
.format(repo=self.repo, status=status, tag=tag, build_code=build_code)
if self.verbose:
self.msg += ', id: {0}'.format(_id)
self.msg += ', trigger: {0}'.format(trigger)
self.msg += ', created date: {0}'.format(created_date)
self.msg += ', last updated: {0}'.format(last_updated)
self.msg += ', build_latency: {0}'.format(sec2human(build_latency))
self.msg += ', build URL: {0}'.format(build_url)
self.msg += ' | build_latency={0:d}s'.format(build_latency)
def check_last_ingest_age(self, ingestion_date, max_age):
log.info('checking last ingest age')
age_timedelta = self.get_timedelta(ingestion_date=ingestion_date)
if self.verbose:
self.msg += ", last ingest start date = '{ingestion_date}'".format(
ingestion_date=ingestion_date)
self.msg += ', started {0} ago'.format(
sec2human(age_timedelta.seconds))
if max_age is not None and age_timedelta.seconds > (max_age * 60.0):
self.warning()
self.msg += ' (last run started more than {0} min{1} ago!)'.format(
str(max_age).rstrip('0').rstrip('.'), plural(max_age))
return age_timedelta
def parse_json(self, json_data):
if self.list_jobs:
print('Jenkins Jobs:\n')
for job in json_data['jobs']:
print(job['name'])
sys.exit(ERRORS['UNKNOWN'])
if 'lastCompletedBuild' in json_data:
last_completed_build = json_data['lastCompletedBuild']
if not last_completed_build:
raise WarningError(
"job '{job}' not built yet".format(job=self.job))
self.path = '/job/{job}/{number}/api/json'.format(
job=self.job, number=last_completed_build['number'])
req = self.query()
self.process_json(req.content)
return
displayname = json_data['displayName']
duration = json_data['duration']
if not isInt(duration):
raise UnknownError(
'duration field returned non-integer! {0}'.format(
support_msg_api()))
duration = int(duration) / 1000
result = json_data['result']
timestamp = json_data['timestamp']
if not isInt(timestamp):
raise UnknownError(
'timestamp field returned non-integer! {0}'.format(
support_msg_api()))
timestamp = int(timestamp)
building = json_data['building']
self.msg += "build {build} status: ".format(build=displayname)
if building:
self.unknown()
self.msg += 'STILL BUILDING!'
return
self.msg += result
if result != 'SUCCESS':
self.critical()
self.msg += ', duration={duration} secs'.format(duration=duration)
self.check_thresholds(duration)
age = time.time() - (timestamp / 1000)
self.msg += ', age={age} secs'.format(age=sec2human(age))
if age < 0:
self.warning()
self.msg += ' (< 0!)'
if self.age and age > self.age:
self.critical()
self.msg += ' (> {0:d})'.format(self.age)
self.msg += ' | build_duration={duration}s{perf_thresholds}'.format(
duration=duration, perf_thresholds=self.get_perf_thresholds())
def process_result(self, result):
_id = result['id']
log.info('latest build id: %s', _id)
status = result['status']
log.info('status: %s', status)
if not isInt(status, allow_negative=True):
raise UnknownError('non-integer status returned by DockerHub API. {0}'.format(support_msg_api()))
tag = result['dockertag_name']
log.info('tag: %s', tag)
trigger = result['cause']
log.info('trigger: %s', trigger)
created_date = result['created_date']
log.info('created date: %s', created_date)
last_updated = result['last_updated']
log.info('last updated: %s', last_updated)
created_datetime = datetime.datetime.strptime(created_date.split('.')[0], '%Y-%m-%dT%H:%M:%S')
updated_datetime = datetime.datetime.strptime(last_updated.split('.')[0], '%Y-%m-%dT%H:%M:%S')
build_latency_timedelta = updated_datetime - created_datetime
build_latency = build_latency_timedelta.total_seconds()
log.info('build latency (creation to last updated): %s', build_latency)
# results in .0 floats anyway
build_latency = int(build_latency)
build_code = result['build_code']
build_url = 'https://hub.docker.com/r/{0}/builds/{1}'.format(self.repo, build_code)
log.info('latest build URL: %s', build_url)
if str(status) in self.statuses:
status = self.statuses[str(status)]
else:
log.warning("status code '%s' not recognized! %s", status, support_msg_api())
log.warning('defaulting to assume status is an Error')
status = 'Error'
if status != 'Success':
self.critical()
self.msg += "'{repo}' last completed build status: '{status}', tag: '{tag}', build code: {build_code}"\
.format(repo=self.repo, status=status, tag=tag, build_code=build_code)
if self.verbose:
self.msg += ', id: {0}'.format(_id)
self.msg += ', trigger: {0}'.format(trigger)
self.msg += ', created date: {0}'.format(created_date)
self.msg += ', last updated: {0}'.format(last_updated)
self.msg += ', build_latency: {0}'.format(sec2human(build_latency))
self.msg += ', build URL: {0}'.format(build_url)
self.msg += ' | build_latency={0:d}s'.format(build_latency)
def parse(self, req):
if not isJson(req.content):
raise UnknownError('non-JSON returned by HiveServer2 Interactive instance at {0}:{1}'\
.format(self.host, self.port))
_ = json.loads(req.content)
status = self.get_key(_, 'status')
uptime = self.get_key(_, 'uptime')
self.msg2 = 'uptime = {0}'.format(sec2human(int(uptime/1000)))
if self.verbose:
self.msg2 += ', version ' + self.get_key(_, 'build').split('from')[0]
if status == 'STARTED':
self.ok()
else:
self.critical()
return status
def parse(self, req):
if not isJson(req.content):
raise UnknownError('non-JSON returned by HiveServer2 Interactive instance at {0}:{1}'\
.format(self.host, self.port))
_ = json.loads(req.content)
status = self.get_key(_, 'status')
uptime = self.get_key(_, 'uptime')
self.msg2 = 'uptime = {0}'.format(sec2human(int(uptime / 1000)))
if self.verbose:
self.msg2 += ', version ' + self.get_key(_,
'build').split('from')[0]
if status == 'STARTED':
self.ok()
else:
self.critical()
return status
def parse_json(self, json_data):
if self.list_jobs:
print('Jenkins Jobs:\n')
for job in json_data['jobs']:
print(job['name'])
sys.exit(ERRORS['UNKNOWN'])
if 'lastCompletedBuild' in json_data:
last_completed_build = json_data['lastCompletedBuild']
if not last_completed_build:
raise WarningError("job '{job}' not built yet".format(job=self.job))
self.path = '/job/{job}/{number}/api/json'.format(job=self.job,
number=last_completed_build['number'])
req = self.query()
self.process_json(req.content)
return
displayname = json_data['displayName']
duration = json_data['duration']
if not isInt(duration):
raise UnknownError('duration field returned non-integer! {0}'.format(support_msg_api()))
duration = int(duration) / 1000
result = json_data['result']
timestamp = json_data['timestamp']
if not isInt(timestamp):
raise UnknownError('timestamp field returned non-integer! {0}'.format(support_msg_api()))
timestamp = int(timestamp)
building = json_data['building']
self.msg += "build {build} status: ".format(build=displayname)
if building:
self.unknown()
self.msg += 'STILL BUILDING!'
return
self.msg += result
if result != 'SUCCESS':
self.critical()
self.msg += ', duration={duration} secs'.format(duration=duration)
self.check_thresholds(duration)
age = time.time() - (timestamp/1000)
self.msg += ', age={age} secs'.format(age=sec2human(age))
if age < 0:
self.warning()
self.msg += ' (< 0!)'
if self.age and age > self.age:
self.critical()
self.msg += ' (> {0:d})'.format(self.age)
self.msg += ' | build_duration={duration}s{perf_thresholds}'.format(duration=duration,
perf_thresholds=self.get_perf_thresholds())
def check_ingestion(self,
num,
filter_opts=None,
max_age=None,
max_runtime=None):
log.info('checking ingestion history')
json_dict = self.get_ingestions(num, filter_opts)
info = ''
if self.verbose:
for key in sorted(filter_opts):
info += " {0}='{1}'".format(key, filter_opts[key])
try:
results = json_dict['result']
if not results:
qquit('CRITICAL', "no results found for ingestion{0}"\
.format('{0}. {1}'.format(info, self.extract_response_message(json_dict)) + \
'Perhaps you specified incorrect filters? Use --list to see existing ingestions'))
num_results = len(results)
log.info('%s ingestion history results returned', num_results)
self.check_statuses(results)
if num:
self.msg += ' out of last {0} ingest{1}'.format(
num_results, plural(num_results))
if self.history_mins:
self.msg += ' within last {0} ({1} min{2})'.format(
sec2human(self.history_mins * 60),
str(self.history_mins).rstrip('0').rstrip('.'),
plural(self.history_mins))
longest_incomplete_timedelta = self.check_longest_incomplete_ingest(
results, max_runtime)
age_timedelta_secs = self.check_last_ingest_age(results,
max_age=max_age)
self.msg_filter_details(filter_opts=filter_opts)
self.msg += ' |'
self.msg += ' last_ingest_age={0}s;{1}'.format(
age_timedelta_secs, max_age * 3600 if max_age else '')
self.msg += ' longest_incomplete_ingest_age={0}s;{1}'\
.format(self.timedelta_seconds(longest_incomplete_timedelta)
if longest_incomplete_timedelta else 0,
max_age * 3600 if max_age else '')
self.msg += ' auth_time={auth_time}s query_time={query_time}s'.format(
auth_time=self.auth_time, query_time=self.query_time)
except KeyError as _:
qquit('UNKNOWN',
'error parsing workflow execution history: {0}'.format(_))
def check_longest_incomplete_ingest(self, result, max_runtime=None):
log.info('checking longest running incomplete ingest')
longest_incomplete_timedelta = None
for item in result:
status = item['status']
if status == 'INCOMPLETE' and max_runtime is not None:
runtime_delta = self.get_timedelta(item['ingestionTimeFormatted'])
if longest_incomplete_timedelta is None or \
self.timedelta_seconds(runtime_delta) > self.timedelta_seconds(longest_incomplete_timedelta):
longest_incomplete_timedelta = runtime_delta
if max_runtime is not None and \
longest_incomplete_timedelta is not None and \
self.timedelta(longest_incomplete_timedelta) > max_runtime * 60.0:
self.warning()
self.msg += ', longest incomplete ingest runtime = {0} ago! '\
.format(sec2human(self.timedelta_seconds(longest_incomplete_timedelta))) + \
'(greater than expected {0} min{1})'\
.format(str(max_runtime).rstrip('0').rstrip('.'), plural(max_runtime))
return longest_incomplete_timedelta
def check_last_ingest_age(self, results, max_age):
log.info('checking last ingest age')
if not isList(results):
code_error('passed non-list to check_last_ingest_age()')
# newest is first
# effectiveDate is null in testing (docs says it's a placeholder for future use)
# using ingestionTimeFormatted instead, could also use ingestionTime which is timestamp in millis
ingestion_date = results[0]['ingestionTimeFormatted']
age_timedelta = self.get_timedelta(ingestion_date=ingestion_date)
if self.verbose:
self.msg += ", last ingest start date = '{ingestion_date}'".format(
ingestion_date=ingestion_date)
self.msg += ', started {0} ago'.format(
sec2human(age_timedelta.seconds))
if max_age is not None and age_timedelta.seconds > (max_age * 60.0):
self.warning()
self.msg += ' (last run started more than {0} min{1} ago!)'.format(
str(max_age).rstrip('0').rstrip('.'), plural(max_age))
return age_timedelta
def check_file_age(self, filename):
log.info('checking hbck log file age')
now = int(time.time())
mtime = int(os.stat(filename).st_mtime)
age = now - mtime
log.info('file age = %s secs', age)
self.msg += ' HBCK log file age is '
# sec2human doesn't handle negative
if age < 0:
self.msg += '{0} secs (modified timestamp is in the future!)'.format(age)
else:
self.msg += sec2human(age)
if self.max_file_age == 0:
return age
elif age < 0:
self.unknown()
elif age > self.max_file_age:
self.warning()
self.msg += ' (greater than max age of {0} secs!)'.format(self.max_file_age)
return age
def check_last_ingest_age(self, results, max_age):
log.info('checking last ingest age')
if not isList(results):
code_error('passed non-list to check_last_ingest_age()')
# newest is first
# effectiveDate is null in testing (docs says it's a placeholder for future use)
# using ingestionTimeFormatted instead, could also use ingestionTime which is timestamp in millis
ingestion_date = results[0]['ingestionTimeFormatted']
age_timedelta = self.get_timedelta(ingestion_date=ingestion_date)
age_timedelta_secs = self.timedelta_seconds(age_timedelta)
if self.verbose:
self.msg += ", last ingest start date = '{ingestion_date}'".format(ingestion_date=ingestion_date)
self.msg += ', started {0} ago'.format(sec2human(age_timedelta_secs))
if max_age is not None and age_timedelta_secs > (max_age * 60.0):
self.warning()
self.msg += ' (last run started more than {0} min{1} ago!)'.format(str(max_age)
.rstrip('0')
.rstrip('.'),
plural(max_age))
return age_timedelta_secs
def check_ingestion(self, num, filter_opts=None, max_age=None, max_runtime=None):
log.info('checking ingestion history')
json_dict = self.get_ingestions(num, filter_opts)
info = ''
if self.verbose:
for key in sorted(filter_opts):
info += " {0}='{1}'".format(key, filter_opts[key])
try:
results = json_dict['result']
if not results:
qquit('CRITICAL', "no results found for ingestion{0}"\
.format('{0}. {1}'.format(info, self.extract_response_message(json_dict)) + \
'Perhaps you specified incorrect filters? Use --list to see existing ingestions'))
num_results = len(results)
log.info('%s ingestion history results returned', num_results)
self.check_statuses(results)
if num:
self.msg += ' out of last {0} ingest{1}'.format(num_results, plural(num_results))
if self.history_mins:
self.msg += ' within last {0} ({1} min{2})'.format(sec2human(self.history_mins * 60),
str(self.history_mins).rstrip('0').rstrip('.'),
plural(self.history_mins))
longest_incomplete_timedelta = self.check_longest_incomplete_ingest(results, max_runtime)
age_timedelta_secs = self.check_last_ingest_age(results, max_age=max_age)
self.msg_filter_details(filter_opts=filter_opts)
self.msg += ' |'
self.msg += ' last_ingest_age={0}s;{1}'.format(age_timedelta_secs,
max_age * 3600 if max_age else '')
self.msg += ' longest_incomplete_ingest_age={0}s;{1}'\
.format(self.timedelta_seconds(longest_incomplete_timedelta)
if longest_incomplete_timedelta else 0,
max_age * 3600 if max_age else '')
self.msg += ' auth_time={auth_time}s query_time={query_time}s'.format(auth_time=self.auth_time,
query_time=self.query_time)
except KeyError as _:
qquit('UNKNOWN', 'error parsing workflow execution history: {0}'.format(_))