def save_result2(self, name, monitor):
if self.only_failures and monitor.virtual_fail_count() == 0:
return
dateformat = 'timestamp'
if self.dateformat == 'iso8601':
dateformat = 'iso8601'
if dateformat == 'timestamp':
datestring = str(int(time.time()))
elif dateformat == 'iso8601':
datestring = format_datetime(datetime.datetime.now())
try:
if monitor.virtual_fail_count() > 0:
self.file_handle.write("%s %s: failed since %s; VFC=%d (%s) (%0.3fs)" % (
datestring,
name,
format_datetime(monitor.first_failure_time()),
monitor.virtual_fail_count(),
monitor.get_result(),
monitor.last_run_duration
))
else:
self.file_handle.write("%s %s: ok (%0.3fs)" % (
datestring,
name,
monitor.last_run_duration
))
self.file_handle.write("\n")
if not self.buffered:
self.file_handle.flush()
except Exception as e:
self.logger_logger.exception("Error writing to logfile %s", self.filename)
def send_alert(self, name, monitor):
"""Build up the content for the push notification."""
type = self.should_alert(monitor)
(days, hours, minutes, seconds) = monitor.get_downtime()
if monitor.is_remote():
host = " on %s " % monitor.running_on
else:
host = " on host %s" % self.hostname
subject = ""
body = ""
if type == "":
return
elif type == "failure":
subject = "[%s] Monitor %s Failed!" % (self.hostname, name)
body = """Monitor %s%s has failed.\n
Failed at: %s
Downtime: %d+%02d:%02d:%02d
Virtual failure count: %d
Additional info: %s
Description: %s""" % (
name, host, format_datetime(monitor.first_failure_time()),
days, hours, minutes, seconds, monitor.virtual_fail_count(),
monitor.get_result(), monitor.describe())
try:
if monitor.recover_info != "":
body += "\nRecovery info: %s" % monitor.recover_info
except AttributeError:
body += "\nNo recovery info available"
elif type == "success":
subject = "[%s] Monitor %s succeeded" % (self.hostname, name)
body = "Monitor %s%s is back up.\nOriginally failed at: %s\nDowntime: %d+%02d:%02d:%02d\nDescription: %s" % (
name, host, format_datetime(monitor.first_failure_time()),
days, hours, minutes, seconds, monitor.describe())
elif type == "catchup":
subject = "[%s] Monitor %s failed earlier!" % (self.hostname, name)
body = "Monitor %s%s failed earlier while this alerter was out of hours.\nFailed at: %s\nVirtual failure count: %d\nAdditional info: %s\nDescription: %s" % (
name, host, format_datetime(monitor.first_failure_time()),
monitor.virtual_fail_count(), monitor.get_result(),
monitor.describe())
else:
self.alerter_logger.error("Unknown alert type %s", type)
return
if not self.dry_run:
try:
self.send_pushbullet_notification(subject, body)
except Exception:
self.alerter_logger.exception(
"Couldn't send push notification")
self.available = False
else:
self.alerter_logger.info(
"dry_run: would send push notification: %s" % body)
def test_Format(self):
self.assertEqual(util.format_datetime(None), "")
self.assertEqual(util.format_datetime("a string"), "a string")
self.assertEqual(
util.format_datetime(datetime.datetime(2018, 5, 8, 13, 37, 0)),
"2018-05-08 13:37:00"
)
def test_lte(self):
self.file_entry.ctime = int(self.file_entry.ctime)
f1 = FileRule("ctime", "lte", format_datetime(self.file_entry.ctime), "include")
f2 = FileRule("ctime", "lte", format_datetime(time() - 3600), "include")
f3 = FileRule("ctime", "eq", format_datetime(self.file_entry.ctime), "include")
self.assertEqual(f1.test(self.file_entry), "include")
self.assertEqual(f2.test(self.file_entry), None)
self.assertEqual(f3.test(self.file_entry), "include")
def test_gte(self):
for op in ['gte', '>=']:
f1 = FileRule('mtime', op, format_datetime(time() - 3600),
'exclude')
f2 = FileRule('mtime', op, format_datetime(time() + 3600),
'exclude')
self.assertEqual(f1.test(self.file_entry), 'exclude')
self.assertEqual(f2.test(self.file_entry), None)
def test_gte(self):
for op in ['gte', '>=']:
f1 = FileRule('mtime', op,
format_datetime(time()-3600), 'exclude')
f2 = FileRule('mtime', op,
format_datetime(time()+3600), 'exclude')
self.assertEqual(f1.test(self.file_entry), 'exclude')
self.assertEqual(f2.test(self.file_entry), None)
def test_lte(self):
self.file_entry.ctime = int(self.file_entry.ctime)
f1 = FileRule('ctime', 'lte',
format_datetime(self.file_entry.ctime), 'include')
f2 = FileRule('ctime', 'lte',
format_datetime(time()-3600), 'include')
f3 = FileRule('ctime', 'eq',
format_datetime(self.file_entry.ctime), 'include')
self.assertEqual(f1.test(self.file_entry), 'include')
self.assertEqual(f2.test(self.file_entry), None)
self.assertEqual(f3.test(self.file_entry), 'include')
def test_lte(self):
self.file_entry.ctime = int(self.file_entry.ctime)
f1 = FileRule('ctime', 'lte', format_datetime(self.file_entry.ctime),
'include')
f2 = FileRule('ctime', 'lte', format_datetime(time() - 3600),
'include')
f3 = FileRule('ctime', 'eq', format_datetime(self.file_entry.ctime),
'include')
self.assertEqual(f1.test(self.file_entry), 'include')
self.assertEqual(f2.test(self.file_entry), None)
self.assertEqual(f3.test(self.file_entry), 'include')
def start_end(tt):
# the code is bad!! ugly
keys = ['dateTime', 'date']
if tt:
for key in keys:
if key in tt:
if key == keys[0]:
return util.format_datetime(iso8601.parse_date(tt[key]))
elif key ==keys[1]:
return util.format_datetime(datetime.datetime.strptime('2012-07-07', '%Y-%m-%d'))
else:
return None
def send_alert(self, name, monitor):
"""Build up the content for the push notification."""
type = self.should_alert(monitor)
(days, hours, minutes, seconds) = monitor.get_downtime()
if monitor.is_remote():
host = " on %s " % monitor.running_on
else:
host = " on host %s" % self.hostname
body = ""
if type == "":
return
elif type == "failure":
body = """Monitor %s DOWN
Failed at: %s
Downtime: %d+%02d:%02d:%02d
Description: %s""" % (name, format_datetime(monitor.first_failure_time()),
days, hours, minutes, seconds, monitor.describe())
try:
if monitor.recover_info != "":
body += "\nRecovery info: %s" % monitor.recover_info
except AttributeError:
body += "\nNo recovery info available"
elif type == "success":
body = """Monitor %s UP
Originally failed at: %s
Downtime: %d+%02d:%02d:%02d
Description: %s""" % (name, format_datetime(monitor.first_failure_time()),
days, hours, minutes, seconds, monitor.describe())
elif type == "catchup":
body = "Monitor %s%s failed earlier while this alerter was out of hours.\nFailed at: %s\nDescription: %s" % (
name, host, format_datetime(
monitor.first_failure_time()), monitor.describe())
else:
self.alerter_logger.error("Unknown alert type %s", type)
return
if not self.dry_run:
try:
self.send_telegram_notification(body)
except Exception:
self.alerter_logger.exception(
"Couldn't send push notification")
self.available = False
else:
self.alerter_logger.info(
"dry_run: would send push notification: %s" % body)
def start_end(tt):
# the code is bad!! ugly
keys = ['dateTime', 'date']
if tt:
for key in keys:
if key in tt:
if key == keys[0]:
return util.format_datetime(iso8601.parse_date(tt[key]))
elif key == keys[1]:
return util.format_datetime(
datetime.datetime.strptime('2012-07-07', '%Y-%m-%d'))
else:
return None
def save_result2(self, name, monitor):
if self.only_failures and monitor.virtual_fail_count() == 0:
return
try:
if monitor.virtual_fail_count() > 0:
self.file_handle.write("%s %s: failed since %s; VFC=%d (%s) (%0.3fs)" % (
self._get_datestring(),
name,
format_datetime(monitor.first_failure_time()),
monitor.virtual_fail_count(),
monitor.get_result(),
monitor.last_run_duration
))
else:
self.file_handle.write("%s %s: ok (%0.3fs)" % (
self._get_datestring(),
name,
monitor.last_run_duration
))
self.file_handle.write("\n")
if not self.buffered:
self.file_handle.flush()
except Exception:
self.logger_logger.exception("Error writing to logfile %s", self.filename)
def test(data_in, fmt, data_exp):
data_out = util.format_datetime(data_in, fmt)
if data_out == data_exp:
ret = 'OK'
else:
ret = 'NG'
print('[' + ret + '] IN=' + data_in + ' ' + 'EXP=' + data_exp + ' OUT=' + data_out)
def get_push_list(self):
import const
from util import format_datetime
sep = self.get_list_separator_line()
res = "push message nad ip address:port status program alc program downloaded response\r\n"
res += sep
for controller in globals.controllers.get_sorted():
if controller.config.created_from_push:
file_transfer_timestamp = controller.alloc.get_file_transfer_timestamp()
if file_transfer_timestamp != None:
file_transfer_timestamp = file_transfer_timestamp.strftime(const.Timeformat)
else:
file_transfer_timestamp = "unknown "
try:
response_time = "%d" % controller.sys_status.get_sys_response_time()
except:
response_time = controller.sys_status.get_sys_response_time()
res += "%s %s %s%s %s %s %s %s\r\n" % \
(
format_datetime(controller.sys_status.last_push_timestamp),
repr(controller.config.nad).ljust(5),
("%s:%s" % (controller.config.ip, repr(controller.config.port))).ljust(22),
controller.sys_status.get_sys_plc_status().ljust(7),
controller.sys_status.get_sys_plc_program_status().ljust(7),
controller.sys_status.get_sys_alc_file_status().ljust(7),
file_transfer_timestamp,
response_time.ljust(8),
)
return res + sep
def _get_show_artist_time(self):
return {
'artist_id': self.artist_id,
'artist_name': self.artist.name,
'artist_image_link': self.artist.image_link,
'start_time': format_datetime(str(self.start_time), format='full')
}
def _get_show_venue_time(self):
return {
'venue_id': self.venue_id,
'venue_name': self.venue.name,
'venue_image_link': self.venue.image_link,
'start_time': format_datetime(str(self.start_time), format='full')
}
def gen_report(self, rcu, qcu, usermap, created=None):
# first, title of the report
title = "{0}|{1}|{2}".format(self.clustername, self.quota,
self.cores_per_node)
# second, created datetime
created = created if created else datetime.datetime.now()
report_content = []
# datetime, an addtional "\n" is just for making the reportprettier
report_content.append("{0}\n".format(util.format_datetime(created)))
if not rcu and not qcu:
report_content.append("data not available at the moment")
else:
rcu, qcu = util.prune(rcu), util.prune(qcu)
total_usage = {}
for realname in set(usermap.values()):
total_usage[realname] = sum(
dd.get(realname, 0) for dd in [rcu, qcu])
total_usage = util.prune(total_usage)
# from this step on, basically it's about print data from 3 dicts
# in a pretty way: rcu, qcu, total_usage
# 1. print headers
report_content.append("{0:13s} {1:8s} {2:8s} {3:8s}".format(
'USERNAME', 'Running', 'NotRunning', 'TOTAL'))
# 2. sort the order of key by total_usage
sorted_keys = reversed(sorted(total_usage, key=total_usage.get))
if not len(total_usage) == 0: # not the usage of everyone is zero
report_content.append('=' * 44)
# 3. print the table
for k in sorted_keys:
# full name is too long, so last name is used since
# firstname is confusing
name = k.split()[0]
report_content.append(
"{0:13s} {1:<8d} {2:<8d} {3:<8d}".format(
name, rcu.get(k, 0), qcu.get(k, 0),
total_usage.get(k, 0)))
# 4. print the footer sum
report_content.append('=' * 44)
report_content.append("{0:13s} {1:<8d} {2:<8d} {3:<8d}".format(
'SUM', sum(rcu.values()), sum(qcu.values()),
sum(total_usage.values())))
report_content.append('=' * 44)
# 5. join the final work
report_content = '\n'.join(report_content)
# 6. since it's displayed on line, need such replacements
report_content = report_content.replace("\n",
"<br>").replace(" ", " ")
return Report(self, report_content, created)
def send_alert(self, name, monitor):
"""Build up the content for the push notification."""
type = self.should_alert(monitor)
(days, hours, minutes, seconds) = monitor.get_downtime()
if monitor.is_remote():
host = " on %s " % monitor.running_on
else:
host = " on host %s" % self.hostname
subject = ""
body = ""
if type == "":
return
elif type == "failure":
subject = "[%s] Monitor %s Failed!" % (self.hostname, name)
body = """Monitor %s%s has failed.\n
Failed at: %s
Downtime: %d+%02d:%02d:%02d
Virtual failure count: %d
Additional info: %s
Description: %s""" % (
name,
host,
format_datetime(monitor.first_failure_time()),
days, hours, minutes, seconds,
monitor.virtual_fail_count(),
monitor.get_result(),
monitor.describe())
try:
if monitor.recover_info != "":
body += "\nRecovery info: %s" % monitor.recover_info
except AttributeError:
body += "\nNo recovery info available"
elif type == "success":
subject = "[%s] Monitor %s succeeded" % (self.hostname, name)
body = "Monitor %s%s is back up.\nOriginally failed at: %s\nDowntime: %d+%02d:%02d:%02d\nDescription: %s" % (name, host, format_datetime(monitor.first_failure_time()), days, hours, minutes, seconds, monitor.describe())
elif type == "catchup":
subject = "[%s] Monitor %s failed earlier!" % (self.from_addr, self.to_addr, self.hostname, name)
body = "Monitor %s%s failed earlier while this alerter was out of hours.\nFailed at: %s\nVirtual failure count: %d\nAdditional info: %s\nDescription: %s" % (name, host, format_datetime(monitor.first_failure_time()), monitor.virtual_fail_count(), monitor.get_result(), monitor.describe())
else:
self.alerter_logger.error("Unknown alert type %s", type)
return
if not self.dry_run:
try:
self.send_pushover_notification(subject, body)
except Exception:
self.alerter_logger.exception("Couldn't send push notification")
self.available = False
else:
self.alerter_logger.info("dry_run: would send push notification: %s", body)
def parse_file(self, file_handle):
lines = []
for line in file_handle:
line = line.replace("_NOW_", format_datetime(datetime.datetime.now()))
line = line.replace("_HOST_", socket.gethostname())
line = line.replace("_COUNTS_", self.count_data)
line = line.replace("_TIMESTAMP_", str(int(time.time())))
line = line.replace("_STATUS_", self.status)
lines.append(line)
return lines
def parse_file(self, file_handle):
lines = []
for line in file_handle:
line = line.replace("_NOW_", format_datetime(datetime.datetime.now()))
line = line.replace("_HOST_", socket.gethostname())
line = line.replace("_COUNTS_", self.count_data)
line = line.replace("_TIMESTAMP_", str(int(time.time())))
line = line.replace("_STATUS_", self.status)
lines.append(line)
return lines
def gen_report(self, rcu, qcu, usermap, created=None):
# first, title of the report
title = "{0}|{1}|{2}".format(self.clustername, self.quota, self.cores_per_node)
# second, created datetime
created = created if created else datetime.datetime.now()
report_content = []
# datetime, an addtional "\n" is just for making the reportprettier
report_content.append("{0}\n".format(util.format_datetime(created)))
if not rcu and not qcu:
report_content.append("data not available at the moment")
else:
rcu, qcu = util.prune(rcu), util.prune(qcu)
total_usage = {}
for realname in set(usermap.values()):
total_usage[realname] = sum(dd.get(realname, 0) for dd in [rcu, qcu])
total_usage = util.prune(total_usage)
# from this step on, basically it's about print data from 3 dicts
# in a pretty way: rcu, qcu, total_usage
# 1. print headers
report_content.append("{0:13s} {1:8s} {2:8s} {3:8s}".format(
'USERNAME', 'Running', 'NotRunning', 'TOTAL'))
# 2. sort the order of key by total_usage
sorted_keys = reversed(sorted(total_usage, key=total_usage.get))
if not len(total_usage) == 0: # not the usage of everyone is zero
report_content.append('=' * 44)
# 3. print the table
for k in sorted_keys:
# full name is too long, so last name is used since
# firstname is confusing
name = k.split()[0]
report_content.append("{0:13s} {1:<8d} {2:<8d} {3:<8d}".format(
name, rcu.get(k, 0), qcu.get(k, 0), total_usage.get(k, 0)))
# 4. print the footer sum
report_content.append('=' * 44)
report_content.append("{0:13s} {1:<8d} {2:<8d} {3:<8d}".format(
'SUM', sum(rcu.values()), sum(qcu.values()),
sum(total_usage.values())))
report_content.append('=' * 44)
# 5. join the final work
report_content = '\n'.join(report_content)
# 6. since it's displayed on line, need such replacements
report_content = report_content.replace("\n", "<br>").replace(" ", " ")
return Report(self, report_content, created)
def process_batch(self):
payload = MonitorJsonPayload()
payload.generated = format_datetime(datetime.datetime.now())
payload.monitors = self.batch_data
with open(self.filename, 'w') as outfile:
json.dump(payload, outfile,
indent=4,
separators=(',', ':'),
ensure_ascii=False,
cls=PayloadEncoder)
self.batch_data = {}
def process_batch(self):
payload = MonitorJsonPayload()
payload.generated = format_datetime(datetime.datetime.now())
payload.monitors = self.batch_data
with open(self.filename, 'w') as outfile:
json.dump(payload, outfile,
indent=4,
separators=(',', ':'),
ensure_ascii=False,
cls=PayloadEncoder)
self.batch_data = {}
def save_result2(self, name, monitor):
result = MonitorResult()
result.first_failure_time = format_datetime(monitor.first_failure_time())
result.virtual_fail_count = monitor.virtual_fail_count()
result.last_run_duration = monitor.last_run_duration
result.result = monitor.get_result()
if hasattr(monitor, "was_skipped") and monitor.was_skipped:
result.status = "Skipped"
elif monitor.virtual_fail_count() <= 0:
result.status = "OK"
result.dependencies = monitor._dependencies
self.batch_data[name] = result
def save_result2(self, name, monitor):
result = MonitorResult()
result.first_failure_time = format_datetime(monitor.first_failure_time())
result.virtual_fail_count = monitor.virtual_fail_count()
result.last_run_duration = monitor.last_run_duration
result.result = monitor.get_result()
if hasattr(monitor, "was_skipped") and monitor.was_skipped:
result.status = "Skipped"
elif monitor.virtual_fail_count() <= 0:
result.status = "OK"
result.dependencies = monitor._dependencies
self.batch_data[name] = result
def groupcalendar():
if flask.request.method == "GET":
storage = Storage(thedata.STORAGE)
cred = storage.get()
# the next line copied from
# https://code.google.com/p/google-api-python-client/wiki/OAuth2
if cred is None or cred.invalid == True:
flow = get_flow()
callback = thedata.REDIRECT_URI
authorize_url = flow.step1_get_authorize_url(callback)
thedata.MEMC.set("OAUTH2_FLOW", pickle.dumps(flow))
return flask.redirect(authorize_url)
else:
http = httplib2.Http()
http = cred.authorize(http)
probe = build('calendar', 'v3', http=http)
# comments are just for records: ['acl', 'calendarList',
# 'calendars', 'colors', 'events', 'freebusy', 'settings']
interested_calId = thedata.GROUP_CAL_ID
current = datetime.datetime.now()
delta_30d = datetime.timedelta(days=30)
events = probe.events().list(
calendarId=interested_calId,
timeMin=util.format_datetime(current-delta_30d, iso=True),
timeMax=util.format_datetime(current, iso=True)).execute()
eVents = []
for event in events['items']:
eVent = Event(
event.get('creator', '{}'),
event.get('start', '{}'),
event.get('end', '{}'),
event.get('location', 'noloc'),
event.get('summary', 'nosummary'),
)
eVents.append(eVent)
return flask.render_template("cal.html", events=eVents)
def groupcalendar():
if flask.request.method == "GET":
storage = Storage(thedata.STORAGE)
cred = storage.get()
# the next line copied from
# https://code.google.com/p/google-api-python-client/wiki/OAuth2
if cred is None or cred.invalid == True:
flow = get_flow()
callback = thedata.REDIRECT_URI
authorize_url = flow.step1_get_authorize_url(callback)
thedata.MEMC.set("OAUTH2_FLOW", pickle.dumps(flow))
return flask.redirect(authorize_url)
else:
http = httplib2.Http()
http = cred.authorize(http)
probe = build('calendar', 'v3', http=http)
# comments are just for records: ['acl', 'calendarList',
# 'calendars', 'colors', 'events', 'freebusy', 'settings']
interested_calId = thedata.GROUP_CAL_ID
current = datetime.datetime.now()
delta_30d = datetime.timedelta(days=30)
events = probe.events().list(
calendarId=interested_calId,
timeMin=util.format_datetime(current - delta_30d, iso=True),
timeMax=util.format_datetime(current, iso=True)).execute()
eVents = []
for event in events['items']:
eVent = Event(
event.get('creator', '{}'),
event.get('start', '{}'),
event.get('end', '{}'),
event.get('location', 'noloc'),
event.get('summary', 'nosummary'),
)
eVents.append(eVent)
return flask.render_template("cal.html", events=eVents)
def send_alert(self, name, monitor):
type_ = self.should_alert(monitor)
command = None
(days, hours, minutes, seconds) = monitor.get_downtime()
if monitor.is_remote():
host = monitor.running_on
else:
host = self.hostname
if type_ == "":
return
elif type_ == "failure":
command = self.fail_command
elif type_ == "success":
command = self.success_command
elif type_ == "catchup":
if self.catchup_command == 'fail_command':
command = self.fail_command
else:
self.alerter_logger.error("Unknown alert type %s", type_)
return
if command is None:
return
command = command.format(
hostname=host,
name=name,
days=days,
hours=hours,
minutes=minutes,
seconds=seconds,
failed_at=format_datetime(monitor.first_failure_time()),
virtual_fail_count=monitor.virtual_fail_count(),
info=monitor.get_result(),
description=monitor.describe(),
last_virtual_fail_count=monitor.last_virtual_fail_count())
if not self.dry_run:
self.alerter_logger.debug("About to execute command: %s", command)
try:
subprocess.call(shlex.split(command))
except Exception:
self.alerter_logger.exception(
"Exception encountered running command: %s", command)
if self.debug:
self.alerter_logger.debug("Command has finished.")
else:
self.alerter_logger.info("Would run command: %s", command)
def save_result2(self, name, monitor):
if not self.doing_batch:
self.logger_logger.error(
"HTMLLogger.save_result2() called while not doing batch.")
return
if monitor.virtual_fail_count() == 0:
status = True
else:
status = False
if not status:
fail_time = format_datetime(monitor.first_failure_time())
fail_count = monitor.virtual_fail_count()
fail_data = monitor.get_result()
downtime = monitor.get_downtime()
else:
fail_time = ""
fail_count = 0
fail_data = monitor.get_result()
downtime = ""
failures = monitor.failures
last_failure = monitor.last_failure
try:
age = datetime.datetime.utcnow() - monitor.last_update
age = age.days * 3600 + age.seconds
update = monitor.last_update
except Exception:
age = 0
update = ""
data_line = {
"status": status,
"fail_time": fail_time,
"fail_count": fail_count,
"fail_data": fail_data,
"downtime": downtime,
"age": age,
"update": update,
"host": monitor.running_on,
"failures": failures,
"last_failure": last_failure
}
self.batch_data[monitor.name] = data_line
def save_result2(self, name, monitor):
if not self.doing_batch:
self.logger_logger.error("HTMLLogger.save_result2() called while not doing batch.")
return
if monitor.virtual_fail_count() == 0:
status = True
else:
status = False
if not status:
fail_time = format_datetime(monitor.first_failure_time())
fail_count = monitor.virtual_fail_count()
fail_data = monitor.get_result()
downtime = monitor.get_downtime()
else:
fail_time = ""
fail_count = 0
fail_data = monitor.get_result()
downtime = ""
failures = monitor.failures
last_failure = monitor.last_failure
try:
age = datetime.datetime.utcnow() - monitor.last_update
age = age.days * 3600 + age.seconds
update = monitor.last_update
except Exception:
age = 0
update = ""
data_line = {
"status": status,
"fail_time": fail_time,
"fail_count": fail_count,
"fail_data": fail_data,
"downtime": downtime,
"age": age,
"update": update,
"host": monitor.running_on,
"failures": failures,
"last_failure": last_failure
}
self.batch_data[monitor.name] = data_line
def get_abus_list(self):
from util import format_datetime
sep = self.get_list_separator_line()
res = "nad abus total abus error last error at code bandwidth\r\n"
res += sep
for controller in globals.controllers.get_sorted():
if controller.comm_proxy.last_error_timestamp != None:
last_error_timestamp = format_datetime(controller.comm_proxy.last_error_timestamp)
else:
last_error_timestamp = "-"
res += "%s %s %s %s %s %.2f%%\r\n" % \
(
repr(controller.config.nad).ljust(5),
repr(controller.comm_proxy.abus_messages_tx_count).ljust(10),
repr(controller.comm_proxy.abus_error_count).ljust(10),
last_error_timestamp.ljust(19),
controller.comm_proxy.last_error_code.ljust(5),
controller.comm_proxy.get_bandwidth(),
)
return res + sep
def send_alert(self, name, monitor):
"""Send the email."""
type = self.should_alert(monitor)
(days, hours, minutes, seconds) = monitor.get_downtime()
if monitor.is_remote():
host = " on %s " % monitor.running_on
else:
host = " on host %s" % self.hostname
message = MIMEMultipart()
message['From'] = self.from_addr
message['To'] = self.to_addr
if type == "":
return
elif type == "failure":
message['Subject'] = "[%s] Monitor %s Failed!" % (self.hostname,
name)
body = """Monitor %s%s has failed.
Failed at: %s
Downtime: %d+%02d:%02d:%02d
Virtual failure count: %d
Additional info: %s
Description: %s""" % (
name, host, format_datetime(monitor.first_failure_time()),
days, hours, minutes, seconds, monitor.virtual_fail_count(),
monitor.get_result(), monitor.describe())
try:
if monitor.recover_info != "":
body += "\nRecovery info: %s" % monitor.recover_info
except AttributeError:
body += "\nNo recovery info available"
elif type == "success":
message['Subject'] = "[%s] Monitor %s succeeded" % (self.hostname,
name)
body = "Monitor %s%s is back up.\nOriginally failed at: %s\nDowntime: %d+%02d:%02d:%02d\nDescription: %s" % (
name, host, format_datetime(monitor.first_failure_time()),
days, hours, minutes, seconds, monitor.describe())
elif type == "catchup":
message['Subject'] = "[%s] Monitor %s failed earlier!" % (
self.hostname, name)
body = "Monitor %s%s failed earlier while this alerter was out of hours.\nFailed at: %s\nVirtual failure count: %d\nAdditional info: %s\nDescription: %s" % (
name, host, format_datetime(monitor.first_failure_time()),
monitor.virtual_fail_count(), monitor.get_result(),
monitor.describe())
else:
self.alerter_logger.critical("unknown alert type %s", type)
return
message.attach(MIMEText(body, 'plain'))
if not self.dry_run:
try:
if self.ssl is None or self.ssl == 'starttls':
server = smtplib.SMTP(self.mail_host, self.mail_port)
elif self.ssl == 'yes':
server = smtplib.SMTP_SSL(self.mail_host, self.mail_port)
if self.ssl == 'starttls':
server.starttls()
if self.username is not None:
server.login(self.username, self.password)
server.sendmail(self.from_addr, self.to_addr.split(';'),
message.as_string())
server.quit()
except Exception:
self.alerter_logger.exception("couldn't send mail")
self.available = False
else:
self.alerter_logger.info("dry_run: would send email: %s",
message.as_string())
def send_alert(self, name, monitor):
"""Send the message."""
type = self.should_alert(monitor)
(days, hours, minutes, seconds) = monitor.get_downtime()
if self.channel is not None:
message_json = {'channel': self.channel}
elif self.username is not None:
message_json = {'username': self.username}
else:
message_json = {}
message_json['attachments'] = [{}]
if type == "":
return
elif type == "failure":
message_json['text'] = "Monitor {} failed!".format(name)
message_json['attachments'][0]['color'] = 'danger'
fields = [
{
'title': 'Failed at',
'value': format_datetime(monitor.first_failure_time()),
'short': True
},
{
'title': 'Downtime',
'value': "{}+{:02d}:{:02d}:{:02d}".format(days, hours, minutes, seconds),
'short': True
},
{
'title': 'Virtual failure count',
'value': monitor.virtual_fail_count(),
'short': True
},
{
'title': 'Host',
'value': self.hostname,
'short': True
},
{
'title': 'Additional info',
'value': monitor.get_result()
},
{
'title': 'Description',
'value': monitor.describe()
}
]
try:
if monitor.recover_info != "":
fields.append({
'title': 'Recovery info',
'value': "Recovery info: %s" % monitor.recover_info
})
message_json['attachments'][0]['color'] = 'warning'
except AttributeError:
pass
message_json['attachments'][0]['fields'] = fields
elif type == "success":
message_json['text'] = "Monitor {} succeeded.".format(name)
fields = [
{
'title': 'Failed at',
'value': format_datetime(monitor.first_failure_time()),
'short': True
},
{
'title': 'Downtime',
'value': "{}+{:02d}:{:02d}:{:02d}".format(days, hours, minutes, seconds),
'short': True
},
{
'title': 'Host',
'value': self.hostname,
'short': True
},
{
'title': 'Description',
'value': monitor.describe()
}
]
message_json['attachments'][0]['color'] = 'good'
message_json['attachments'][0]['fields'] = fields
else:
self.alerter_logger.error("unknown alert type %s", type)
return
if not self.dry_run:
try:
r = requests.post(self.url, json=message_json)
if not r.status_code == 200:
self.alerter_logger.error("POST to slack webhook failed: %s", r)
except Exception:
self.alerter_logger.exception("Failed to post to slack webhook")
self.available = False
else:
self.alerter_logger.info("dry_run: would send slack: %s", message_json.__repr__())
def send_alert(self, name, monitor):
"""Send an SMS alert."""
if not monitor.is_urgent():
return
type = self.should_alert(monitor)
message = ""
url = ""
(days, hours, minutes, seconds) = monitor.get_downtime()
if type == "":
return
elif type == "catchup":
message = "catchup: %s failed on %s at %s (%d+%02d:%02d:%02d)\n%s" % (
name, monitor.running_on,
format_datetime(monitor.first_failure_time()), days, hours,
minutes, seconds, monitor.get_result())
if len(message) > 160:
self.alerter_logger.warning(
"Truncating SMS message to 160 chars.")
message = message[:156] + "..."
url = "https://{}/a1/SMS".format(self.api_host)
auth = (self.username, self.password)
params = {
'from': self.sender,
'to': self.target,
'message': message,
}
elif type == "failure":
message = "%s failed on %s at %s (%d+%02d:%02d:%02d)\n%s" % (
name, monitor.running_on,
format_datetime(monitor.first_failure_time()), days, hours,
minutes, seconds, monitor.get_result())
if len(message) > 160:
self.alerter_logger.warning(
"Truncating SMS message to 160 chars.")
message = message[:156] + "..."
url = "https://{}/a1/SMS".format(self.api_host)
auth = (self.username, self.password)
params = {
'from': self.sender,
'to': self.target,
'message': message,
}
else:
# we don't handle other types of message
pass
if url == "":
return
if not self.dry_run:
try:
response = requests.post(url, data=params, auth=auth)
s = response.json()
if s['status'] not in ('created', 'delivered'):
self.alerter_logger.error("Unable to send SMS: %s", s)
self.available = False
except Exception as e:
self.alerter_logger.exception("SMS sending failed")
self.available = False
else:
self.alerter_logger.info("dry_run: would send SMS: %s", url)
return
def send_alert(self, name, monitor):
"""Send an alert."""
if not monitor.is_urgent():
return
type = self.should_alert(monitor)
message = ""
url = ""
(days, hours, minutes, seconds) = monitor.get_downtime()
if type == "":
return
elif type == "catchup":
message = "catchup: %s failed on %s at %s (%d+%02d:%02d:%02d)\n%s" % (
name, monitor.running_on,
format_datetime(monitor.first_failure_time()), days, hours,
minutes, seconds, monitor.get_result())
url = "https://{}/publicapi/notify".format(self.api_host)
params = {
'apikey': self.apikey,
'application': self.application,
'description': message,
'event': "%s: %s" % (name, monitor.get_result())
}
elif type == "failure":
message = "%s failed on %s at %s (%d+%02d:%02d:%02d)\n%s" % (
name, monitor.running_on,
format_datetime(monitor.first_failure_time()), days, hours,
minutes, seconds, monitor.get_result())
url = "https://{}/publicapi/notify".format(self.api_host)
params = {
'apikey': self.apikey,
'application': self.application,
'description': message,
'event': "%s: %s" % (name, monitor.get_result())
}
else:
# we don't handle other types of message
pass
if url == "":
return
if not self.dry_run:
try:
r = requests.get(url, params=params)
s = r.text
if not s.startswith(
'<?xml version="1.0" encoding="UTF-8"?><nma><success code="200"'
):
self.alerter_logger.error("Unable to send NMA: %s (%s)",
s.split("|")[0],
s.split("|")[1])
self.alerter_logger.error("URL: %s, PARAMS: %s", url,
params)
self.available = False
except Exception:
self.alerter_logger.exception("NMA sending failed")
self.available = False
else:
self.alerter_logger.info("dry_run: would send NMA: %s", url)
return
def __repr__(self):
return "{0}|{1}\n{2} ".format(self.title,
util.format_datetime(self.created),
self.content)
def send_alert(self, name, monitor):
"""Send the email."""
type = self.should_alert(monitor)
(days, hours, minutes, seconds) = monitor.get_downtime()
if monitor.is_remote():
host = " on %s " % monitor.running_on
else:
host = " on host %s" % self.hostname
mail = {'Source': self.from_addr}
mail['Destination'] = {'ToAddresses': [self.to_addr]}
if type == "":
return
elif type == "failure":
message = {
'Subject': {
'Data': "[%s] Monitor %s Failed!" % (self.hostname, name)
}
}
message['Body'] = {
'Text': {
'Data':
"""Monitor %s%s has failed.
Failed at: %s
Downtime: %d+%02d:%02d:%02d
Virtual failure count: %d
Additional info: %s
Description: %s""" %
(name, host, format_datetime(
monitor.first_failure_time()), days, hours,
minutes, seconds, monitor.virtual_fail_count(),
monitor.get_result(), monitor.describe())
}
}
try:
if monitor.recover_info != "":
message['Body']['Text'][
'Data'] += "\nRecovery info: %s" % monitor.recover_info
except AttributeError:
message['Body']['Text'][
'Data'] += "\nNo recovery info available"
elif type == "success":
message = {
'Subject': {
'Data': "[%s] Monitor %s succeeded" % (self.hostname, name)
}
}
message['Body'] = {
'Text': {
'Data':
"Monitor %s%s is back up.\nOriginally failed at: %s\nDowntime: %d+%02d:%02d:%02d\nDescription: %s"
%
(name, host, format_datetime(monitor.first_failure_time()),
days, hours, minutes, seconds, monitor.describe())
}
}
elif type == "catchup":
message = {
'Subject': {
'Data':
"[%s] Monitor %s failed earlier!" % (self.hostname, name)
}
}
message['Body'] = {
'Text': {
'Data':
"Monitor %s%s failed earlier while this alerter was out of hours.\nFailed at: %s\nVirtual failure count: %d\nAdditional info: %s\nDescription: %s"
%
(name, host, format_datetime(monitor.first_failure_time()),
monitor.virtual_fail_count(), monitor.get_result(),
monitor.describe())
}
}
else:
self.alerter_logger.critical("Unknown alert type %s", type)
return
mail['Message'] = message
if not self.dry_run:
try:
client = boto3.client('ses', **self.ses_client_params)
client.send_email(**mail)
except Exception:
self.alerter_logger.exception("couldn't send mail")
self.available = False
else:
self.alerter_logger.info("dry_run: would send email:")
self.alerter_logger.info(" Subject: %s",
message['Subject']['Data'])
self.alerter_logger.info(" Body: %s",
message['Body']['Text']['Data'])
def __repr__(self):
return "{0}|{1}\n{2} ".format(self.title,
util.format_datetime(self.created),
self.content)
def send_alert(self, name, monitor):
"""Send the message."""
type = self.should_alert(monitor)
(days, hours, minutes, seconds) = monitor.get_downtime()
if self.channel is not None:
message_json = {'channel': self.channel}
elif self.username is not None:
message_json = {'username': self.username}
else:
message_json = {}
message_json['attachments'] = [{}]
if type == "":
return
elif type == "failure":
message_json['text'] = "Monitor {} failed!".format(name)
message_json['attachments'][0]['color'] = 'danger'
fields = [
{
'title': 'Failed at',
'value': format_datetime(monitor.first_failure_time()),
'short': True
},
{
'title': 'Downtime',
'value': "{}+{:02d}:{:02d}:{:02d}".format(days, hours, minutes, seconds),
'short': True
},
{
'title': 'Virtual failure count',
'value': monitor.virtual_fail_count(),
'short': True
},
{
'title': 'Host',
'value': self.hostname,
'short': True
},
{
'title': 'Additional info',
'value': monitor.get_result()
},
{
'title': 'Description',
'value': monitor.describe()
}
]
try:
if monitor.recover_info != "":
fields.append({
'title': 'Recovery info',
'value': "Recovery info: %s" % monitor.recover_info
})
message_json['attachments'][0]['color'] = 'warning'
except AttributeError:
pass
message_json['attachments'][0]['fields'] = fields
elif type == "success":
message_json['text'] = "Monitor {} succeeded.".format(name)
fields = [
{
'title': 'Failed at',
'value': format_datetime(monitor.first_failure_time()),
'short': True
},
{
'title': 'Downtime',
'value': "{}+{:02d}:{:02d}:{:02d}".format(days, hours, minutes, seconds),
'short': True
},
{
'title': 'Host',
'value': self.hostname,
'short': True
},
{
'title': 'Description',
'value': monitor.describe()
}
]
message_json['attachments'][0]['color'] = 'good'
message_json['attachments'][0]['fields'] = fields
else:
self.alerter_logger.error("unknown alert type %s", type)
return
if not self.dry_run:
try:
r = requests.post(self.url, json=message_json)
if not r.status_code == 200:
self.alerter_logger.error("POST to slack webhook failed: %s", r)
except Exception:
self.alerter_logger.exception("Failed to post to slack webhook")
self.available = False
else:
self.alerter_logger.info("dry_run: would send slack: %s", message_json.__repr__())
def send_alert(self, name, monitor):
"""Send an SMS alert."""
if not monitor.is_urgent():
return
type = self.should_alert(monitor)
message = ""
url = ""
(days, hours, minutes, seconds) = monitor.get_downtime()
if type == "":
return
elif type == "catchup":
message = "catchup: %s failed on %s at %s (%d+%02d:%02d:%02d)\n%s" % (
name,
monitor.running_on,
format_datetime(monitor.first_failure_time()),
days, hours, minutes, seconds,
monitor.get_result())
if len(message) > 160:
self.alerter_logger.warning("Truncating SMS message to 160 chars.")
message = message[:156] + "..."
url = "https://{}/a1/SMS".format(self.api_host)
auth = (self.username, self.password)
params = {
'from': self.sender,
'to': self.target,
'message': message,
}
elif type == "failure":
message = "%s failed on %s at %s (%d+%02d:%02d:%02d)\n%s" % (
name,
monitor.running_on,
format_datetime(monitor.first_failure_time()),
days, hours, minutes, seconds,
monitor.get_result())
if len(message) > 160:
self.alerter_logger.warning("Truncating SMS message to 160 chars.")
message = message[:156] + "..."
url = "https://{}/a1/SMS".format(self.api_host)
auth = (self.username, self.password)
params = {
'from': self.sender,
'to': self.target,
'message': message,
}
else:
# we don't handle other types of message
pass
if url == "":
return
if not self.dry_run:
try:
response = requests.post(url, data=params, auth=auth)
s = response.json()
if s['status'] not in ('created', 'delivered'):
self.alerter_logger.error("Unable to send SMS: %s", s)
self.available = False
except Exception:
self.alerter_logger.exception("SMS sending failed")
self.available = False
else:
self.alerter_logger.info("dry_run: would send SMS: %s", url)
return
def test_gte(self):
for op in ["gte", ">="]:
f1 = FileRule("mtime", op, format_datetime(time() - 3600), "exclude")
f2 = FileRule("mtime", op, format_datetime(time() + 3600), "exclude")
self.assertEqual(f1.test(self.file_entry), "exclude")
self.assertEqual(f2.test(self.file_entry), None)
def process_batch(self):
"""Save the HTML file."""
ok_count = 0
fail_count = 0
old_count = 0
remote_count = 0
my_host = short_hostname()
try:
temp_file = tempfile.mkstemp()
file_handle = os.fdopen(temp_file[0], "w")
file_name = temp_file[1]
except Exception:
sys.stderr.write("Couldn't create temporary file for HTML output\n")
return
output_ok = StringIO()
output_fail = StringIO()
keys = list(self.batch_data.keys())
keys.sort()
for entry in keys:
if self.batch_data[entry]["age"] > 120:
status = "OLD"
old_count += 1
elif self.batch_data[entry]["status"]:
status = "OK"
ok_count += 1
else:
status = "FAIL"
fail_count += 1
if self.batch_data[entry]["host"] != my_host:
remote_count += 1
try:
monitor_name = entry.split("/")[1]
except Exception:
monitor_name = entry
if status == "FAIL":
output = output_fail
else:
output = output_ok
output.write("<tr class=\"%srow\">" % status.lower())
output.write("""
<td class="monitor_name">%s</td>
<td class="status %s">%s</td>
<td>%s</td>
<td>%s</td>
""" % (
monitor_name,
status.lower(), status,
self.batch_data[entry]["host"],
self.batch_data[entry]["fail_time"],
)
)
if self.batch_data[entry]["fail_count"] == 0:
output.write("<td class=\"vfc\"> </td>")
else:
output.write("<td class=\"vfc\">%s</td>" % self.batch_data[entry]["fail_count"])
try:
output.write("<td>%d+%02d:%02d:%02d</td>" % (self.batch_data[entry]["downtime"][0], self.batch_data[entry]["downtime"][1], self.batch_data[entry]["downtime"][2], self.batch_data[entry]["downtime"][3]))
except Exception:
output.write("<td> </td>")
output.write("<td>%s </td>" % (self.batch_data[entry]["fail_data"]))
if self.batch_data[entry]["failures"] == 0:
output.write("<td></td><td></td>")
else:
output.write("""<td>%s</td>
<td>%s</td>""" % (
self.batch_data[entry]["failures"],
format_datetime(self.batch_data[entry]["last_failure"])
)
)
if self.batch_data[entry]["host"] == my_host:
output.write("<td></td>")
else:
output.write("<td>%d</td>" % self.batch_data[entry]["age"])
output.write("</tr>\n")
count_data = "<div id=\"summary\""
if old_count > 0:
cls = "old"
elif fail_count > 0:
cls = "fail"
else:
cls = "ok"
count_data = count_data + " class=\"%s\">%s" % (cls, cls.upper())
self.count_data = count_data + "<div id=\"details\"><span class=\"ok\">%d OK</span> <span class=\"fail\">%d FAIL</span> <span class=\"old\">%d OLD</span> <span class=\"remote\">%d remote</span></div></div>" % (ok_count, fail_count, old_count, remote_count)
self.status = cls.upper()
with open(os.path.join(self.folder, self.header), "r") as file_input:
file_handle.writelines(self.parse_file(file_input))
file_handle.write(output_fail.getvalue())
file_handle.write(output_ok.getvalue())
with open(os.path.join(self.folder, self.footer), "r") as file_input:
file_handle.writelines(self.parse_file(file_input))
try:
file_handle.flush()
file_handle.close()
os.chmod(file_name, stat.S_IREAD | stat.S_IWRITE | stat.S_IRGRP | stat.S_IROTH)
shutil.move(file_name, os.path.join(self.folder, self.filename))
except Exception:
self.logger_logger.exception("problem closing temporary file for HTML output")
def send_alert(self, name, monitor):
"""Send the email."""
type = self.should_alert(monitor)
(days, hours, minutes, seconds) = monitor.get_downtime()
if monitor.is_remote():
host = " on %s " % monitor.running_on
else:
host = " on host %s" % self.hostname
message = MIMEMultipart()
message['From'] = self.from_addr
message['To'] = self.to_addr
if type == "":
return
elif type == "failure":
message['Subject'] = "[%s] Monitor %s Failed!" % (self.hostname, name)
body = """Monitor %s%s has failed.
Failed at: %s
Downtime: %d+%02d:%02d:%02d
Virtual failure count: %d
Additional info: %s
Description: %s""" % (
name,
host,
format_datetime(monitor.first_failure_time()),
days, hours, minutes, seconds,
monitor.virtual_fail_count(),
monitor.get_result(),
monitor.describe())
try:
if monitor.recover_info != "":
body += "\nRecovery info: %s" % monitor.recover_info
except AttributeError:
body += "\nNo recovery info available"
elif type == "success":
message['Subject'] = "[%s] Monitor %s succeeded" % (self.hostname, name)
body = "Monitor %s%s is back up.\nOriginally failed at: %s\nDowntime: %d+%02d:%02d:%02d\nDescription: %s" % (name, host, format_datetime(monitor.first_failure_time()), days, hours, minutes, seconds, monitor.describe())
elif type == "catchup":
message['Subject'] = "[%s] Monitor %s failed earlier!" % (self.hostname, name)
body = "Monitor %s%s failed earlier while this alerter was out of hours.\nFailed at: %s\nVirtual failure count: %d\nAdditional info: %s\nDescription: %s" % (name, host, format_datetime(monitor.first_failure_time()), monitor.virtual_fail_count(), monitor.get_result(), monitor.describe())
else:
self.alerter_logger.critical("unknown alert type %s", type)
return
message.attach(MIMEText(body, 'plain'))
if not self.dry_run:
try:
if self.ssl is None or self.ssl == 'starttls':
server = smtplib.SMTP(self.mail_host, self.mail_port)
elif self.ssl == 'yes':
server = smtplib.SMTP_SSL(self.mail_host, self.mail_port)
if self.ssl == 'starttls':
server.starttls()
if self.username is not None:
server.login(self.username, self.password)
server.sendmail(self.from_addr, self.to_addr.split(';'), message.as_string())
server.quit()
except Exception:
self.alerter_logger.exception("couldn't send mail")
self.available = False
else:
self.alerter_logger.info("dry_run: would send email: %s", message.as_string())
def test_Format(self):
self.assertEqual(util.format_datetime(None), "")
self.assertEqual(util.format_datetime("a string"), "a string")
self.assertEqual(
util.format_datetime(datetime.datetime(2018, 5, 8, 13, 37, 0)),
"2018-05-08 13:37:00")
def process_batch(self):
"""Save the HTML file."""
ok_count = 0
fail_count = 0
old_count = 0
remote_count = 0
my_host = short_hostname
try:
temp_file = tempfile.mkstemp()
file_handle = os.fdopen(temp_file[0], "w")
file_name = temp_file[1]
except Exception:
sys.stderr.write("Couldn't create temporary file for HTML output\n")
return
output_ok = StringIO()
output_fail = StringIO()
keys = list(self.batch_data.keys())
keys.sort()
for entry in keys:
if self.batch_data[entry]["age"] > 120:
status = "OLD"
old_count += 1
elif self.batch_data[entry]["status"]:
status = "OK"
ok_count += 1
else:
status = "FAIL"
fail_count += 1
if self.batch_data[entry]["host"] != my_host:
remote_count += 1
try:
monitor_name = entry.split("/")[1]
except Exception:
monitor_name = entry
if status == "FAIL":
output = output_fail
else:
output = output_ok
output.write("<tr class=\"%srow\">" % status.lower())
output.write("""
<td class="monitor_name">%s</td>
<td class="status %s">%s</td>
<td>%s</td>
<td>%s</td>
""" % (
monitor_name,
status.lower(), status,
self.batch_data[entry]["host"],
self.batch_data[entry]["fail_time"],
)
)
if self.batch_data[entry]["fail_count"] == 0:
output.write("<td class=\"vfc\"> </td>")
else:
output.write("<td class=\"vfc\">%s</td>" % self.batch_data[entry]["fail_count"])
try:
output.write("<td>%d+%02d:%02d:%02d</td>" % (self.batch_data[entry]["downtime"][0], self.batch_data[entry]["downtime"][1], self.batch_data[entry]["downtime"][2], self.batch_data[entry]["downtime"][3]))
except Exception:
output.write("<td> </td>")
output.write("<td>%s </td>" % (self.batch_data[entry]["fail_data"]))
if self.batch_data[entry]["failures"] == 0:
output.write("<td></td><td></td>")
else:
output.write("""<td>%s</td>
<td>%s</td>""" % (
self.batch_data[entry]["failures"],
format_datetime(self.batch_data[entry]["last_failure"])
)
)
if self.batch_data[entry]["host"] == my_host:
output.write("<td></td>")
else:
output.write("<td>%d</td>" % self.batch_data[entry]["age"])
output.write("</tr>\n")
count_data = "<div id=\"summary\""
if old_count > 0:
cls = "old"
elif fail_count > 0:
cls = "fail"
else:
cls = "ok"
count_data = count_data + " class=\"%s\">%s" % (cls, cls.upper())
self.count_data = count_data + "<div id=\"details\"><span class=\"ok\">%d OK</span> <span class=\"fail\">%d FAIL</span> <span class=\"old\">%d OLD</span> <span class=\"remote\">%d remote</span></div></div>" % (ok_count, fail_count, old_count, remote_count)
self.status = cls.upper()
with open(os.path.join(self.folder, self.header), "r") as file_input:
file_handle.writelines(self.parse_file(file_input))
file_handle.write(output_fail.getvalue())
file_handle.write(output_ok.getvalue())
with open(os.path.join(self.folder, self.footer), "r") as file_input:
file_handle.writelines(self.parse_file(file_input))
try:
file_handle.flush()
file_handle.close()
os.chmod(file_name, stat.S_IREAD | stat.S_IWRITE | stat.S_IRGRP | stat.S_IROTH)
shutil.move(file_name, os.path.join(self.folder, self.filename))
except Exception as e:
self.logger_logger.exception("problem closing temporary file for HTML output")
def parse_bill(congress, bill_type, bill_number):
"""Downloads and parses THOMAS bill status and summary files."""
# map our bill type code to the THOMAS bill type code (namely, hr is confused with H.R.
# so make it hres).
bill_type2 = bill_type
if bill_type2 == "hr": bill_type2 = "hres"
# Start with the All Actions page, from which we'll also grab basic metadata.
url = "http://thomas.loc.gov/cgi-bin/bdquery/z?d%03d:%s%s:@@@X" % (congress, bill_type2, bill_number)
content, mtime = download(url)
if not content:
raise Exception("Failed to download bill status page: " + url)
root = etree.Element("bill")
root.set("session", str(congress))
root.set("type", bill_type)
root.set("number", str(bill_number))
root.set("updated", format_datetime(mtime))
sponsor = None
introduced_date = None
title = None
actions = []
action_indentation = 0
state_name, state_date = None, None
for line in content.split("\n"):
# Match Sponsor line, which can be either No Sponsor or a name/state/district.
# Parse the name later, after we find the introduced date.
m = re.search(r"<b>Sponsor: </b>(No Sponsor|<a [^>]+>(.*)</a>\s+\[((\w\w)(-(\d+))?)\])", line, re.I)
if m != None:
if m.group(1) == "No Sponsor":
sponsor = (None, None, None, None)
else:
name = m.group(2)
if name.startswith("Sen "):
senrep = "sen"
name = name[4:]
elif name.startswith("Rep "):
senrep = "rep"
name = name[4:]
else:
raise Exception("Invalid name: missing title: " + name)
sponsor = (name, senrep, m.group(4), m.group(6)) # name, type, state, district
# Match the line giving the date of introduction.
m = re.search(r"\(introduced ([\d\/]+)\)", line, re.I)
if m != None:
introduced_date = datetime.datetime.strptime(m.group(1), "%m/%d/%Y").date()
state_name, state_date = ("INTRODUCED", introduced_date)
# Match the Title line, which we use 1) to check if this is a bill number reserved for the speaker,
# 2) for determining whether this is a proposal for a constitutional amendment for parsing
# vote status, and 3) as a backup in case the THOMAS Titles page cannot be parsed.
m = re.search(r"<B>(Latest )?Title:</B> (.+)", line, re.I)
if m != None:
title = m.group(2)
# Match action lines.
m = re.search(r"<dt><strong>([\d/ :apm]+):</strong><dd>(.+)", line, re.I)
if m != None:
# Indentation indicates committee action.
if line.startswith("<dl>"): action_indentation += 1
if line.startswith("</dl>"): action_indentation -= 1
text = re.sub(r"</?[Aa]( \S.*?)?>", "", m.group(2))
# The date can be either a date or a date and time.
try:
action_date = datetime.datetime.strptime(m.group(1), "%m/%d/%Y %I:%M%p")
except:
#print repr(m.group(1))
try:
action_date = datetime.datetime.strptime(m.group(1), "%m/%d/%Y").date()
except:
raise ValueError("Could not parse date: " + m.group(1))
# references are given in parentheses at the end
considerations = []
m = re.search("\s+\((.*)\)\s*$", text)
if m:
text = text[0:m.start()] + text[m.end():]
for con in m.group(1).split("; "):
if ": " not in con:
considerations.append( ("", con) )
else:
considerations.append( con.split(": ") )
# Parse the actual action line.
attrs = parse_bill_action(text, bill_type, state_name, title)
if "state" in attrs:
state_name, state_date = attrs["state"], action_date
actions.append((action_date, action_indentation, text, attrs, considerations))
if introduced_date == None:
raise Exception("No introduced date.")
if sponsor == None:
raise Exception("No sponsor line.")
elif sponsor[0] == None:
sponsor = 0
else:
sponsor = parse_name(sponsor[0], introduced_date, nameformat="lastfirst", role_type=sponsor[1], state=sponsor[2], district=sponsor[3])
if "Reserved for the" in title:
raise Exception("Skipping bill " + title.lower())
# Start building the XML
state = etree.Element("state")
state.set("datetime", format_datetime(state_date))
state.text = state_name
root.append(state)
intronode = etree.Element("introduced")
intronode.set("datetime", introduced_date.isoformat())
root.append(intronode)
if sponsor != 0:
sponsornode = etree.Element("sponsor")
sponsornode.set("id", str(sponsor))
root.append(sponsornode)
# Download and parse the cosponsors page.
cosponsors = etree.Element("cosponsors")
root.append(cosponsors)
url = url.replace("@@@X", "@@@P")
content, mtime = download(url)
if not content:
raise Exception("Failed to download cosponsors page: " + url)
content = re.sub(r"(\[[A-Z\d\-]+\])\n( - \d\d?\/)", lambda m : m.group(1) + m.group(2), content) # bring cosponsorship date onto previous line
content = re.sub(r"</br>", "\n", content)
for line in content.split("\n"):
m = re.search(r"(<br ?/>)?<a href=[^>]+>(Rep|Sen) (.+)</a> \[([A-Z\d\-]+)\] - (\d\d?/\d\d?/\d\d\d\d)(\(withdrawn - (\d\d?/\d\d?/\d\d\d\d)\))?", line, re.I)
if m:
title, name, state_district, join_date, withdrawn_date = m.group(2), m.group(3), m.group(4), m.group(5), m.group(7)
join_date = datetime.datetime.strptime(join_date, "%m/%d/%Y").date()
if withdrawn_date != None:
withdrawn_date = datetime.datetime.strptime(withdrawn_date, "%m/%d/%Y").date()
name = name.replace("Colordao", "Colorado") # typo
if not "-" in state_district:
state = state_district
district = None
else:
state, district = state_district.split("-")
person = parse_name(name, join_date, nameformat="lastfirst", role_type=title.lower(), state=state, district=district)
csp = etree.Element("cosponsor")
csp.set("id", str(person))
csp.set("joined", join_date.isoformat())
if withdrawn_date: csp.set("withdrawn", withdrawn_date.isoformat())
cosponsors.append(csp)
# Download and parse the titles page.
titles = etree.Element("titles")
root.append(titles)
url = url.replace("@@@P", "@@@T")
content, mtime = download(url)
if not content:
raise Exception("Failed to download titles page: " + url)
content = re.sub(r"(<I>)?(<br/>|<p>)", lambda m : "\n" + ("" if not m.group(1) else m.group(1)), content)
title_type, title_as = None, None
for line in content.split("\n"):
if line == "</ul>":
break
m = re.search(r"<li>(.*) title(\(s\))?( as ([\w ]*))?:", line, re.I)
if m:
title_type, title_as = m.group(1).lower(), m.group(4).lower()
elif title_type != None and line.strip() != "":
partial = False
if line.startswith("<I>"):
partial = True
line = line[3:]
line = line.replace("<I>", "")
line = line.replace("</I>", "")
line = line.replace(" (identified by CRS)", "")
t = etree.Element("title")
t.set("type", title_type)
t.set("as", title_as)
t.set("partial", "yes" if partial else "no")
t.text = line.strip()
titles.append(t)
if len(titles) == 0:
# Sometimes titles aren't available.
t = etree.Element("title")
t.set("type", "official")
t.set("as", "introduced")
t.set("partial", "no")
t.text = title
titles.append(t)
# Download and parse the committees page.
committees = etree.Element("committees")
root.append(committees)
url = url.replace("@@@T", "@@@C")
content, mtime = download(url)
if not content:
raise Exception("Failed to download committees page: " + url)
last_committee = None
for line in content.split("\n"):
m = re.search(r'<a href="/cgi-bin/bdquery(tr)?/R\?[^"]+">(.*)</a>\s*</td><td width="65\%">(.+)</td></tr>', line, re.I)
if m:
committee = m.group(2)
activity = m.group(3)
committee = re.sub(r"\s+", " ", committee).strip()
if not committee.startswith("Subcommittee on "):
last_committee = committee
committee = find_committee(committee, None, congress)
else:
committee = committee[len("Subcommittee on "):]
committee = find_committee(last_committee, committee, congress)
cx = etree.Element("committee")
cx.set("code", committee)
cx.set("activity", activity)
committees.append(cx)
# Download and parse related bills page.
related_bill_type_map = { }
for a, b in thomas_bill_type_codes:
related_bill_type_map[a] = b
related_bill_relationship_map = {
"Identical bill identified by CRS": "identical",
"Related bill identified by CRS": "related",
"Related bill as identified by the House Clerk's office": "related",
"passed in House in lieu of this bill": "supersedes",
"passed in Senate in lieu of this bill": "supersedes",
}
related_bills = etree.Element("relatedbills")
root.append(related_bills)
url = url.replace("@@@C", "@@@K")
content, mtime = download(url)
if not content:
raise Exception("Failed to download related bills page: " + url)
for line in content.split("\n"):
m = re.search(r'<a href="/cgi-bin/bdquery(tr)?/z\?d(\d\d\d):(\w+)(\d\d\d\d\d):">.*</a></td><td>(.*)</td></tr>', line, re.I)
if m:
related_bill_congress = int(m.group(2))
related_bill_type = related_bill_type_map[m.group(3)]
related_bill_number = int(m.group(4))
if re.search("Rule related to", m.group(5)):
related_bill_relationship = "rule"
else:
related_bill_relationship = related_bill_relationship_map[m.group(5)]
rb = etree.Element("bill")
rb.set("relation", related_bill_relationship)
rb.set("session", str(related_bill_congress))
rb.set("type", related_bill_type)
rb.set("number", str(related_bill_number))
related_bills.append(rb)
# Download and parse CRS subject terms page.
subjects = etree.Element("subjects")
root.append(subjects)
url = url.replace("@@@K", "@@@J")
content, mtime = download(url)
if not content:
raise Exception("Failed to download subject terms page: " + url)
for line in content.split("\n"):
m = re.search(r'<a href="/cgi-bin/bdquery/\?.*@FIELD\(FLD001.*\)">(.*)</a> ', line, re.I)
if m:
term = m.group(1)
term = re.sub(r"\s+", " ", term).strip()
s = etree.Element("term")
s.set("name", term)
subjects.append(s)
# Download and parse amendments page.
amendments = etree.Element("amendments")
root.append(amendments)
url = url.replace("@@@J", "@@@A")
content, mtime = download(url)
if not content:
raise Exception("Failed to download amendments page: " + url)
for m in re.finditer(r'<a href="/cgi-bin/bdquery/z\?d\d+:([HS])([ZP])(\d+):">[HS]\.AMDT\.\d+</a>', content, re.I):
amendment_chamber = m.group(1).lower()
amendment_number = int(m.group(3))
a = etree.Element("amendment")
a.set("number", amendment_chamber + str(amendment_number))
amendments.append(a)
# Put the actions in here.
actionsnode = etree.Element("actions")
root.append(actionsnode)
for adate, aindent, text, attrs, considerations in actions:
nodename = "action"
if "nodename" in attrs:
nodename = attrs["nodename"]
del attrs["nodename"]
node = etree.Element(nodename)
actionsnode.append(node)
node.set("datetime", format_datetime(adate))
for k, v in sorted(attrs.items()):
if v == None:
continue
node.set(k, v)
n = etree.Element("text")
n.text = text
node.append(n)
for c in considerations:
n = etree.Element("reference")
n.set("label", c[0])
n.set("ref", c[1])
node.append(n)
# Download the CRS summary text.
url = url.replace("@@@A", "@@@D&summ2=m&")
content, mtime = download(url)
if not content:
raise Exception("Failed to download summary page: " + url)
mode = 0
summary = ""
for line in content.split("\n"):
if mode == 1:
if "<hr" in line:
mode = 0
elif "THOMAS Home" in line or "id=\"footer\"" in line:
break
else:
line = re.sub(r"<a.*?>(.*?)</a>", lambda m : m.group(1), line, re.I)
summary += line + "\n"
elif "SUMMARY AS OF" in line:
mode = 1
summary = re.sub(r"\(There (is|are) \d+ other summar(y|ies)\)", "", summary, re.I)
root.append(fragment_fromstring(summary, create_parent="summary"))
try:
os.makedirs("../data/us/%d/bills" % congress)
except:
pass
return etree.tostring(root, pretty_print=True)
def __repr__(self):
return "<{0:20s} on {1:10s} at {2} - running_cores: {3:8d}; not_running_cores: {4:8d}".format(
self.username, self.clustername,
util.format_datetime(self.created),
self.runningcores, self.notrunningcores)
def __repr__(self):
return "<{0} joined at {1})>".format(
self.email,
util.format_datetime(self.created))
def _get_datestring(self):
if self.dateformat == 'iso8601':
return format_datetime(datetime.datetime.now())
return str(int(time.time()))
def send_alert(self, name, monitor):
"""Send an SMS alert."""
if not monitor.is_urgent():
return
type_ = self.should_alert(monitor)
message = ""
url = ""
(days, hours, minutes, seconds) = monitor.get_downtime()
if type_ == "":
return
elif type_ == "catchup":
message = "catchup: %s failed on %s at %s (%d+%02d:%02d:%02d)\n%s" % (
name,
monitor.running_on,
format_datetime(monitor.first_failure_time()),
days, hours, minutes, seconds,
monitor.get_result())
if len(message) > 160:
self.alerter_logger.warning("Truncating SMS message to 160 chars.")
message = message[:156] + "..."
url = "https://{}/eapi/submission/send_sms/2/2.0".format(self.api_host)
params = {
'username': self.username,
'password': self.password,
'message': message,
'msisdn': self.target,
'sender': self.sender,
'repliable': '0'
}
elif type_ == "failure":
message = "%s failed on %s at %s (%d+%02d:%02d:%02d)\n%s" % (
name,
monitor.running_on,
format_datetime(monitor.first_failure_time()),
days, hours, minutes, seconds,
monitor.get_result())
if len(message) > 160:
self.alerter_logger.warning("Truncating SMS message to 160 chars.")
message = message[:156] + "..."
url = "https://{}/eapi/submission/send_sms/2/2.0".format(self.api_host)
params = {
'username': self.username,
'password': self.password,
'message': message,
'msisdn': self.target,
'sender': self.sender,
'repliable': '0'
}
else:
# we don't handle other types of message
pass
if url == "":
return
if not self.dry_run:
try:
r = requests.get(url, params=params)
s = r.text
if not s.startswith("0"):
self.alerter_logger.error("Unable to send SMS: %s (%s)", s.split("|")[0], s.split("|")[1])
self.available = False
except Exception:
self.alerter_logger.exception("SMS sending failed")
self.available = False
else:
self.alerter_logger.info("dry_run: would send SMS: %s", url)
return
def send_alert(self, name, monitor):
"""Send an SMS alert."""
if not monitor.is_urgent():
return
type_ = self.should_alert(monitor)
message = ""
url = ""
(days, hours, minutes, seconds) = monitor.get_downtime()
if type_ == "":
return
elif type_ == "catchup":
message = "catchup: %s failed on %s at %s (%d+%02d:%02d:%02d)\n%s" % (
name, monitor.running_on,
format_datetime(monitor.first_failure_time()), days, hours,
minutes, seconds, monitor.get_result())
if len(message) > 160:
self.alerter_logger.warning(
"Truncating SMS message to 160 chars.")
message = message[:156] + "..."
url = "https://{}/eapi/submission/send_sms/2/2.0".format(
self.api_host)
params = {
'username': self.username,
'password': self.password,
'message': message,
'msisdn': self.target,
'sender': self.sender,
'repliable': '0'
}
elif type_ == "failure":
message = "%s failed on %s at %s (%d+%02d:%02d:%02d)\n%s" % (
name, monitor.running_on,
format_datetime(monitor.first_failure_time()), days, hours,
minutes, seconds, monitor.get_result())
if len(message) > 160:
self.alerter_logger.warning(
"Truncating SMS message to 160 chars.")
message = message[:156] + "..."
url = "https://{}/eapi/submission/send_sms/2/2.0".format(
self.api_host)
params = {
'username': self.username,
'password': self.password,
'message': message,
'msisdn': self.target,
'sender': self.sender,
'repliable': '0'
}
else:
# we don't handle other types of message
pass
if url == "":
return
if not self.dry_run:
try:
r = requests.get(url, params=params)
s = r.text
if not s.startswith("0"):
self.alerter_logger.error("Unable to send SMS: %s (%s)",
s.split("|")[0],
s.split("|")[1])
self.available = False
except Exception as e:
self.alerter_logger.exception("SMS sending failed")
self.available = False
else:
self.alerter_logger.info("dry_run: would send SMS: %s", url)
return
def _serialize(self, value, attr, obj):
return util.format_datetime(value)
