def save_result2(self, name, monitor): if self.only_failures and monitor.virtual_fail_count() == 0: return dateformat = 'timestamp' if self.dateformat == 'iso8601': dateformat = 'iso8601' if dateformat == 'timestamp': datestring = str(int(time.time())) elif dateformat == 'iso8601': datestring = format_datetime(datetime.datetime.now()) try: if monitor.virtual_fail_count() > 0: self.file_handle.write("%s %s: failed since %s; VFC=%d (%s) (%0.3fs)" % ( datestring, name, format_datetime(monitor.first_failure_time()), monitor.virtual_fail_count(), monitor.get_result(), monitor.last_run_duration )) else: self.file_handle.write("%s %s: ok (%0.3fs)" % ( datestring, name, monitor.last_run_duration )) self.file_handle.write("\n") if not self.buffered: self.file_handle.flush() except Exception as e: self.logger_logger.exception("Error writing to logfile %s", self.filename)
def send_alert(self, name, monitor): """Build up the content for the push notification.""" type = self.should_alert(monitor) (days, hours, minutes, seconds) = monitor.get_downtime() if monitor.is_remote(): host = " on %s " % monitor.running_on else: host = " on host %s" % self.hostname subject = "" body = "" if type == "": return elif type == "failure": subject = "[%s] Monitor %s Failed!" % (self.hostname, name) body = """Monitor %s%s has failed.\n Failed at: %s Downtime: %d+%02d:%02d:%02d Virtual failure count: %d Additional info: %s Description: %s""" % ( name, host, format_datetime(monitor.first_failure_time()), days, hours, minutes, seconds, monitor.virtual_fail_count(), monitor.get_result(), monitor.describe()) try: if monitor.recover_info != "": body += "\nRecovery info: %s" % monitor.recover_info except AttributeError: body += "\nNo recovery info available" elif type == "success": subject = "[%s] Monitor %s succeeded" % (self.hostname, name) body = "Monitor %s%s is back up.\nOriginally failed at: %s\nDowntime: %d+%02d:%02d:%02d\nDescription: %s" % ( name, host, format_datetime(monitor.first_failure_time()), days, hours, minutes, seconds, monitor.describe()) elif type == "catchup": subject = "[%s] Monitor %s failed earlier!" % (self.hostname, name) body = "Monitor %s%s failed earlier while this alerter was out of hours.\nFailed at: %s\nVirtual failure count: %d\nAdditional info: %s\nDescription: %s" % ( name, host, format_datetime(monitor.first_failure_time()), monitor.virtual_fail_count(), monitor.get_result(), monitor.describe()) else: self.alerter_logger.error("Unknown alert type %s", type) return if not self.dry_run: try: self.send_pushbullet_notification(subject, body) except Exception: self.alerter_logger.exception( "Couldn't send push notification") self.available = False else: self.alerter_logger.info( "dry_run: would send push notification: %s" % body)
def test_Format(self): self.assertEqual(util.format_datetime(None), "") self.assertEqual(util.format_datetime("a string"), "a string") self.assertEqual( util.format_datetime(datetime.datetime(2018, 5, 8, 13, 37, 0)), "2018-05-08 13:37:00" )
def test_lte(self): self.file_entry.ctime = int(self.file_entry.ctime) f1 = FileRule("ctime", "lte", format_datetime(self.file_entry.ctime), "include") f2 = FileRule("ctime", "lte", format_datetime(time() - 3600), "include") f3 = FileRule("ctime", "eq", format_datetime(self.file_entry.ctime), "include") self.assertEqual(f1.test(self.file_entry), "include") self.assertEqual(f2.test(self.file_entry), None) self.assertEqual(f3.test(self.file_entry), "include")
def test_gte(self): for op in ['gte', '>=']: f1 = FileRule('mtime', op, format_datetime(time() - 3600), 'exclude') f2 = FileRule('mtime', op, format_datetime(time() + 3600), 'exclude') self.assertEqual(f1.test(self.file_entry), 'exclude') self.assertEqual(f2.test(self.file_entry), None)
def test_gte(self): for op in ['gte', '>=']: f1 = FileRule('mtime', op, format_datetime(time()-3600), 'exclude') f2 = FileRule('mtime', op, format_datetime(time()+3600), 'exclude') self.assertEqual(f1.test(self.file_entry), 'exclude') self.assertEqual(f2.test(self.file_entry), None)
def test_lte(self): self.file_entry.ctime = int(self.file_entry.ctime) f1 = FileRule('ctime', 'lte', format_datetime(self.file_entry.ctime), 'include') f2 = FileRule('ctime', 'lte', format_datetime(time()-3600), 'include') f3 = FileRule('ctime', 'eq', format_datetime(self.file_entry.ctime), 'include') self.assertEqual(f1.test(self.file_entry), 'include') self.assertEqual(f2.test(self.file_entry), None) self.assertEqual(f3.test(self.file_entry), 'include')
def test_lte(self): self.file_entry.ctime = int(self.file_entry.ctime) f1 = FileRule('ctime', 'lte', format_datetime(self.file_entry.ctime), 'include') f2 = FileRule('ctime', 'lte', format_datetime(time() - 3600), 'include') f3 = FileRule('ctime', 'eq', format_datetime(self.file_entry.ctime), 'include') self.assertEqual(f1.test(self.file_entry), 'include') self.assertEqual(f2.test(self.file_entry), None) self.assertEqual(f3.test(self.file_entry), 'include')
def start_end(tt): # the code is bad!! ugly keys = ['dateTime', 'date'] if tt: for key in keys: if key in tt: if key == keys[0]: return util.format_datetime(iso8601.parse_date(tt[key])) elif key ==keys[1]: return util.format_datetime(datetime.datetime.strptime('2012-07-07', '%Y-%m-%d')) else: return None
def send_alert(self, name, monitor): """Build up the content for the push notification.""" type = self.should_alert(monitor) (days, hours, minutes, seconds) = monitor.get_downtime() if monitor.is_remote(): host = " on %s " % monitor.running_on else: host = " on host %s" % self.hostname body = "" if type == "": return elif type == "failure": body = """Monitor %s DOWN Failed at: %s Downtime: %d+%02d:%02d:%02d Description: %s""" % (name, format_datetime(monitor.first_failure_time()), days, hours, minutes, seconds, monitor.describe()) try: if monitor.recover_info != "": body += "\nRecovery info: %s" % monitor.recover_info except AttributeError: body += "\nNo recovery info available" elif type == "success": body = """Monitor %s UP Originally failed at: %s Downtime: %d+%02d:%02d:%02d Description: %s""" % (name, format_datetime(monitor.first_failure_time()), days, hours, minutes, seconds, monitor.describe()) elif type == "catchup": body = "Monitor %s%s failed earlier while this alerter was out of hours.\nFailed at: %s\nDescription: %s" % ( name, host, format_datetime( monitor.first_failure_time()), monitor.describe()) else: self.alerter_logger.error("Unknown alert type %s", type) return if not self.dry_run: try: self.send_telegram_notification(body) except Exception: self.alerter_logger.exception( "Couldn't send push notification") self.available = False else: self.alerter_logger.info( "dry_run: would send push notification: %s" % body)
def start_end(tt): # the code is bad!! ugly keys = ['dateTime', 'date'] if tt: for key in keys: if key in tt: if key == keys[0]: return util.format_datetime(iso8601.parse_date(tt[key])) elif key == keys[1]: return util.format_datetime( datetime.datetime.strptime('2012-07-07', '%Y-%m-%d')) else: return None
def save_result2(self, name, monitor): if self.only_failures and monitor.virtual_fail_count() == 0: return try: if monitor.virtual_fail_count() > 0: self.file_handle.write("%s %s: failed since %s; VFC=%d (%s) (%0.3fs)" % ( self._get_datestring(), name, format_datetime(monitor.first_failure_time()), monitor.virtual_fail_count(), monitor.get_result(), monitor.last_run_duration )) else: self.file_handle.write("%s %s: ok (%0.3fs)" % ( self._get_datestring(), name, monitor.last_run_duration )) self.file_handle.write("\n") if not self.buffered: self.file_handle.flush() except Exception: self.logger_logger.exception("Error writing to logfile %s", self.filename)
def test(data_in, fmt, data_exp): data_out = util.format_datetime(data_in, fmt) if data_out == data_exp: ret = 'OK' else: ret = 'NG' print('[' + ret + '] IN=' + data_in + ' ' + 'EXP=' + data_exp + ' OUT=' + data_out)
def get_push_list(self): import const from util import format_datetime sep = self.get_list_separator_line() res = "push message nad ip address:port status program alc program downloaded response\r\n" res += sep for controller in globals.controllers.get_sorted(): if controller.config.created_from_push: file_transfer_timestamp = controller.alloc.get_file_transfer_timestamp() if file_transfer_timestamp != None: file_transfer_timestamp = file_transfer_timestamp.strftime(const.Timeformat) else: file_transfer_timestamp = "unknown " try: response_time = "%d" % controller.sys_status.get_sys_response_time() except: response_time = controller.sys_status.get_sys_response_time() res += "%s %s %s%s %s %s %s %s\r\n" % \ ( format_datetime(controller.sys_status.last_push_timestamp), repr(controller.config.nad).ljust(5), ("%s:%s" % (controller.config.ip, repr(controller.config.port))).ljust(22), controller.sys_status.get_sys_plc_status().ljust(7), controller.sys_status.get_sys_plc_program_status().ljust(7), controller.sys_status.get_sys_alc_file_status().ljust(7), file_transfer_timestamp, response_time.ljust(8), ) return res + sep
def _get_show_artist_time(self): return { 'artist_id': self.artist_id, 'artist_name': self.artist.name, 'artist_image_link': self.artist.image_link, 'start_time': format_datetime(str(self.start_time), format='full') }
def _get_show_venue_time(self): return { 'venue_id': self.venue_id, 'venue_name': self.venue.name, 'venue_image_link': self.venue.image_link, 'start_time': format_datetime(str(self.start_time), format='full') }
def gen_report(self, rcu, qcu, usermap, created=None): # first, title of the report title = "{0}|{1}|{2}".format(self.clustername, self.quota, self.cores_per_node) # second, created datetime created = created if created else datetime.datetime.now() report_content = [] # datetime, an addtional "\n" is just for making the reportprettier report_content.append("{0}\n".format(util.format_datetime(created))) if not rcu and not qcu: report_content.append("data not available at the moment") else: rcu, qcu = util.prune(rcu), util.prune(qcu) total_usage = {} for realname in set(usermap.values()): total_usage[realname] = sum( dd.get(realname, 0) for dd in [rcu, qcu]) total_usage = util.prune(total_usage) # from this step on, basically it's about print data from 3 dicts # in a pretty way: rcu, qcu, total_usage # 1. print headers report_content.append("{0:13s} {1:8s} {2:8s} {3:8s}".format( 'USERNAME', 'Running', 'NotRunning', 'TOTAL')) # 2. sort the order of key by total_usage sorted_keys = reversed(sorted(total_usage, key=total_usage.get)) if not len(total_usage) == 0: # not the usage of everyone is zero report_content.append('=' * 44) # 3. print the table for k in sorted_keys: # full name is too long, so last name is used since # firstname is confusing name = k.split()[0] report_content.append( "{0:13s} {1:<8d} {2:<8d} {3:<8d}".format( name, rcu.get(k, 0), qcu.get(k, 0), total_usage.get(k, 0))) # 4. print the footer sum report_content.append('=' * 44) report_content.append("{0:13s} {1:<8d} {2:<8d} {3:<8d}".format( 'SUM', sum(rcu.values()), sum(qcu.values()), sum(total_usage.values()))) report_content.append('=' * 44) # 5. join the final work report_content = '\n'.join(report_content) # 6. since it's displayed on line, need such replacements report_content = report_content.replace("\n", "<br>").replace(" ", " ") return Report(self, report_content, created)
def send_alert(self, name, monitor): """Build up the content for the push notification.""" type = self.should_alert(monitor) (days, hours, minutes, seconds) = monitor.get_downtime() if monitor.is_remote(): host = " on %s " % monitor.running_on else: host = " on host %s" % self.hostname subject = "" body = "" if type == "": return elif type == "failure": subject = "[%s] Monitor %s Failed!" % (self.hostname, name) body = """Monitor %s%s has failed.\n Failed at: %s Downtime: %d+%02d:%02d:%02d Virtual failure count: %d Additional info: %s Description: %s""" % ( name, host, format_datetime(monitor.first_failure_time()), days, hours, minutes, seconds, monitor.virtual_fail_count(), monitor.get_result(), monitor.describe()) try: if monitor.recover_info != "": body += "\nRecovery info: %s" % monitor.recover_info except AttributeError: body += "\nNo recovery info available" elif type == "success": subject = "[%s] Monitor %s succeeded" % (self.hostname, name) body = "Monitor %s%s is back up.\nOriginally failed at: %s\nDowntime: %d+%02d:%02d:%02d\nDescription: %s" % (name, host, format_datetime(monitor.first_failure_time()), days, hours, minutes, seconds, monitor.describe()) elif type == "catchup": subject = "[%s] Monitor %s failed earlier!" % (self.from_addr, self.to_addr, self.hostname, name) body = "Monitor %s%s failed earlier while this alerter was out of hours.\nFailed at: %s\nVirtual failure count: %d\nAdditional info: %s\nDescription: %s" % (name, host, format_datetime(monitor.first_failure_time()), monitor.virtual_fail_count(), monitor.get_result(), monitor.describe()) else: self.alerter_logger.error("Unknown alert type %s", type) return if not self.dry_run: try: self.send_pushover_notification(subject, body) except Exception: self.alerter_logger.exception("Couldn't send push notification") self.available = False else: self.alerter_logger.info("dry_run: would send push notification: %s", body)
def parse_file(self, file_handle): lines = [] for line in file_handle: line = line.replace("_NOW_", format_datetime(datetime.datetime.now())) line = line.replace("_HOST_", socket.gethostname()) line = line.replace("_COUNTS_", self.count_data) line = line.replace("_TIMESTAMP_", str(int(time.time()))) line = line.replace("_STATUS_", self.status) lines.append(line) return lines
def gen_report(self, rcu, qcu, usermap, created=None): # first, title of the report title = "{0}|{1}|{2}".format(self.clustername, self.quota, self.cores_per_node) # second, created datetime created = created if created else datetime.datetime.now() report_content = [] # datetime, an addtional "\n" is just for making the reportprettier report_content.append("{0}\n".format(util.format_datetime(created))) if not rcu and not qcu: report_content.append("data not available at the moment") else: rcu, qcu = util.prune(rcu), util.prune(qcu) total_usage = {} for realname in set(usermap.values()): total_usage[realname] = sum(dd.get(realname, 0) for dd in [rcu, qcu]) total_usage = util.prune(total_usage) # from this step on, basically it's about print data from 3 dicts # in a pretty way: rcu, qcu, total_usage # 1. print headers report_content.append("{0:13s} {1:8s} {2:8s} {3:8s}".format( 'USERNAME', 'Running', 'NotRunning', 'TOTAL')) # 2. sort the order of key by total_usage sorted_keys = reversed(sorted(total_usage, key=total_usage.get)) if not len(total_usage) == 0: # not the usage of everyone is zero report_content.append('=' * 44) # 3. print the table for k in sorted_keys: # full name is too long, so last name is used since # firstname is confusing name = k.split()[0] report_content.append("{0:13s} {1:<8d} {2:<8d} {3:<8d}".format( name, rcu.get(k, 0), qcu.get(k, 0), total_usage.get(k, 0))) # 4. print the footer sum report_content.append('=' * 44) report_content.append("{0:13s} {1:<8d} {2:<8d} {3:<8d}".format( 'SUM', sum(rcu.values()), sum(qcu.values()), sum(total_usage.values()))) report_content.append('=' * 44) # 5. join the final work report_content = '\n'.join(report_content) # 6. since it's displayed on line, need such replacements report_content = report_content.replace("\n", "<br>").replace(" ", " ") return Report(self, report_content, created)
def process_batch(self): payload = MonitorJsonPayload() payload.generated = format_datetime(datetime.datetime.now()) payload.monitors = self.batch_data with open(self.filename, 'w') as outfile: json.dump(payload, outfile, indent=4, separators=(',', ':'), ensure_ascii=False, cls=PayloadEncoder) self.batch_data = {}
def save_result2(self, name, monitor): result = MonitorResult() result.first_failure_time = format_datetime(monitor.first_failure_time()) result.virtual_fail_count = monitor.virtual_fail_count() result.last_run_duration = monitor.last_run_duration result.result = monitor.get_result() if hasattr(monitor, "was_skipped") and monitor.was_skipped: result.status = "Skipped" elif monitor.virtual_fail_count() <= 0: result.status = "OK" result.dependencies = monitor._dependencies self.batch_data[name] = result
def groupcalendar(): if flask.request.method == "GET": storage = Storage(thedata.STORAGE) cred = storage.get() # the next line copied from # https://code.google.com/p/google-api-python-client/wiki/OAuth2 if cred is None or cred.invalid == True: flow = get_flow() callback = thedata.REDIRECT_URI authorize_url = flow.step1_get_authorize_url(callback) thedata.MEMC.set("OAUTH2_FLOW", pickle.dumps(flow)) return flask.redirect(authorize_url) else: http = httplib2.Http() http = cred.authorize(http) probe = build('calendar', 'v3', http=http) # comments are just for records: ['acl', 'calendarList', # 'calendars', 'colors', 'events', 'freebusy', 'settings'] interested_calId = thedata.GROUP_CAL_ID current = datetime.datetime.now() delta_30d = datetime.timedelta(days=30) events = probe.events().list( calendarId=interested_calId, timeMin=util.format_datetime(current-delta_30d, iso=True), timeMax=util.format_datetime(current, iso=True)).execute() eVents = [] for event in events['items']: eVent = Event( event.get('creator', '{}'), event.get('start', '{}'), event.get('end', '{}'), event.get('location', 'noloc'), event.get('summary', 'nosummary'), ) eVents.append(eVent) return flask.render_template("cal.html", events=eVents)
def groupcalendar(): if flask.request.method == "GET": storage = Storage(thedata.STORAGE) cred = storage.get() # the next line copied from # https://code.google.com/p/google-api-python-client/wiki/OAuth2 if cred is None or cred.invalid == True: flow = get_flow() callback = thedata.REDIRECT_URI authorize_url = flow.step1_get_authorize_url(callback) thedata.MEMC.set("OAUTH2_FLOW", pickle.dumps(flow)) return flask.redirect(authorize_url) else: http = httplib2.Http() http = cred.authorize(http) probe = build('calendar', 'v3', http=http) # comments are just for records: ['acl', 'calendarList', # 'calendars', 'colors', 'events', 'freebusy', 'settings'] interested_calId = thedata.GROUP_CAL_ID current = datetime.datetime.now() delta_30d = datetime.timedelta(days=30) events = probe.events().list( calendarId=interested_calId, timeMin=util.format_datetime(current - delta_30d, iso=True), timeMax=util.format_datetime(current, iso=True)).execute() eVents = [] for event in events['items']: eVent = Event( event.get('creator', '{}'), event.get('start', '{}'), event.get('end', '{}'), event.get('location', 'noloc'), event.get('summary', 'nosummary'), ) eVents.append(eVent) return flask.render_template("cal.html", events=eVents)
def send_alert(self, name, monitor): type_ = self.should_alert(monitor) command = None (days, hours, minutes, seconds) = monitor.get_downtime() if monitor.is_remote(): host = monitor.running_on else: host = self.hostname if type_ == "": return elif type_ == "failure": command = self.fail_command elif type_ == "success": command = self.success_command elif type_ == "catchup": if self.catchup_command == 'fail_command': command = self.fail_command else: self.alerter_logger.error("Unknown alert type %s", type_) return if command is None: return command = command.format( hostname=host, name=name, days=days, hours=hours, minutes=minutes, seconds=seconds, failed_at=format_datetime(monitor.first_failure_time()), virtual_fail_count=monitor.virtual_fail_count(), info=monitor.get_result(), description=monitor.describe(), last_virtual_fail_count=monitor.last_virtual_fail_count()) if not self.dry_run: self.alerter_logger.debug("About to execute command: %s", command) try: subprocess.call(shlex.split(command)) except Exception: self.alerter_logger.exception( "Exception encountered running command: %s", command) if self.debug: self.alerter_logger.debug("Command has finished.") else: self.alerter_logger.info("Would run command: %s", command)
def save_result2(self, name, monitor): if not self.doing_batch: self.logger_logger.error( "HTMLLogger.save_result2() called while not doing batch.") return if monitor.virtual_fail_count() == 0: status = True else: status = False if not status: fail_time = format_datetime(monitor.first_failure_time()) fail_count = monitor.virtual_fail_count() fail_data = monitor.get_result() downtime = monitor.get_downtime() else: fail_time = "" fail_count = 0 fail_data = monitor.get_result() downtime = "" failures = monitor.failures last_failure = monitor.last_failure try: age = datetime.datetime.utcnow() - monitor.last_update age = age.days * 3600 + age.seconds update = monitor.last_update except Exception: age = 0 update = "" data_line = { "status": status, "fail_time": fail_time, "fail_count": fail_count, "fail_data": fail_data, "downtime": downtime, "age": age, "update": update, "host": monitor.running_on, "failures": failures, "last_failure": last_failure } self.batch_data[monitor.name] = data_line
def save_result2(self, name, monitor): if not self.doing_batch: self.logger_logger.error("HTMLLogger.save_result2() called while not doing batch.") return if monitor.virtual_fail_count() == 0: status = True else: status = False if not status: fail_time = format_datetime(monitor.first_failure_time()) fail_count = monitor.virtual_fail_count() fail_data = monitor.get_result() downtime = monitor.get_downtime() else: fail_time = "" fail_count = 0 fail_data = monitor.get_result() downtime = "" failures = monitor.failures last_failure = monitor.last_failure try: age = datetime.datetime.utcnow() - monitor.last_update age = age.days * 3600 + age.seconds update = monitor.last_update except Exception: age = 0 update = "" data_line = { "status": status, "fail_time": fail_time, "fail_count": fail_count, "fail_data": fail_data, "downtime": downtime, "age": age, "update": update, "host": monitor.running_on, "failures": failures, "last_failure": last_failure } self.batch_data[monitor.name] = data_line
def get_abus_list(self): from util import format_datetime sep = self.get_list_separator_line() res = "nad abus total abus error last error at code bandwidth\r\n" res += sep for controller in globals.controllers.get_sorted(): if controller.comm_proxy.last_error_timestamp != None: last_error_timestamp = format_datetime(controller.comm_proxy.last_error_timestamp) else: last_error_timestamp = "-" res += "%s %s %s %s %s %.2f%%\r\n" % \ ( repr(controller.config.nad).ljust(5), repr(controller.comm_proxy.abus_messages_tx_count).ljust(10), repr(controller.comm_proxy.abus_error_count).ljust(10), last_error_timestamp.ljust(19), controller.comm_proxy.last_error_code.ljust(5), controller.comm_proxy.get_bandwidth(), ) return res + sep
def send_alert(self, name, monitor): """Send the email.""" type = self.should_alert(monitor) (days, hours, minutes, seconds) = monitor.get_downtime() if monitor.is_remote(): host = " on %s " % monitor.running_on else: host = " on host %s" % self.hostname message = MIMEMultipart() message['From'] = self.from_addr message['To'] = self.to_addr if type == "": return elif type == "failure": message['Subject'] = "[%s] Monitor %s Failed!" % (self.hostname, name) body = """Monitor %s%s has failed. Failed at: %s Downtime: %d+%02d:%02d:%02d Virtual failure count: %d Additional info: %s Description: %s""" % ( name, host, format_datetime(monitor.first_failure_time()), days, hours, minutes, seconds, monitor.virtual_fail_count(), monitor.get_result(), monitor.describe()) try: if monitor.recover_info != "": body += "\nRecovery info: %s" % monitor.recover_info except AttributeError: body += "\nNo recovery info available" elif type == "success": message['Subject'] = "[%s] Monitor %s succeeded" % (self.hostname, name) body = "Monitor %s%s is back up.\nOriginally failed at: %s\nDowntime: %d+%02d:%02d:%02d\nDescription: %s" % ( name, host, format_datetime(monitor.first_failure_time()), days, hours, minutes, seconds, monitor.describe()) elif type == "catchup": message['Subject'] = "[%s] Monitor %s failed earlier!" % ( self.hostname, name) body = "Monitor %s%s failed earlier while this alerter was out of hours.\nFailed at: %s\nVirtual failure count: %d\nAdditional info: %s\nDescription: %s" % ( name, host, format_datetime(monitor.first_failure_time()), monitor.virtual_fail_count(), monitor.get_result(), monitor.describe()) else: self.alerter_logger.critical("unknown alert type %s", type) return message.attach(MIMEText(body, 'plain')) if not self.dry_run: try: if self.ssl is None or self.ssl == 'starttls': server = smtplib.SMTP(self.mail_host, self.mail_port) elif self.ssl == 'yes': server = smtplib.SMTP_SSL(self.mail_host, self.mail_port) if self.ssl == 'starttls': server.starttls() if self.username is not None: server.login(self.username, self.password) server.sendmail(self.from_addr, self.to_addr.split(';'), message.as_string()) server.quit() except Exception: self.alerter_logger.exception("couldn't send mail") self.available = False else: self.alerter_logger.info("dry_run: would send email: %s", message.as_string())
def send_alert(self, name, monitor): """Send the message.""" type = self.should_alert(monitor) (days, hours, minutes, seconds) = monitor.get_downtime() if self.channel is not None: message_json = {'channel': self.channel} elif self.username is not None: message_json = {'username': self.username} else: message_json = {} message_json['attachments'] = [{}] if type == "": return elif type == "failure": message_json['text'] = "Monitor {} failed!".format(name) message_json['attachments'][0]['color'] = 'danger' fields = [ { 'title': 'Failed at', 'value': format_datetime(monitor.first_failure_time()), 'short': True }, { 'title': 'Downtime', 'value': "{}+{:02d}:{:02d}:{:02d}".format(days, hours, minutes, seconds), 'short': True }, { 'title': 'Virtual failure count', 'value': monitor.virtual_fail_count(), 'short': True }, { 'title': 'Host', 'value': self.hostname, 'short': True }, { 'title': 'Additional info', 'value': monitor.get_result() }, { 'title': 'Description', 'value': monitor.describe() } ] try: if monitor.recover_info != "": fields.append({ 'title': 'Recovery info', 'value': "Recovery info: %s" % monitor.recover_info }) message_json['attachments'][0]['color'] = 'warning' except AttributeError: pass message_json['attachments'][0]['fields'] = fields elif type == "success": message_json['text'] = "Monitor {} succeeded.".format(name) fields = [ { 'title': 'Failed at', 'value': format_datetime(monitor.first_failure_time()), 'short': True }, { 'title': 'Downtime', 'value': "{}+{:02d}:{:02d}:{:02d}".format(days, hours, minutes, seconds), 'short': True }, { 'title': 'Host', 'value': self.hostname, 'short': True }, { 'title': 'Description', 'value': monitor.describe() } ] message_json['attachments'][0]['color'] = 'good' message_json['attachments'][0]['fields'] = fields else: self.alerter_logger.error("unknown alert type %s", type) return if not self.dry_run: try: r = requests.post(self.url, json=message_json) if not r.status_code == 200: self.alerter_logger.error("POST to slack webhook failed: %s", r) except Exception: self.alerter_logger.exception("Failed to post to slack webhook") self.available = False else: self.alerter_logger.info("dry_run: would send slack: %s", message_json.__repr__())
def send_alert(self, name, monitor): """Send an SMS alert.""" if not monitor.is_urgent(): return type = self.should_alert(monitor) message = "" url = "" (days, hours, minutes, seconds) = monitor.get_downtime() if type == "": return elif type == "catchup": message = "catchup: %s failed on %s at %s (%d+%02d:%02d:%02d)\n%s" % ( name, monitor.running_on, format_datetime(monitor.first_failure_time()), days, hours, minutes, seconds, monitor.get_result()) if len(message) > 160: self.alerter_logger.warning( "Truncating SMS message to 160 chars.") message = message[:156] + "..." url = "https://{}/a1/SMS".format(self.api_host) auth = (self.username, self.password) params = { 'from': self.sender, 'to': self.target, 'message': message, } elif type == "failure": message = "%s failed on %s at %s (%d+%02d:%02d:%02d)\n%s" % ( name, monitor.running_on, format_datetime(monitor.first_failure_time()), days, hours, minutes, seconds, monitor.get_result()) if len(message) > 160: self.alerter_logger.warning( "Truncating SMS message to 160 chars.") message = message[:156] + "..." url = "https://{}/a1/SMS".format(self.api_host) auth = (self.username, self.password) params = { 'from': self.sender, 'to': self.target, 'message': message, } else: # we don't handle other types of message pass if url == "": return if not self.dry_run: try: response = requests.post(url, data=params, auth=auth) s = response.json() if s['status'] not in ('created', 'delivered'): self.alerter_logger.error("Unable to send SMS: %s", s) self.available = False except Exception as e: self.alerter_logger.exception("SMS sending failed") self.available = False else: self.alerter_logger.info("dry_run: would send SMS: %s", url) return
def send_alert(self, name, monitor): """Send an alert.""" if not monitor.is_urgent(): return type = self.should_alert(monitor) message = "" url = "" (days, hours, minutes, seconds) = monitor.get_downtime() if type == "": return elif type == "catchup": message = "catchup: %s failed on %s at %s (%d+%02d:%02d:%02d)\n%s" % ( name, monitor.running_on, format_datetime(monitor.first_failure_time()), days, hours, minutes, seconds, monitor.get_result()) url = "https://{}/publicapi/notify".format(self.api_host) params = { 'apikey': self.apikey, 'application': self.application, 'description': message, 'event': "%s: %s" % (name, monitor.get_result()) } elif type == "failure": message = "%s failed on %s at %s (%d+%02d:%02d:%02d)\n%s" % ( name, monitor.running_on, format_datetime(monitor.first_failure_time()), days, hours, minutes, seconds, monitor.get_result()) url = "https://{}/publicapi/notify".format(self.api_host) params = { 'apikey': self.apikey, 'application': self.application, 'description': message, 'event': "%s: %s" % (name, monitor.get_result()) } else: # we don't handle other types of message pass if url == "": return if not self.dry_run: try: r = requests.get(url, params=params) s = r.text if not s.startswith( '<?xml version="1.0" encoding="UTF-8"?><nma><success code="200"' ): self.alerter_logger.error("Unable to send NMA: %s (%s)", s.split("|")[0], s.split("|")[1]) self.alerter_logger.error("URL: %s, PARAMS: %s", url, params) self.available = False except Exception: self.alerter_logger.exception("NMA sending failed") self.available = False else: self.alerter_logger.info("dry_run: would send NMA: %s", url) return
def __repr__(self): return "{0}|{1}\n{2} ".format(self.title, util.format_datetime(self.created), self.content)
def send_alert(self, name, monitor): """Send the email.""" type = self.should_alert(monitor) (days, hours, minutes, seconds) = monitor.get_downtime() if monitor.is_remote(): host = " on %s " % monitor.running_on else: host = " on host %s" % self.hostname mail = {'Source': self.from_addr} mail['Destination'] = {'ToAddresses': [self.to_addr]} if type == "": return elif type == "failure": message = { 'Subject': { 'Data': "[%s] Monitor %s Failed!" % (self.hostname, name) } } message['Body'] = { 'Text': { 'Data': """Monitor %s%s has failed. Failed at: %s Downtime: %d+%02d:%02d:%02d Virtual failure count: %d Additional info: %s Description: %s""" % (name, host, format_datetime( monitor.first_failure_time()), days, hours, minutes, seconds, monitor.virtual_fail_count(), monitor.get_result(), monitor.describe()) } } try: if monitor.recover_info != "": message['Body']['Text'][ 'Data'] += "\nRecovery info: %s" % monitor.recover_info except AttributeError: message['Body']['Text'][ 'Data'] += "\nNo recovery info available" elif type == "success": message = { 'Subject': { 'Data': "[%s] Monitor %s succeeded" % (self.hostname, name) } } message['Body'] = { 'Text': { 'Data': "Monitor %s%s is back up.\nOriginally failed at: %s\nDowntime: %d+%02d:%02d:%02d\nDescription: %s" % (name, host, format_datetime(monitor.first_failure_time()), days, hours, minutes, seconds, monitor.describe()) } } elif type == "catchup": message = { 'Subject': { 'Data': "[%s] Monitor %s failed earlier!" % (self.hostname, name) } } message['Body'] = { 'Text': { 'Data': "Monitor %s%s failed earlier while this alerter was out of hours.\nFailed at: %s\nVirtual failure count: %d\nAdditional info: %s\nDescription: %s" % (name, host, format_datetime(monitor.first_failure_time()), monitor.virtual_fail_count(), monitor.get_result(), monitor.describe()) } } else: self.alerter_logger.critical("Unknown alert type %s", type) return mail['Message'] = message if not self.dry_run: try: client = boto3.client('ses', **self.ses_client_params) client.send_email(**mail) except Exception: self.alerter_logger.exception("couldn't send mail") self.available = False else: self.alerter_logger.info("dry_run: would send email:") self.alerter_logger.info(" Subject: %s", message['Subject']['Data']) self.alerter_logger.info(" Body: %s", message['Body']['Text']['Data'])
def send_alert(self, name, monitor): """Send an SMS alert.""" if not monitor.is_urgent(): return type = self.should_alert(monitor) message = "" url = "" (days, hours, minutes, seconds) = monitor.get_downtime() if type == "": return elif type == "catchup": message = "catchup: %s failed on %s at %s (%d+%02d:%02d:%02d)\n%s" % ( name, monitor.running_on, format_datetime(monitor.first_failure_time()), days, hours, minutes, seconds, monitor.get_result()) if len(message) > 160: self.alerter_logger.warning("Truncating SMS message to 160 chars.") message = message[:156] + "..." url = "https://{}/a1/SMS".format(self.api_host) auth = (self.username, self.password) params = { 'from': self.sender, 'to': self.target, 'message': message, } elif type == "failure": message = "%s failed on %s at %s (%d+%02d:%02d:%02d)\n%s" % ( name, monitor.running_on, format_datetime(monitor.first_failure_time()), days, hours, minutes, seconds, monitor.get_result()) if len(message) > 160: self.alerter_logger.warning("Truncating SMS message to 160 chars.") message = message[:156] + "..." url = "https://{}/a1/SMS".format(self.api_host) auth = (self.username, self.password) params = { 'from': self.sender, 'to': self.target, 'message': message, } else: # we don't handle other types of message pass if url == "": return if not self.dry_run: try: response = requests.post(url, data=params, auth=auth) s = response.json() if s['status'] not in ('created', 'delivered'): self.alerter_logger.error("Unable to send SMS: %s", s) self.available = False except Exception: self.alerter_logger.exception("SMS sending failed") self.available = False else: self.alerter_logger.info("dry_run: would send SMS: %s", url) return
def test_gte(self): for op in ["gte", ">="]: f1 = FileRule("mtime", op, format_datetime(time() - 3600), "exclude") f2 = FileRule("mtime", op, format_datetime(time() + 3600), "exclude") self.assertEqual(f1.test(self.file_entry), "exclude") self.assertEqual(f2.test(self.file_entry), None)
def process_batch(self): """Save the HTML file.""" ok_count = 0 fail_count = 0 old_count = 0 remote_count = 0 my_host = short_hostname() try: temp_file = tempfile.mkstemp() file_handle = os.fdopen(temp_file[0], "w") file_name = temp_file[1] except Exception: sys.stderr.write("Couldn't create temporary file for HTML output\n") return output_ok = StringIO() output_fail = StringIO() keys = list(self.batch_data.keys()) keys.sort() for entry in keys: if self.batch_data[entry]["age"] > 120: status = "OLD" old_count += 1 elif self.batch_data[entry]["status"]: status = "OK" ok_count += 1 else: status = "FAIL" fail_count += 1 if self.batch_data[entry]["host"] != my_host: remote_count += 1 try: monitor_name = entry.split("/")[1] except Exception: monitor_name = entry if status == "FAIL": output = output_fail else: output = output_ok output.write("<tr class=\"%srow\">" % status.lower()) output.write(""" <td class="monitor_name">%s</td> <td class="status %s">%s</td> <td>%s</td> <td>%s</td> """ % ( monitor_name, status.lower(), status, self.batch_data[entry]["host"], self.batch_data[entry]["fail_time"], ) ) if self.batch_data[entry]["fail_count"] == 0: output.write("<td class=\"vfc\"> </td>") else: output.write("<td class=\"vfc\">%s</td>" % self.batch_data[entry]["fail_count"]) try: output.write("<td>%d+%02d:%02d:%02d</td>" % (self.batch_data[entry]["downtime"][0], self.batch_data[entry]["downtime"][1], self.batch_data[entry]["downtime"][2], self.batch_data[entry]["downtime"][3])) except Exception: output.write("<td> </td>") output.write("<td>%s </td>" % (self.batch_data[entry]["fail_data"])) if self.batch_data[entry]["failures"] == 0: output.write("<td></td><td></td>") else: output.write("""<td>%s</td> <td>%s</td>""" % ( self.batch_data[entry]["failures"], format_datetime(self.batch_data[entry]["last_failure"]) ) ) if self.batch_data[entry]["host"] == my_host: output.write("<td></td>") else: output.write("<td>%d</td>" % self.batch_data[entry]["age"]) output.write("</tr>\n") count_data = "<div id=\"summary\"" if old_count > 0: cls = "old" elif fail_count > 0: cls = "fail" else: cls = "ok" count_data = count_data + " class=\"%s\">%s" % (cls, cls.upper()) self.count_data = count_data + "<div id=\"details\"><span class=\"ok\">%d OK</span> <span class=\"fail\">%d FAIL</span> <span class=\"old\">%d OLD</span> <span class=\"remote\">%d remote</span></div></div>" % (ok_count, fail_count, old_count, remote_count) self.status = cls.upper() with open(os.path.join(self.folder, self.header), "r") as file_input: file_handle.writelines(self.parse_file(file_input)) file_handle.write(output_fail.getvalue()) file_handle.write(output_ok.getvalue()) with open(os.path.join(self.folder, self.footer), "r") as file_input: file_handle.writelines(self.parse_file(file_input)) try: file_handle.flush() file_handle.close() os.chmod(file_name, stat.S_IREAD | stat.S_IWRITE | stat.S_IRGRP | stat.S_IROTH) shutil.move(file_name, os.path.join(self.folder, self.filename)) except Exception: self.logger_logger.exception("problem closing temporary file for HTML output")
def send_alert(self, name, monitor): """Send the email.""" type = self.should_alert(monitor) (days, hours, minutes, seconds) = monitor.get_downtime() if monitor.is_remote(): host = " on %s " % monitor.running_on else: host = " on host %s" % self.hostname message = MIMEMultipart() message['From'] = self.from_addr message['To'] = self.to_addr if type == "": return elif type == "failure": message['Subject'] = "[%s] Monitor %s Failed!" % (self.hostname, name) body = """Monitor %s%s has failed. Failed at: %s Downtime: %d+%02d:%02d:%02d Virtual failure count: %d Additional info: %s Description: %s""" % ( name, host, format_datetime(monitor.first_failure_time()), days, hours, minutes, seconds, monitor.virtual_fail_count(), monitor.get_result(), monitor.describe()) try: if monitor.recover_info != "": body += "\nRecovery info: %s" % monitor.recover_info except AttributeError: body += "\nNo recovery info available" elif type == "success": message['Subject'] = "[%s] Monitor %s succeeded" % (self.hostname, name) body = "Monitor %s%s is back up.\nOriginally failed at: %s\nDowntime: %d+%02d:%02d:%02d\nDescription: %s" % (name, host, format_datetime(monitor.first_failure_time()), days, hours, minutes, seconds, monitor.describe()) elif type == "catchup": message['Subject'] = "[%s] Monitor %s failed earlier!" % (self.hostname, name) body = "Monitor %s%s failed earlier while this alerter was out of hours.\nFailed at: %s\nVirtual failure count: %d\nAdditional info: %s\nDescription: %s" % (name, host, format_datetime(monitor.first_failure_time()), monitor.virtual_fail_count(), monitor.get_result(), monitor.describe()) else: self.alerter_logger.critical("unknown alert type %s", type) return message.attach(MIMEText(body, 'plain')) if not self.dry_run: try: if self.ssl is None or self.ssl == 'starttls': server = smtplib.SMTP(self.mail_host, self.mail_port) elif self.ssl == 'yes': server = smtplib.SMTP_SSL(self.mail_host, self.mail_port) if self.ssl == 'starttls': server.starttls() if self.username is not None: server.login(self.username, self.password) server.sendmail(self.from_addr, self.to_addr.split(';'), message.as_string()) server.quit() except Exception: self.alerter_logger.exception("couldn't send mail") self.available = False else: self.alerter_logger.info("dry_run: would send email: %s", message.as_string())
def test_Format(self): self.assertEqual(util.format_datetime(None), "") self.assertEqual(util.format_datetime("a string"), "a string") self.assertEqual( util.format_datetime(datetime.datetime(2018, 5, 8, 13, 37, 0)), "2018-05-08 13:37:00")
def process_batch(self): """Save the HTML file.""" ok_count = 0 fail_count = 0 old_count = 0 remote_count = 0 my_host = short_hostname try: temp_file = tempfile.mkstemp() file_handle = os.fdopen(temp_file[0], "w") file_name = temp_file[1] except Exception: sys.stderr.write("Couldn't create temporary file for HTML output\n") return output_ok = StringIO() output_fail = StringIO() keys = list(self.batch_data.keys()) keys.sort() for entry in keys: if self.batch_data[entry]["age"] > 120: status = "OLD" old_count += 1 elif self.batch_data[entry]["status"]: status = "OK" ok_count += 1 else: status = "FAIL" fail_count += 1 if self.batch_data[entry]["host"] != my_host: remote_count += 1 try: monitor_name = entry.split("/")[1] except Exception: monitor_name = entry if status == "FAIL": output = output_fail else: output = output_ok output.write("<tr class=\"%srow\">" % status.lower()) output.write(""" <td class="monitor_name">%s</td> <td class="status %s">%s</td> <td>%s</td> <td>%s</td> """ % ( monitor_name, status.lower(), status, self.batch_data[entry]["host"], self.batch_data[entry]["fail_time"], ) ) if self.batch_data[entry]["fail_count"] == 0: output.write("<td class=\"vfc\"> </td>") else: output.write("<td class=\"vfc\">%s</td>" % self.batch_data[entry]["fail_count"]) try: output.write("<td>%d+%02d:%02d:%02d</td>" % (self.batch_data[entry]["downtime"][0], self.batch_data[entry]["downtime"][1], self.batch_data[entry]["downtime"][2], self.batch_data[entry]["downtime"][3])) except Exception: output.write("<td> </td>") output.write("<td>%s </td>" % (self.batch_data[entry]["fail_data"])) if self.batch_data[entry]["failures"] == 0: output.write("<td></td><td></td>") else: output.write("""<td>%s</td> <td>%s</td>""" % ( self.batch_data[entry]["failures"], format_datetime(self.batch_data[entry]["last_failure"]) ) ) if self.batch_data[entry]["host"] == my_host: output.write("<td></td>") else: output.write("<td>%d</td>" % self.batch_data[entry]["age"]) output.write("</tr>\n") count_data = "<div id=\"summary\"" if old_count > 0: cls = "old" elif fail_count > 0: cls = "fail" else: cls = "ok" count_data = count_data + " class=\"%s\">%s" % (cls, cls.upper()) self.count_data = count_data + "<div id=\"details\"><span class=\"ok\">%d OK</span> <span class=\"fail\">%d FAIL</span> <span class=\"old\">%d OLD</span> <span class=\"remote\">%d remote</span></div></div>" % (ok_count, fail_count, old_count, remote_count) self.status = cls.upper() with open(os.path.join(self.folder, self.header), "r") as file_input: file_handle.writelines(self.parse_file(file_input)) file_handle.write(output_fail.getvalue()) file_handle.write(output_ok.getvalue()) with open(os.path.join(self.folder, self.footer), "r") as file_input: file_handle.writelines(self.parse_file(file_input)) try: file_handle.flush() file_handle.close() os.chmod(file_name, stat.S_IREAD | stat.S_IWRITE | stat.S_IRGRP | stat.S_IROTH) shutil.move(file_name, os.path.join(self.folder, self.filename)) except Exception as e: self.logger_logger.exception("problem closing temporary file for HTML output")
def parse_bill(congress, bill_type, bill_number): """Downloads and parses THOMAS bill status and summary files.""" # map our bill type code to the THOMAS bill type code (namely, hr is confused with H.R. # so make it hres). bill_type2 = bill_type if bill_type2 == "hr": bill_type2 = "hres" # Start with the All Actions page, from which we'll also grab basic metadata. url = "http://thomas.loc.gov/cgi-bin/bdquery/z?d%03d:%s%s:@@@X" % (congress, bill_type2, bill_number) content, mtime = download(url) if not content: raise Exception("Failed to download bill status page: " + url) root = etree.Element("bill") root.set("session", str(congress)) root.set("type", bill_type) root.set("number", str(bill_number)) root.set("updated", format_datetime(mtime)) sponsor = None introduced_date = None title = None actions = [] action_indentation = 0 state_name, state_date = None, None for line in content.split("\n"): # Match Sponsor line, which can be either No Sponsor or a name/state/district. # Parse the name later, after we find the introduced date. m = re.search(r"<b>Sponsor: </b>(No Sponsor|<a [^>]+>(.*)</a>\s+\[((\w\w)(-(\d+))?)\])", line, re.I) if m != None: if m.group(1) == "No Sponsor": sponsor = (None, None, None, None) else: name = m.group(2) if name.startswith("Sen "): senrep = "sen" name = name[4:] elif name.startswith("Rep "): senrep = "rep" name = name[4:] else: raise Exception("Invalid name: missing title: " + name) sponsor = (name, senrep, m.group(4), m.group(6)) # name, type, state, district # Match the line giving the date of introduction. m = re.search(r"\(introduced ([\d\/]+)\)", line, re.I) if m != None: introduced_date = datetime.datetime.strptime(m.group(1), "%m/%d/%Y").date() state_name, state_date = ("INTRODUCED", introduced_date) # Match the Title line, which we use 1) to check if this is a bill number reserved for the speaker, # 2) for determining whether this is a proposal for a constitutional amendment for parsing # vote status, and 3) as a backup in case the THOMAS Titles page cannot be parsed. m = re.search(r"<B>(Latest )?Title:</B> (.+)", line, re.I) if m != None: title = m.group(2) # Match action lines. m = re.search(r"<dt><strong>([\d/ :apm]+):</strong><dd>(.+)", line, re.I) if m != None: # Indentation indicates committee action. if line.startswith("<dl>"): action_indentation += 1 if line.startswith("</dl>"): action_indentation -= 1 text = re.sub(r"</?[Aa]( \S.*?)?>", "", m.group(2)) # The date can be either a date or a date and time. try: action_date = datetime.datetime.strptime(m.group(1), "%m/%d/%Y %I:%M%p") except: #print repr(m.group(1)) try: action_date = datetime.datetime.strptime(m.group(1), "%m/%d/%Y").date() except: raise ValueError("Could not parse date: " + m.group(1)) # references are given in parentheses at the end considerations = [] m = re.search("\s+\((.*)\)\s*$", text) if m: text = text[0:m.start()] + text[m.end():] for con in m.group(1).split("; "): if ": " not in con: considerations.append( ("", con) ) else: considerations.append( con.split(": ") ) # Parse the actual action line. attrs = parse_bill_action(text, bill_type, state_name, title) if "state" in attrs: state_name, state_date = attrs["state"], action_date actions.append((action_date, action_indentation, text, attrs, considerations)) if introduced_date == None: raise Exception("No introduced date.") if sponsor == None: raise Exception("No sponsor line.") elif sponsor[0] == None: sponsor = 0 else: sponsor = parse_name(sponsor[0], introduced_date, nameformat="lastfirst", role_type=sponsor[1], state=sponsor[2], district=sponsor[3]) if "Reserved for the" in title: raise Exception("Skipping bill " + title.lower()) # Start building the XML state = etree.Element("state") state.set("datetime", format_datetime(state_date)) state.text = state_name root.append(state) intronode = etree.Element("introduced") intronode.set("datetime", introduced_date.isoformat()) root.append(intronode) if sponsor != 0: sponsornode = etree.Element("sponsor") sponsornode.set("id", str(sponsor)) root.append(sponsornode) # Download and parse the cosponsors page. cosponsors = etree.Element("cosponsors") root.append(cosponsors) url = url.replace("@@@X", "@@@P") content, mtime = download(url) if not content: raise Exception("Failed to download cosponsors page: " + url) content = re.sub(r"(\[[A-Z\d\-]+\])\n( - \d\d?\/)", lambda m : m.group(1) + m.group(2), content) # bring cosponsorship date onto previous line content = re.sub(r"</br>", "\n", content) for line in content.split("\n"): m = re.search(r"(<br ?/>)?<a href=[^>]+>(Rep|Sen) (.+)</a> \[([A-Z\d\-]+)\] - (\d\d?/\d\d?/\d\d\d\d)(\(withdrawn - (\d\d?/\d\d?/\d\d\d\d)\))?", line, re.I) if m: title, name, state_district, join_date, withdrawn_date = m.group(2), m.group(3), m.group(4), m.group(5), m.group(7) join_date = datetime.datetime.strptime(join_date, "%m/%d/%Y").date() if withdrawn_date != None: withdrawn_date = datetime.datetime.strptime(withdrawn_date, "%m/%d/%Y").date() name = name.replace("Colordao", "Colorado") # typo if not "-" in state_district: state = state_district district = None else: state, district = state_district.split("-") person = parse_name(name, join_date, nameformat="lastfirst", role_type=title.lower(), state=state, district=district) csp = etree.Element("cosponsor") csp.set("id", str(person)) csp.set("joined", join_date.isoformat()) if withdrawn_date: csp.set("withdrawn", withdrawn_date.isoformat()) cosponsors.append(csp) # Download and parse the titles page. titles = etree.Element("titles") root.append(titles) url = url.replace("@@@P", "@@@T") content, mtime = download(url) if not content: raise Exception("Failed to download titles page: " + url) content = re.sub(r"(<I>)?(<br/>|<p>)", lambda m : "\n" + ("" if not m.group(1) else m.group(1)), content) title_type, title_as = None, None for line in content.split("\n"): if line == "</ul>": break m = re.search(r"<li>(.*) title(\(s\))?( as ([\w ]*))?:", line, re.I) if m: title_type, title_as = m.group(1).lower(), m.group(4).lower() elif title_type != None and line.strip() != "": partial = False if line.startswith("<I>"): partial = True line = line[3:] line = line.replace("<I>", "") line = line.replace("</I>", "") line = line.replace(" (identified by CRS)", "") t = etree.Element("title") t.set("type", title_type) t.set("as", title_as) t.set("partial", "yes" if partial else "no") t.text = line.strip() titles.append(t) if len(titles) == 0: # Sometimes titles aren't available. t = etree.Element("title") t.set("type", "official") t.set("as", "introduced") t.set("partial", "no") t.text = title titles.append(t) # Download and parse the committees page. committees = etree.Element("committees") root.append(committees) url = url.replace("@@@T", "@@@C") content, mtime = download(url) if not content: raise Exception("Failed to download committees page: " + url) last_committee = None for line in content.split("\n"): m = re.search(r'<a href="/cgi-bin/bdquery(tr)?/R\?[^"]+">(.*)</a>\s*</td><td width="65\%">(.+)</td></tr>', line, re.I) if m: committee = m.group(2) activity = m.group(3) committee = re.sub(r"\s+", " ", committee).strip() if not committee.startswith("Subcommittee on "): last_committee = committee committee = find_committee(committee, None, congress) else: committee = committee[len("Subcommittee on "):] committee = find_committee(last_committee, committee, congress) cx = etree.Element("committee") cx.set("code", committee) cx.set("activity", activity) committees.append(cx) # Download and parse related bills page. related_bill_type_map = { } for a, b in thomas_bill_type_codes: related_bill_type_map[a] = b related_bill_relationship_map = { "Identical bill identified by CRS": "identical", "Related bill identified by CRS": "related", "Related bill as identified by the House Clerk's office": "related", "passed in House in lieu of this bill": "supersedes", "passed in Senate in lieu of this bill": "supersedes", } related_bills = etree.Element("relatedbills") root.append(related_bills) url = url.replace("@@@C", "@@@K") content, mtime = download(url) if not content: raise Exception("Failed to download related bills page: " + url) for line in content.split("\n"): m = re.search(r'<a href="/cgi-bin/bdquery(tr)?/z\?d(\d\d\d):(\w+)(\d\d\d\d\d):">.*</a></td><td>(.*)</td></tr>', line, re.I) if m: related_bill_congress = int(m.group(2)) related_bill_type = related_bill_type_map[m.group(3)] related_bill_number = int(m.group(4)) if re.search("Rule related to", m.group(5)): related_bill_relationship = "rule" else: related_bill_relationship = related_bill_relationship_map[m.group(5)] rb = etree.Element("bill") rb.set("relation", related_bill_relationship) rb.set("session", str(related_bill_congress)) rb.set("type", related_bill_type) rb.set("number", str(related_bill_number)) related_bills.append(rb) # Download and parse CRS subject terms page. subjects = etree.Element("subjects") root.append(subjects) url = url.replace("@@@K", "@@@J") content, mtime = download(url) if not content: raise Exception("Failed to download subject terms page: " + url) for line in content.split("\n"): m = re.search(r'<a href="/cgi-bin/bdquery/\?.*@FIELD\(FLD001.*\)">(.*)</a> ', line, re.I) if m: term = m.group(1) term = re.sub(r"\s+", " ", term).strip() s = etree.Element("term") s.set("name", term) subjects.append(s) # Download and parse amendments page. amendments = etree.Element("amendments") root.append(amendments) url = url.replace("@@@J", "@@@A") content, mtime = download(url) if not content: raise Exception("Failed to download amendments page: " + url) for m in re.finditer(r'<a href="/cgi-bin/bdquery/z\?d\d+:([HS])([ZP])(\d+):">[HS]\.AMDT\.\d+</a>', content, re.I): amendment_chamber = m.group(1).lower() amendment_number = int(m.group(3)) a = etree.Element("amendment") a.set("number", amendment_chamber + str(amendment_number)) amendments.append(a) # Put the actions in here. actionsnode = etree.Element("actions") root.append(actionsnode) for adate, aindent, text, attrs, considerations in actions: nodename = "action" if "nodename" in attrs: nodename = attrs["nodename"] del attrs["nodename"] node = etree.Element(nodename) actionsnode.append(node) node.set("datetime", format_datetime(adate)) for k, v in sorted(attrs.items()): if v == None: continue node.set(k, v) n = etree.Element("text") n.text = text node.append(n) for c in considerations: n = etree.Element("reference") n.set("label", c[0]) n.set("ref", c[1]) node.append(n) # Download the CRS summary text. url = url.replace("@@@A", "@@@D&summ2=m&") content, mtime = download(url) if not content: raise Exception("Failed to download summary page: " + url) mode = 0 summary = "" for line in content.split("\n"): if mode == 1: if "<hr" in line: mode = 0 elif "THOMAS Home" in line or "id=\"footer\"" in line: break else: line = re.sub(r"<a.*?>(.*?)</a>", lambda m : m.group(1), line, re.I) summary += line + "\n" elif "SUMMARY AS OF" in line: mode = 1 summary = re.sub(r"\(There (is|are) \d+ other summar(y|ies)\)", "", summary, re.I) root.append(fragment_fromstring(summary, create_parent="summary")) try: os.makedirs("../data/us/%d/bills" % congress) except: pass return etree.tostring(root, pretty_print=True)
def __repr__(self): return "<{0:20s} on {1:10s} at {2} - running_cores: {3:8d}; not_running_cores: {4:8d}".format( self.username, self.clustername, util.format_datetime(self.created), self.runningcores, self.notrunningcores)
def __repr__(self): return "<{0} joined at {1})>".format( self.email, util.format_datetime(self.created))
def _get_datestring(self): if self.dateformat == 'iso8601': return format_datetime(datetime.datetime.now()) return str(int(time.time()))
def send_alert(self, name, monitor): """Send an SMS alert.""" if not monitor.is_urgent(): return type_ = self.should_alert(monitor) message = "" url = "" (days, hours, minutes, seconds) = monitor.get_downtime() if type_ == "": return elif type_ == "catchup": message = "catchup: %s failed on %s at %s (%d+%02d:%02d:%02d)\n%s" % ( name, monitor.running_on, format_datetime(monitor.first_failure_time()), days, hours, minutes, seconds, monitor.get_result()) if len(message) > 160: self.alerter_logger.warning("Truncating SMS message to 160 chars.") message = message[:156] + "..." url = "https://{}/eapi/submission/send_sms/2/2.0".format(self.api_host) params = { 'username': self.username, 'password': self.password, 'message': message, 'msisdn': self.target, 'sender': self.sender, 'repliable': '0' } elif type_ == "failure": message = "%s failed on %s at %s (%d+%02d:%02d:%02d)\n%s" % ( name, monitor.running_on, format_datetime(monitor.first_failure_time()), days, hours, minutes, seconds, monitor.get_result()) if len(message) > 160: self.alerter_logger.warning("Truncating SMS message to 160 chars.") message = message[:156] + "..." url = "https://{}/eapi/submission/send_sms/2/2.0".format(self.api_host) params = { 'username': self.username, 'password': self.password, 'message': message, 'msisdn': self.target, 'sender': self.sender, 'repliable': '0' } else: # we don't handle other types of message pass if url == "": return if not self.dry_run: try: r = requests.get(url, params=params) s = r.text if not s.startswith("0"): self.alerter_logger.error("Unable to send SMS: %s (%s)", s.split("|")[0], s.split("|")[1]) self.available = False except Exception: self.alerter_logger.exception("SMS sending failed") self.available = False else: self.alerter_logger.info("dry_run: would send SMS: %s", url) return
def send_alert(self, name, monitor): """Send an SMS alert.""" if not monitor.is_urgent(): return type_ = self.should_alert(monitor) message = "" url = "" (days, hours, minutes, seconds) = monitor.get_downtime() if type_ == "": return elif type_ == "catchup": message = "catchup: %s failed on %s at %s (%d+%02d:%02d:%02d)\n%s" % ( name, monitor.running_on, format_datetime(monitor.first_failure_time()), days, hours, minutes, seconds, monitor.get_result()) if len(message) > 160: self.alerter_logger.warning( "Truncating SMS message to 160 chars.") message = message[:156] + "..." url = "https://{}/eapi/submission/send_sms/2/2.0".format( self.api_host) params = { 'username': self.username, 'password': self.password, 'message': message, 'msisdn': self.target, 'sender': self.sender, 'repliable': '0' } elif type_ == "failure": message = "%s failed on %s at %s (%d+%02d:%02d:%02d)\n%s" % ( name, monitor.running_on, format_datetime(monitor.first_failure_time()), days, hours, minutes, seconds, monitor.get_result()) if len(message) > 160: self.alerter_logger.warning( "Truncating SMS message to 160 chars.") message = message[:156] + "..." url = "https://{}/eapi/submission/send_sms/2/2.0".format( self.api_host) params = { 'username': self.username, 'password': self.password, 'message': message, 'msisdn': self.target, 'sender': self.sender, 'repliable': '0' } else: # we don't handle other types of message pass if url == "": return if not self.dry_run: try: r = requests.get(url, params=params) s = r.text if not s.startswith("0"): self.alerter_logger.error("Unable to send SMS: %s (%s)", s.split("|")[0], s.split("|")[1]) self.available = False except Exception as e: self.alerter_logger.exception("SMS sending failed") self.available = False else: self.alerter_logger.info("dry_run: would send SMS: %s", url) return
def _serialize(self, value, attr, obj): return util.format_datetime(value)