class MSGEgaugeNewDataChecker(object): """ Provide notification of newly loaded MSG eGauge data. This uses notification type MSG_EGAUGE_SERVICE. """ def __init__(self): """ Constructor. """ print __name__ self.logger = SEKLogger(__name__) self.connector = MSGDBConnector() self.dbUtil = MSGDBUtil() self.notifier = MSGNotifier() self.configer = MSGConfiger() def newDataCount(self): """ Measure the amount of new data that is present since the last time new data was reported. """ cursor = self.connector.conn.cursor() tableName = 'EgaugeEnergyAutoload' lastTime = self.lastReportDate('MSG_EGAUGE_SERVICE') if lastTime is None: lastTime = '1900-01-01' sql = """SELECT COUNT(*) FROM "%s" WHERE datetime > '%s'""" % ( tableName, lastTime) success = self.dbUtil.executeSQL(cursor, sql) if success: rows = cursor.fetchall() if not rows[0][0]: return 0 else: return rows[0][0] else: # @todo Raise an exception. return None def lastReportDate(self, notificationType): """ Get the last time a notification was reported. :param notificationType: A string indicating the type of the notification. It is stored in the event history. :returns: datetime of last report date. """ cursor = self.connector.conn.cursor() sql = """SELECT MAX("notificationTime") FROM "%s" WHERE "notificationType" = '%s'""" % ( NOTIFICATION_HISTORY_TABLE, notificationType) success = self.dbUtil.executeSQL(cursor, sql) if success: rows = cursor.fetchall() if not rows[0][0]: return None else: return rows[0][0] else: # @todo Raise an exception. return None def saveNotificationTime(self): """ Save the notification event to the notification history. """ cursor = self.connector.conn.cursor() sql = """INSERT INTO "%s" ("notificationType", "notificationTime") VALUES ('MSG_EGAUGE_SERVICE', NOW())""" % NOTIFICATION_HISTORY_TABLE success = self.dbUtil.executeSQL(cursor, sql) self.connector.conn.commit() if not success: # @todo Raise an exception. self.logger.log( 'An error occurred while saving the notification time.') def sendNewDataNotification(self, testing = False): """ Sending notification reporting on new data being available since the last time new data was reported. :param testing: Use testing mode when True. """ lastReportDate = self.lastReportDate('MSG_EGAUGE_SERVICE') if not lastReportDate: lastReportDate = "never" msgBody = '\nNew MSG eGauge data has been loaded to %s.' % self\ .connector.dbName msgBody += '\n\n' msgBody += 'The new data count is %s readings.' % self.newDataCount() msgBody += '\n\n' msgBody += 'The last report date was %s.' % lastReportDate msgBody += '\n\n' self.notifier.sendNotificationEmail(msgBody, testing = testing) self.saveNotificationTime()
class NewDataAggregator(object): """ Perform aggregation of new data for a set of predefined data types (self .rawTypes). """ def __init__(self): """ Constructor. """ self.logger = SEKLogger(__name__, 'DEBUG') self.aggregator = MSGDataAggregator() self.notifier = MSGNotifier() self.rawTypes = [x.name for x in list(MSGAggregationTypes)] self.connector = MSGDBConnector() self.conn = self.connector.connectDB() self.cursor = self.conn.cursor() self.dbUtil = MSGDBUtil() def sendNewDataNotification(self, result = None, testing = False): """ Sending notification reporting on new data being available since the last time new data was reported. :param result: list of dicts containing aggregation results as provided by MSGDataAggregator::aggregateNewData. :param testing: Use testing mode when True. """ self.logger.log('result {}'.format(result), 'debug') lastReportDate = self.notifier.lastReportDate( MSGNotificationHistoryTypes.MSG_DATA_AGGREGATOR) if not lastReportDate: lastReportDate = "never" if not result: msgBody = '\nNew data has NOT been aggregated in {}. No result ' \ 'was obtained. This is an error that should be ' \ 'investigated.'.format(self.connector.dbName) else: msgBody = '\nNew data has been aggregated in {}.'.format( self.connector.dbName) msgBody += '\n\n' for i in range(len(result)): msgBody += 'The new data count for type {} is {} readings' \ '.\n'.format(result[i].keys()[0], result[i].values()[0]) msgBody += '\n\n' msgBody += 'The last report date was %s.' % lastReportDate msgBody += '\n\n' self.notifier.sendNotificationEmail(msgBody, testing = testing) self.notifier.recordNotificationEvent( MSGNotificationHistoryTypes.MSG_DATA_AGGREGATOR) def aggregateNewData(self): """ :return: list of dicts obtained from MSGDataAggregator::aggregateNewData. """ result = map(self.aggregator.aggregateNewData, self.rawTypes) self.logger.log('result {}'.format(result)) return result
class TestMECONotifier(unittest.TestCase): """ Unit tests for the MECO Notifier. """ def setUp(self): self.logger = SEKLogger(__name__) self.notifier = MSGNotifier() self.configer = MSGConfiger() def tearDown(self): pass def testInit(self): self.assertIsNotNone(self.notifier, "Notifier has been initialized.") def testEmailServer(self): """ Test connecting to the email server. """ errorOccurred = False user = self.configer.configOptionValue("Notifications", "email_username") password = self.configer.configOptionValue("Notifications", "email_password") server = smtplib.SMTP(self.configer.configOptionValue("Notifications", "smtp_server_and_port")) try: server.starttls() except smtplib.SMTPException as detail: self.logger.log("Exception: {}".format(detail)) try: server.login(user, password) except smtplib.SMTPException as detail: self.logger.log("Exception: {}".format(detail)) self.assertFalse(errorOccurred, "No errors occurred during SMTP setup.") def testSendEmailNotification(self): """ Send a test notification by email. """ if SEND_EMAIL: success = self.notifier.sendNotificationEmail( "This is a message from testSendEmailNotification.", testing=True ) self.assertTrue(success, "Sending an email notification did not produce an" " exception.") else: self.assertTrue(True, "Email is not sent when SEND_EMAIL is False.") def testSendEmailAttachment(self): """ Send a test notification with attachment by email. """ if SEND_EMAIL: body = "Test message" testDataPath = self.configer.configOptionValue("Testing", "test_data_path") file = os.path.join(testDataPath, "graph.png") success = self.notifier.sendMailWithAttachments(body, [file], testing=True) success = success != True self.assertTrue(success, "Sending an email notification did not produce an" " exception.") else: self.assertTrue(True, "Email is not sent when SEND_EMAIL is False.")
if len(data[i]) == 0: data[i] = 'NULL' else: data[i] = "'" + data[i] + "'" sql = """INSERT INTO "LocationRecords" (%s) VALUES (%s)""" % ( ','.join(cols), ','.join(data)) success = dbUtil.executeSQL(cur, sql) if not success: anyFailure = True lineCnt += 1 conn.commit() msg = ("Processed %s lines.\n" % lineCnt) sys.stderr.write(msg) msgBody += msg if not anyFailure: msg = "Finished inserting location records.\n" sys.stderr.write(msg) msgBody += msg else: msg = "Location records were NOT successfully loaded.\n" sys.stderr.write(msg) msgBody += msg notifier.sendNotificationEmail(msgBody)
# PostgreSQL. data[i] = data[i].replace("'", "\'\'") data[i] = "'" + data[i] + "'" sql = """INSERT INTO "MeterLocationHistory" (%s) VALUES (%s)""" % ( ','.join(cols), ','.join(data)) logger.log("SQL: %s" % sql, 'debug') success = dbUtil.executeSQL(cur, sql) if not success: anyFailure = True lineCnt += 1 conn.commit() msg = ("Processed %s lines.\n" % lineCnt) sys.stderr.write(msg) msgBody += msg if not anyFailure: msg = "Finished inserting Meter Location History records.\n" sys.stderr.write(msg) msgBody += msg else: msg = "Meter Location History records were NOT successfully loaded.\n" sys.stderr.write(msg) msgBody += msg if COMMAND_LINE_ARGS.email: notifier.sendNotificationEmail(msgBody, testing = COMMAND_LINE_ARGS.testing)
class NewDataAggregator(object): """ Perform aggregation of new data for a set of predefined data types (self .rawTypes). """ def __init__(self): """ Constructor. """ self.logger = MSGLogger(__name__, 'DEBUG') self.aggregator = MSGDataAggregator() self.notifier = MSGNotifier() self.rawTypes = ['weather', 'egauge', 'circuit', 'irradiance'] self.connector = MSGDBConnector() self.conn = self.connector.connectDB() self.cursor = self.conn.cursor() self.dbUtil = MSGDBUtil() def lastReportDate(self, notificationType): """ Get the last time a notification was reported. :param notificationType: string indicating the type of the notification. It is stored in the event history. :returns: datetime of last report date. """ cursor = self.cursor sql = """SELECT MAX("notificationTime") FROM "{}" WHERE "notificationType" = '{}'""".format(NOTIFICATION_HISTORY_TABLE, notificationType) success = self.dbUtil.executeSQL(cursor, sql) if success: rows = cursor.fetchall() if not rows[0][0]: return None else: return rows[0][0] else: raise Exception('Exception during getting last report date.') def sendNewDataNotification(self, result = None, testing = False): """ Sending notification reporting on new data being available since the last time new data was reported. :param result: list of dicts containing aggregation results as provided by MSGDataAggregator::aggregateNewData. :param testing: Use testing mode when True. """ self.logger.log('result {}'.format(result), 'debug') lastReportDate = self.lastReportDate(NOTIFICATION_HISTORY_TYPE) if not lastReportDate: lastReportDate = "never" if not result: msgBody = '\nNew data has NOT been aggregated in {}. No result ' \ 'was obtained. This is an error that should be ' \ 'investigated.'.format(self.connector.dbName) else: msgBody = '\nNew data has been aggregated in {}.'.format( self.connector.dbName) msgBody += '\n\n' for i in range(len(result)): msgBody += 'The new data count for type {} is {} readings' \ '.\n'.format(result[i].keys()[0], result[i].values()[0]) msgBody += '\n\n' msgBody += 'The last report date was %s.' % lastReportDate msgBody += '\n\n' self.notifier.sendNotificationEmail(msgBody, testing = testing) self.saveNotificationTime() def saveNotificationTime(self): """ Save a notification event to the notification history. """ cursor = self.cursor sql = """INSERT INTO "{}" ("notificationType", "notificationTime") VALUES ('{}', NOW())""".format(NOTIFICATION_HISTORY_TABLE, NOTIFICATION_HISTORY_TYPE) success = self.dbUtil.executeSQL(cursor, sql) self.conn.commit() if not success: raise Exception('Exception while saving the notification time.') def aggregateNewData(self): """ :return: list of dicts obtained from MSGDataAggregator::aggregateNewData. """ result = map(self.aggregator.aggregateNewData, self.rawTypes) self.logger.log('result {}'.format(result)) return result
class MSGEgaugeNewDataChecker(object): """ Provide notification of newly loaded MSG eGauge data. This uses notification type MSG_EGAUGE_SERVICE. """ def __init__(self): """ Constructor. """ print __name__ self.logger = SEKLogger(__name__) self.connector = MSGDBConnector() self.dbUtil = MSGDBUtil() self.notifier = MSGNotifier() self.configer = MSGConfiger() def newDataCount(self): """ Measure the amount of new data that is present since the last time new data was reported. """ cursor = self.connector.conn.cursor() tableName = 'EgaugeEnergyAutoload' lastTime = self.lastReportDate('MSG_EGAUGE_SERVICE') if lastTime is None: lastTime = '1900-01-01' sql = """SELECT COUNT(*) FROM "%s" WHERE datetime > '%s'""" % ( tableName, lastTime) success = self.dbUtil.executeSQL(cursor, sql) if success: rows = cursor.fetchall() if not rows[0][0]: return 0 else: return rows[0][0] else: # @todo Raise an exception. return None def lastReportDate(self, notificationType): """ Get the last time a notification was reported. :param notificationType: A string indicating the type of the notification. It is stored in the event history. :returns: datetime of last report date. """ cursor = self.connector.conn.cursor() sql = """SELECT MAX("notificationTime") FROM "%s" WHERE "notificationType" = '%s'""" % (NOTIFICATION_HISTORY_TABLE, notificationType) success = self.dbUtil.executeSQL(cursor, sql) if success: rows = cursor.fetchall() if not rows[0][0]: return None else: return rows[0][0] else: # @todo Raise an exception. return None def saveNotificationTime(self): """ Save the notification event to the notification history. """ cursor = self.connector.conn.cursor() sql = """INSERT INTO "%s" ("notificationType", "notificationTime") VALUES ('MSG_EGAUGE_SERVICE', NOW())""" % NOTIFICATION_HISTORY_TABLE success = self.dbUtil.executeSQL(cursor, sql) self.connector.conn.commit() if not success: # @todo Raise an exception. self.logger.log( 'An error occurred while saving the notification time.') def sendNewDataNotification(self, testing=False): """ Sending notification reporting on new data being available since the last time new data was reported. :param testing: Use testing mode when True. """ lastReportDate = self.lastReportDate('MSG_EGAUGE_SERVICE') if not lastReportDate: lastReportDate = "never" msgBody = '\nNew MSG eGauge data has been loaded to %s.' % self\ .connector.dbName msgBody += '\n\n' msgBody += 'The new data count is %s readings.' % self.newDataCount() msgBody += '\n\n' msgBody += 'The last report date was %s.' % lastReportDate msgBody += '\n\n' self.notifier.sendNotificationEmail(msgBody, testing=testing) self.saveNotificationTime()
data[i] = data[i].replace("'", "\'\'") data[i] = "'" + data[i] + "'" data = data + ["'" + getDeviceNameFromFileName(name) + "'"] sql = """INSERT INTO %s (%s) VALUES (%s)""" % ( table, ','.join(cols), ','.join(data)) success = dbUtil.executeSQL(cur, sql) if not success: anyFailure = True lineCnt += 1 conn.commit() msg = ("Processed %s lines.\n" % lineCnt) lineCnt = 0 sys.stderr.write(msg) msgBody += msg if not anyFailure: msg = "Finished inserting NREL transformer records.\n" sys.stderr.write(msg) msgBody += msg else: msg = "NREL transformer data records were NOT successfully loaded.\n" sys.stderr.write(msg) msgBody += msg if commandLineArgs.email: notifier.sendNotificationEmail(msgBody, testing = commandLineArgs.testing)
class NewDataAggregator(object): """ Perform aggregation of new data for a set of predefined data types (self .rawTypes). """ def __init__(self): """ Constructor. """ self.logger = SEKLogger(__name__, 'DEBUG') self.aggregator = MSGDataAggregator() self.notifier = MSGNotifier() self.rawTypes = [x.name for x in list(MSGAggregationTypes)] self.connector = MSGDBConnector() self.conn = self.connector.connectDB() self.cursor = self.conn.cursor() self.dbUtil = MSGDBUtil() def sendNewDataNotification(self, result=None, testing=False): """ Sending notification reporting on new data being available since the last time new data was reported. :param result: list of dicts containing aggregation results as provided by MSGDataAggregator::aggregateNewData. :param testing: Use testing mode when True. """ self.logger.log('result {}'.format(result), 'debug') lastReportDate = self.notifier.lastReportDate( MSGNotificationHistoryTypes.MSG_DATA_AGGREGATOR) if not lastReportDate: lastReportDate = "never" if not result: msgBody = '\nNew data has NOT been aggregated in {}. No result ' \ 'was obtained. This is an error that should be ' \ 'investigated.'.format(self.connector.dbName) else: msgBody = '\nNew data has been aggregated in {}.'.format( self.connector.dbName) msgBody += '\n\n' for i in range(len(result)): msgBody += 'The new data count for type {} is {} readings' \ '.\n'.format(result[i].keys()[0], result[i].values()[0]) msgBody += '\n\n' msgBody += 'The last report date was %s.' % lastReportDate msgBody += '\n\n' self.notifier.sendNotificationEmail(msgBody, testing=testing) self.notifier.recordNotificationEvent( MSGNotificationHistoryTypes.MSG_DATA_AGGREGATOR) def aggregateNewData(self): """ :return: list of dicts obtained from MSGDataAggregator::aggregateNewData. """ result = map(self.aggregator.aggregateNewData, self.rawTypes) self.logger.log('result {}'.format(result)) return result
class TestMECONotifier(unittest.TestCase): """ Unit tests for the MECO Notifier. """ def setUp(self): self.logger = SEKLogger(__name__) self.notifier = MSGNotifier() self.configer = MSGConfiger() def tearDown(self): pass def testInit(self): self.assertIsNotNone(self.notifier, "Notifier has been initialized.") def testEmailServer(self): """ Test connecting to the email server. """ errorOccurred = False user = self.configer.configOptionValue('Notifications', 'email_username') password = self.configer.configOptionValue('Notifications', 'email_password') server = smtplib.SMTP( self.configer.configOptionValue('Notifications', 'smtp_server_and_port')) try: server.starttls() except smtplib.SMTPException as detail: self.logger.log("Exception: {}".format(detail)) try: server.login(user, password) except smtplib.SMTPException as detail: self.logger.log("Exception: {}".format(detail)) self.assertFalse(errorOccurred, "No errors occurred during SMTP setup.") def testSendEmailNotification(self): """ Send a test notification by email. """ if SEND_EMAIL: success = self.notifier.sendNotificationEmail( 'This is a message from testSendEmailNotification.', testing=True) self.assertTrue( success, "Sending an email notification did not produce an" " exception.") else: self.assertTrue(True, "Email is not sent when SEND_EMAIL is False.") def testSendEmailAttachment(self): """ Send a test notification with attachment by email. """ if SEND_EMAIL: body = "Test message" testDataPath = self.configer.configOptionValue( 'Testing', 'test_data_path') file = os.path.join(testDataPath, 'graph.png') success = self.notifier.sendMailWithAttachments(body, [file], testing=True) success = (success != True) self.assertTrue( success, "Sending an email notification did not produce an" " exception.") else: self.assertTrue(True, "Email is not sent when SEND_EMAIL is False.")
data = data + [ "'" + getDeviceNameFromFileName(name) + "'" ] sql = """INSERT INTO %s (%s) VALUES (%s)""" % ( table, ','.join(cols), ','.join(data)) success = dbUtil.executeSQL(cur, sql) if not success: anyFailure = True lineCnt += 1 conn.commit() msg = ("Processed %s lines.\n" % lineCnt) lineCnt = 0 sys.stderr.write(msg) msgBody += msg if not anyFailure: msg = "Finished inserting NREL transformer records.\n" sys.stderr.write(msg) msgBody += msg else: msg = "NREL transformer data records were NOT successfully loaded.\n" sys.stderr.write(msg) msgBody += msg if commandLineArgs.email: notifier.sendNotificationEmail(msgBody, testing=commandLineArgs.testing)