class MSGEgaugeNewDataChecker(object):
"""
Provide notification of newly loaded MSG eGauge data.
This uses notification type MSG_EGAUGE_SERVICE.
"""
def __init__(self):
"""
Constructor.
"""
print __name__
self.logger = SEKLogger(__name__)
self.connector = MSGDBConnector()
self.dbUtil = MSGDBUtil()
self.notifier = MSGNotifier()
self.configer = MSGConfiger()
def newDataCount(self):
"""
Measure the amount of new data that is present since the last time
new data was reported.
"""
cursor = self.connector.conn.cursor()
tableName = 'EgaugeEnergyAutoload'
lastTime = self.lastReportDate('MSG_EGAUGE_SERVICE')
if lastTime is None:
lastTime = '1900-01-01'
sql = """SELECT COUNT(*) FROM "%s" WHERE datetime > '%s'""" % (
tableName, lastTime)
success = self.dbUtil.executeSQL(cursor, sql)
if success:
rows = cursor.fetchall()
if not rows[0][0]:
return 0
else:
return rows[0][0]
else:
# @todo Raise an exception.
return None
def lastReportDate(self, notificationType):
"""
Get the last time a notification was reported.
:param notificationType: A string indicating the type of the
notification. It is stored in the event history.
:returns: datetime of last report date.
"""
cursor = self.connector.conn.cursor()
sql = """SELECT MAX("notificationTime") FROM "%s" WHERE
"notificationType" = '%s'""" % (
NOTIFICATION_HISTORY_TABLE, notificationType)
success = self.dbUtil.executeSQL(cursor, sql)
if success:
rows = cursor.fetchall()
if not rows[0][0]:
return None
else:
return rows[0][0]
else:
# @todo Raise an exception.
return None
def saveNotificationTime(self):
"""
Save the notification event to the notification history.
"""
cursor = self.connector.conn.cursor()
sql = """INSERT INTO "%s" ("notificationType", "notificationTime")
VALUES ('MSG_EGAUGE_SERVICE', NOW())""" % NOTIFICATION_HISTORY_TABLE
success = self.dbUtil.executeSQL(cursor, sql)
self.connector.conn.commit()
if not success:
# @todo Raise an exception.
self.logger.log(
'An error occurred while saving the notification time.')
def sendNewDataNotification(self, testing = False):
"""
Sending notification reporting on new data being available since the
last time new data was reported.
:param testing: Use testing mode when True.
"""
lastReportDate = self.lastReportDate('MSG_EGAUGE_SERVICE')
if not lastReportDate:
lastReportDate = "never"
msgBody = '\nNew MSG eGauge data has been loaded to %s.' % self\
.connector.dbName
msgBody += '\n\n'
msgBody += 'The new data count is %s readings.' % self.newDataCount()
msgBody += '\n\n'
msgBody += 'The last report date was %s.' % lastReportDate
msgBody += '\n\n'
self.notifier.sendNotificationEmail(msgBody, testing = testing)
self.saveNotificationTime()
class NewDataAggregator(object):
"""
Perform aggregation of new data for a set of predefined data types (self
.rawTypes).
"""
def __init__(self):
"""
Constructor.
"""
self.logger = SEKLogger(__name__, 'DEBUG')
self.aggregator = MSGDataAggregator()
self.notifier = MSGNotifier()
self.rawTypes = [x.name for x in list(MSGAggregationTypes)]
self.connector = MSGDBConnector()
self.conn = self.connector.connectDB()
self.cursor = self.conn.cursor()
self.dbUtil = MSGDBUtil()
def sendNewDataNotification(self, result = None, testing = False):
"""
Sending notification reporting on new data being available since the
last time new data was reported.
:param result: list of dicts containing aggregation results as
provided by MSGDataAggregator::aggregateNewData.
:param testing: Use testing mode when True.
"""
self.logger.log('result {}'.format(result), 'debug')
lastReportDate = self.notifier.lastReportDate(
MSGNotificationHistoryTypes.MSG_DATA_AGGREGATOR)
if not lastReportDate:
lastReportDate = "never"
if not result:
msgBody = '\nNew data has NOT been aggregated in {}. No result ' \
'was obtained. This is an error that should be ' \
'investigated.'.format(self.connector.dbName)
else:
msgBody = '\nNew data has been aggregated in {}.'.format(
self.connector.dbName)
msgBody += '\n\n'
for i in range(len(result)):
msgBody += 'The new data count for type {} is {} readings' \
'.\n'.format(result[i].keys()[0],
result[i].values()[0])
msgBody += '\n\n'
msgBody += 'The last report date was %s.' % lastReportDate
msgBody += '\n\n'
self.notifier.sendNotificationEmail(msgBody, testing = testing)
self.notifier.recordNotificationEvent(
MSGNotificationHistoryTypes.MSG_DATA_AGGREGATOR)
def aggregateNewData(self):
"""
:return: list of dicts obtained from
MSGDataAggregator::aggregateNewData.
"""
result = map(self.aggregator.aggregateNewData, self.rawTypes)
self.logger.log('result {}'.format(result))
return result
class TestMECONotifier(unittest.TestCase):
"""
Unit tests for the MECO Notifier.
"""
def setUp(self):
self.logger = SEKLogger(__name__)
self.notifier = MSGNotifier()
self.configer = MSGConfiger()
def tearDown(self):
pass
def testInit(self):
self.assertIsNotNone(self.notifier, "Notifier has been initialized.")
def testEmailServer(self):
"""
Test connecting to the email server.
"""
errorOccurred = False
user = self.configer.configOptionValue("Notifications", "email_username")
password = self.configer.configOptionValue("Notifications", "email_password")
server = smtplib.SMTP(self.configer.configOptionValue("Notifications", "smtp_server_and_port"))
try:
server.starttls()
except smtplib.SMTPException as detail:
self.logger.log("Exception: {}".format(detail))
try:
server.login(user, password)
except smtplib.SMTPException as detail:
self.logger.log("Exception: {}".format(detail))
self.assertFalse(errorOccurred, "No errors occurred during SMTP setup.")
def testSendEmailNotification(self):
"""
Send a test notification by email.
"""
if SEND_EMAIL:
success = self.notifier.sendNotificationEmail(
"This is a message from testSendEmailNotification.", testing=True
)
self.assertTrue(success, "Sending an email notification did not produce an" " exception.")
else:
self.assertTrue(True, "Email is not sent when SEND_EMAIL is False.")
def testSendEmailAttachment(self):
"""
Send a test notification with attachment by email.
"""
if SEND_EMAIL:
body = "Test message"
testDataPath = self.configer.configOptionValue("Testing", "test_data_path")
file = os.path.join(testDataPath, "graph.png")
success = self.notifier.sendMailWithAttachments(body, [file], testing=True)
success = success != True
self.assertTrue(success, "Sending an email notification did not produce an" " exception.")
else:
self.assertTrue(True, "Email is not sent when SEND_EMAIL is False.")
if len(data[i]) == 0:
data[i] = 'NULL'
else:
data[i] = "'" + data[i] + "'"
sql = """INSERT INTO "LocationRecords" (%s) VALUES (%s)""" % (
','.join(cols), ','.join(data))
success = dbUtil.executeSQL(cur, sql)
if not success:
anyFailure = True
lineCnt += 1
conn.commit()
msg = ("Processed %s lines.\n" % lineCnt)
sys.stderr.write(msg)
msgBody += msg
if not anyFailure:
msg = "Finished inserting location records.\n"
sys.stderr.write(msg)
msgBody += msg
else:
msg = "Location records were NOT successfully loaded.\n"
sys.stderr.write(msg)
msgBody += msg
notifier.sendNotificationEmail(msgBody)
# PostgreSQL.
data[i] = data[i].replace("'", "\'\'")
data[i] = "'" + data[i] + "'"
sql = """INSERT INTO "MeterLocationHistory" (%s) VALUES (%s)""" % (
','.join(cols), ','.join(data))
logger.log("SQL: %s" % sql, 'debug')
success = dbUtil.executeSQL(cur, sql)
if not success:
anyFailure = True
lineCnt += 1
conn.commit()
msg = ("Processed %s lines.\n" % lineCnt)
sys.stderr.write(msg)
msgBody += msg
if not anyFailure:
msg = "Finished inserting Meter Location History records.\n"
sys.stderr.write(msg)
msgBody += msg
else:
msg = "Meter Location History records were NOT successfully loaded.\n"
sys.stderr.write(msg)
msgBody += msg
if COMMAND_LINE_ARGS.email:
notifier.sendNotificationEmail(msgBody, testing = COMMAND_LINE_ARGS.testing)
class NewDataAggregator(object):
"""
Perform aggregation of new data for a set of predefined data types (self
.rawTypes).
"""
def __init__(self):
"""
Constructor.
"""
self.logger = MSGLogger(__name__, 'DEBUG')
self.aggregator = MSGDataAggregator()
self.notifier = MSGNotifier()
self.rawTypes = ['weather', 'egauge', 'circuit', 'irradiance']
self.connector = MSGDBConnector()
self.conn = self.connector.connectDB()
self.cursor = self.conn.cursor()
self.dbUtil = MSGDBUtil()
def lastReportDate(self, notificationType):
"""
Get the last time a notification was reported.
:param notificationType: string indicating the type of the
notification. It is stored in the event history.
:returns: datetime of last report date.
"""
cursor = self.cursor
sql = """SELECT MAX("notificationTime") FROM "{}" WHERE
"notificationType" = '{}'""".format(NOTIFICATION_HISTORY_TABLE,
notificationType)
success = self.dbUtil.executeSQL(cursor, sql)
if success:
rows = cursor.fetchall()
if not rows[0][0]:
return None
else:
return rows[0][0]
else:
raise Exception('Exception during getting last report date.')
def sendNewDataNotification(self, result = None, testing = False):
"""
Sending notification reporting on new data being available since the
last time new data was reported.
:param result: list of dicts containing aggregation results as
provided by MSGDataAggregator::aggregateNewData.
:param testing: Use testing mode when True.
"""
self.logger.log('result {}'.format(result), 'debug')
lastReportDate = self.lastReportDate(NOTIFICATION_HISTORY_TYPE)
if not lastReportDate:
lastReportDate = "never"
if not result:
msgBody = '\nNew data has NOT been aggregated in {}. No result ' \
'was obtained. This is an error that should be ' \
'investigated.'.format(self.connector.dbName)
else:
msgBody = '\nNew data has been aggregated in {}.'.format(
self.connector.dbName)
msgBody += '\n\n'
for i in range(len(result)):
msgBody += 'The new data count for type {} is {} readings' \
'.\n'.format(result[i].keys()[0],
result[i].values()[0])
msgBody += '\n\n'
msgBody += 'The last report date was %s.' % lastReportDate
msgBody += '\n\n'
self.notifier.sendNotificationEmail(msgBody, testing = testing)
self.saveNotificationTime()
def saveNotificationTime(self):
"""
Save a notification event to the notification history.
"""
cursor = self.cursor
sql = """INSERT INTO "{}" ("notificationType", "notificationTime")
VALUES ('{}', NOW())""".format(NOTIFICATION_HISTORY_TABLE,
NOTIFICATION_HISTORY_TYPE)
success = self.dbUtil.executeSQL(cursor, sql)
self.conn.commit()
if not success:
raise Exception('Exception while saving the notification time.')
def aggregateNewData(self):
"""
:return: list of dicts obtained from
MSGDataAggregator::aggregateNewData.
"""
result = map(self.aggregator.aggregateNewData, self.rawTypes)
self.logger.log('result {}'.format(result))
return result
class MSGEgaugeNewDataChecker(object):
"""
Provide notification of newly loaded MSG eGauge data.
This uses notification type MSG_EGAUGE_SERVICE.
"""
def __init__(self):
"""
Constructor.
"""
print __name__
self.logger = SEKLogger(__name__)
self.connector = MSGDBConnector()
self.dbUtil = MSGDBUtil()
self.notifier = MSGNotifier()
self.configer = MSGConfiger()
def newDataCount(self):
"""
Measure the amount of new data that is present since the last time
new data was reported.
"""
cursor = self.connector.conn.cursor()
tableName = 'EgaugeEnergyAutoload'
lastTime = self.lastReportDate('MSG_EGAUGE_SERVICE')
if lastTime is None:
lastTime = '1900-01-01'
sql = """SELECT COUNT(*) FROM "%s" WHERE datetime > '%s'""" % (
tableName, lastTime)
success = self.dbUtil.executeSQL(cursor, sql)
if success:
rows = cursor.fetchall()
if not rows[0][0]:
return 0
else:
return rows[0][0]
else:
# @todo Raise an exception.
return None
def lastReportDate(self, notificationType):
"""
Get the last time a notification was reported.
:param notificationType: A string indicating the type of the
notification. It is stored in the event history.
:returns: datetime of last report date.
"""
cursor = self.connector.conn.cursor()
sql = """SELECT MAX("notificationTime") FROM "%s" WHERE
"notificationType" = '%s'""" % (NOTIFICATION_HISTORY_TABLE,
notificationType)
success = self.dbUtil.executeSQL(cursor, sql)
if success:
rows = cursor.fetchall()
if not rows[0][0]:
return None
else:
return rows[0][0]
else:
# @todo Raise an exception.
return None
def saveNotificationTime(self):
"""
Save the notification event to the notification history.
"""
cursor = self.connector.conn.cursor()
sql = """INSERT INTO "%s" ("notificationType", "notificationTime")
VALUES ('MSG_EGAUGE_SERVICE', NOW())""" % NOTIFICATION_HISTORY_TABLE
success = self.dbUtil.executeSQL(cursor, sql)
self.connector.conn.commit()
if not success:
# @todo Raise an exception.
self.logger.log(
'An error occurred while saving the notification time.')
def sendNewDataNotification(self, testing=False):
"""
Sending notification reporting on new data being available since the
last time new data was reported.
:param testing: Use testing mode when True.
"""
lastReportDate = self.lastReportDate('MSG_EGAUGE_SERVICE')
if not lastReportDate:
lastReportDate = "never"
msgBody = '\nNew MSG eGauge data has been loaded to %s.' % self\
.connector.dbName
msgBody += '\n\n'
msgBody += 'The new data count is %s readings.' % self.newDataCount()
msgBody += '\n\n'
msgBody += 'The last report date was %s.' % lastReportDate
msgBody += '\n\n'
self.notifier.sendNotificationEmail(msgBody, testing=testing)
self.saveNotificationTime()
data[i] = data[i].replace("'", "\'\'")
data[i] = "'" + data[i] + "'"
data = data + ["'" + getDeviceNameFromFileName(name) + "'"]
sql = """INSERT INTO %s (%s) VALUES (%s)""" % (
table, ','.join(cols), ','.join(data))
success = dbUtil.executeSQL(cur, sql)
if not success:
anyFailure = True
lineCnt += 1
conn.commit()
msg = ("Processed %s lines.\n" % lineCnt)
lineCnt = 0
sys.stderr.write(msg)
msgBody += msg
if not anyFailure:
msg = "Finished inserting NREL transformer records.\n"
sys.stderr.write(msg)
msgBody += msg
else:
msg = "NREL transformer data records were NOT successfully loaded.\n"
sys.stderr.write(msg)
msgBody += msg
if commandLineArgs.email:
notifier.sendNotificationEmail(msgBody, testing = commandLineArgs.testing)
if len(data[i]) == 0:
data[i] = 'NULL'
else:
data[i] = "'" + data[i] + "'"
sql = """INSERT INTO "LocationRecords" (%s) VALUES (%s)""" % (
','.join(cols), ','.join(data))
success = dbUtil.executeSQL(cur, sql)
if not success:
anyFailure = True
lineCnt += 1
conn.commit()
msg = ("Processed %s lines.\n" % lineCnt)
sys.stderr.write(msg)
msgBody += msg
if not anyFailure:
msg = "Finished inserting location records.\n"
sys.stderr.write(msg)
msgBody += msg
else:
msg = "Location records were NOT successfully loaded.\n"
sys.stderr.write(msg)
msgBody += msg
notifier.sendNotificationEmail(msgBody)
class NewDataAggregator(object):
"""
Perform aggregation of new data for a set of predefined data types (self
.rawTypes).
"""
def __init__(self):
"""
Constructor.
"""
self.logger = SEKLogger(__name__, 'DEBUG')
self.aggregator = MSGDataAggregator()
self.notifier = MSGNotifier()
self.rawTypes = [x.name for x in list(MSGAggregationTypes)]
self.connector = MSGDBConnector()
self.conn = self.connector.connectDB()
self.cursor = self.conn.cursor()
self.dbUtil = MSGDBUtil()
def sendNewDataNotification(self, result=None, testing=False):
"""
Sending notification reporting on new data being available since the
last time new data was reported.
:param result: list of dicts containing aggregation results as
provided by MSGDataAggregator::aggregateNewData.
:param testing: Use testing mode when True.
"""
self.logger.log('result {}'.format(result), 'debug')
lastReportDate = self.notifier.lastReportDate(
MSGNotificationHistoryTypes.MSG_DATA_AGGREGATOR)
if not lastReportDate:
lastReportDate = "never"
if not result:
msgBody = '\nNew data has NOT been aggregated in {}. No result ' \
'was obtained. This is an error that should be ' \
'investigated.'.format(self.connector.dbName)
else:
msgBody = '\nNew data has been aggregated in {}.'.format(
self.connector.dbName)
msgBody += '\n\n'
for i in range(len(result)):
msgBody += 'The new data count for type {} is {} readings' \
'.\n'.format(result[i].keys()[0],
result[i].values()[0])
msgBody += '\n\n'
msgBody += 'The last report date was %s.' % lastReportDate
msgBody += '\n\n'
self.notifier.sendNotificationEmail(msgBody, testing=testing)
self.notifier.recordNotificationEvent(
MSGNotificationHistoryTypes.MSG_DATA_AGGREGATOR)
def aggregateNewData(self):
"""
:return: list of dicts obtained from
MSGDataAggregator::aggregateNewData.
"""
result = map(self.aggregator.aggregateNewData, self.rawTypes)
self.logger.log('result {}'.format(result))
return result
class TestMECONotifier(unittest.TestCase):
"""
Unit tests for the MECO Notifier.
"""
def setUp(self):
self.logger = SEKLogger(__name__)
self.notifier = MSGNotifier()
self.configer = MSGConfiger()
def tearDown(self):
pass
def testInit(self):
self.assertIsNotNone(self.notifier, "Notifier has been initialized.")
def testEmailServer(self):
"""
Test connecting to the email server.
"""
errorOccurred = False
user = self.configer.configOptionValue('Notifications',
'email_username')
password = self.configer.configOptionValue('Notifications',
'email_password')
server = smtplib.SMTP(
self.configer.configOptionValue('Notifications',
'smtp_server_and_port'))
try:
server.starttls()
except smtplib.SMTPException as detail:
self.logger.log("Exception: {}".format(detail))
try:
server.login(user, password)
except smtplib.SMTPException as detail:
self.logger.log("Exception: {}".format(detail))
self.assertFalse(errorOccurred,
"No errors occurred during SMTP setup.")
def testSendEmailNotification(self):
"""
Send a test notification by email.
"""
if SEND_EMAIL:
success = self.notifier.sendNotificationEmail(
'This is a message from testSendEmailNotification.',
testing=True)
self.assertTrue(
success, "Sending an email notification did not produce an"
" exception.")
else:
self.assertTrue(True,
"Email is not sent when SEND_EMAIL is False.")
def testSendEmailAttachment(self):
"""
Send a test notification with attachment by email.
"""
if SEND_EMAIL:
body = "Test message"
testDataPath = self.configer.configOptionValue(
'Testing', 'test_data_path')
file = os.path.join(testDataPath, 'graph.png')
success = self.notifier.sendMailWithAttachments(body, [file],
testing=True)
success = (success != True)
self.assertTrue(
success, "Sending an email notification did not produce an"
" exception.")
else:
self.assertTrue(True,
"Email is not sent when SEND_EMAIL is False.")
data = data + [
"'" + getDeviceNameFromFileName(name) + "'"
]
sql = """INSERT INTO %s (%s) VALUES (%s)""" % (
table, ','.join(cols), ','.join(data))
success = dbUtil.executeSQL(cur, sql)
if not success:
anyFailure = True
lineCnt += 1
conn.commit()
msg = ("Processed %s lines.\n" % lineCnt)
lineCnt = 0
sys.stderr.write(msg)
msgBody += msg
if not anyFailure:
msg = "Finished inserting NREL transformer records.\n"
sys.stderr.write(msg)
msgBody += msg
else:
msg = "NREL transformer data records were NOT successfully loaded.\n"
sys.stderr.write(msg)
msgBody += msg
if commandLineArgs.email:
notifier.sendNotificationEmail(msgBody,
testing=commandLineArgs.testing)
