def __init__(self):
"""
Constructor.
"""
print __name__
self.logger = SEKLogger(__name__)
self.connector = MSGDBConnector()
self.dbUtil = MSGDBUtil()
self.notifier = MSGNotifier()
self.configer = MSGConfiger()
def __init__(self):
"""
Constructor.
"""
self.logger = SEKLogger(__name__, 'DEBUG')
self.aggregator = MSGDataAggregator()
self.notifier = MSGNotifier()
self.rawTypes = [x.name for x in list(MSGAggregationTypes)]
self.connector = MSGDBConnector()
self.conn = self.connector.connectDB()
self.cursor = self.conn.cursor()
self.dbUtil = MSGDBUtil()
def __init__(self):
"""
Constructor.
"""
print __name__
self.logger = SEKLogger(__name__)
self.connector = MSGDBConnector()
self.dbUtil = MSGDBUtil()
self.notifier = MSGNotifier()
self.configer = MSGConfiger()
def __init__(self):
"""
Constructor.
"""
self.logger = SEKLogger(__name__, 'DEBUG')
self.aggregator = MSGDataAggregator()
self.notifier = MSGNotifier()
self.rawTypes = [x.name for x in list(MSGAggregationTypes)]
self.connector = MSGDBConnector()
self.conn = self.connector.connectDB()
self.cursor = self.conn.cursor()
self.dbUtil = MSGDBUtil()
def __init__(self):
"""
Constructor.
"""
self.logger = MSGLogger(__name__, 'DEBUG')
self.aggregator = MSGDataAggregator()
self.notifier = MSGNotifier()
self.rawTypes = ['weather', 'egauge', 'circuit', 'irradiance']
self.connector = MSGDBConnector()
self.conn = self.connector.connectDB()
self.cursor = self.conn.cursor()
self.dbUtil = MSGDBUtil()
def __init__(self,
exitOnError=True,
commitOnEveryInsert=False,
testing=False):
"""
Constructor.
:param testing: if True, the testing DB will be connected instead of
the production DB.
"""
self.logger = SEKLogger(__name__, 'info')
self.configer = MSGConfiger()
self.conn = MSGDBConnector().connectDB()
self.cursor = self.conn.cursor()
self.dbUtil = MSGDBUtil()
self.notifier = MSGNotifier()
self.mathUtil = MSGMathUtil()
self.timeUtil = MSGTimeUtil()
self.nextMinuteCrossing = {}
self.nextMinuteCrossingWithoutSubkeys = None
self.exitOnError = exitOnError
self.commitOnEveryInsert = commitOnEveryInsert
section = 'Aggregation'
tableList = [
'irradiance', 'agg_irradiance', 'weather', 'agg_weather',
'circuit', 'agg_circuit', 'egauge', 'agg_egauge'
]
self.dataParams = {
'weather': ('agg_weather', 'timestamp', ''),
'egauge': ('agg_egauge', 'datetime', 'egauge_id'),
'circuit': ('agg_circuit', 'timestamp', 'circuit'),
'irradiance': ('agg_irradiance', 'timestamp', 'sensor_id')
}
self.columns = {}
# tables[datatype] gives the table name for datatype.
self.tables = {
t: self.configer.configOptionValue(section, '{}_table'.format(t))
for t in tableList
}
for t in self.tables.keys():
self.logger.log('t:{}'.format(t), 'DEBUG')
try:
self.columns[t] = self.dbUtil.columnsString(
self.cursor, self.tables[t])
except TypeError as error:
self.logger.log(
'Ignoring missing table: Error is {}.'.format(error),
'error')
def __init__(self):
"""
Constructor.
"""
self.logger = SEKLogger(__name__, 'DEBUG')
self.aggregator = MSGDataAggregator(exitOnError=False,
commitOnEveryInsert=True)
self.notifier = MSGNotifier()
# Available types are in ['weather', 'egauge', 'circuit', 'irradiance'].
self.rawTypes = ['']
self.connector = MSGDBConnector()
self.conn = self.connector.connectDB()
self.cursor = self.conn.cursor()
self.dbUtil = MSGDBUtil()
class TestMECONotifier(unittest.TestCase):
"""
Unit tests for the MECO Notifier.
"""
def setUp(self):
self.logger = SEKLogger(__name__)
self.notifier = MSGNotifier()
self.configer = MSGConfiger()
def tearDown(self):
pass
def testInit(self):
self.assertIsNotNone(self.notifier, "Notifier has been initialized.")
def testEmailServer(self):
"""
Test connecting to the email server.
"""
errorOccurred = False
user = self.configer.configOptionValue('Notifications',
'email_username')
password = self.configer.configOptionValue('Notifications',
'email_password')
server = smtplib.SMTP(
self.configer.configOptionValue('Notifications',
'smtp_server_and_port'))
try:
server.starttls()
except smtplib.SMTPException as detail:
self.logger.log("Exception: {}".format(detail))
try:
server.login(user, password)
except smtplib.SMTPException as detail:
self.logger.log("Exception: {}".format(detail))
self.assertFalse(errorOccurred,
"No errors occurred during SMTP setup.")
def testSendEmailNotification(self):
"""
Send a test notification by email.
"""
if SEND_EMAIL:
success = self.notifier.sendNotificationEmail(
'This is a message from testSendEmailNotification.',
testing=True)
self.assertTrue(
success, "Sending an email notification did not produce an"
" exception.")
else:
self.assertTrue(True,
"Email is not sent when SEND_EMAIL is False.")
def testSendEmailAttachment(self):
"""
Send a test notification with attachment by email.
"""
if SEND_EMAIL:
body = "Test message"
testDataPath = self.configer.configOptionValue(
'Testing', 'test_data_path')
file = os.path.join(testDataPath, 'graph.png')
success = self.notifier.sendMailWithAttachments(body, [file],
testing=True)
success = (success != True)
self.assertTrue(
success, "Sending an email notification did not produce an"
" exception.")
else:
self.assertTrue(True,
"Email is not sent when SEND_EMAIL is False.")
class TestMECONotifier(unittest.TestCase):
"""
Unit tests for the MECO Notifier.
"""
def setUp(self):
self.logger = SEKLogger(__name__)
self.notifier = MSGNotifier()
self.configer = MSGConfiger()
def tearDown(self):
pass
def testInit(self):
self.assertIsNotNone(self.notifier, "Notifier has been initialized.")
def testEmailServer(self):
"""
Test connecting to the email server.
"""
errorOccurred = False
user = self.configer.configOptionValue("Notifications", "email_username")
password = self.configer.configOptionValue("Notifications", "email_password")
server = smtplib.SMTP(self.configer.configOptionValue("Notifications", "smtp_server_and_port"))
try:
server.starttls()
except smtplib.SMTPException as detail:
self.logger.log("Exception: {}".format(detail))
try:
server.login(user, password)
except smtplib.SMTPException as detail:
self.logger.log("Exception: {}".format(detail))
self.assertFalse(errorOccurred, "No errors occurred during SMTP setup.")
def testSendEmailNotification(self):
"""
Send a test notification by email.
"""
if SEND_EMAIL:
success = self.notifier.sendNotificationEmail(
"This is a message from testSendEmailNotification.", testing=True
)
self.assertTrue(success, "Sending an email notification did not produce an" " exception.")
else:
self.assertTrue(True, "Email is not sent when SEND_EMAIL is False.")
def testSendEmailAttachment(self):
"""
Send a test notification with attachment by email.
"""
if SEND_EMAIL:
body = "Test message"
testDataPath = self.configer.configOptionValue("Testing", "test_data_path")
file = os.path.join(testDataPath, "graph.png")
success = self.notifier.sendMailWithAttachments(body, [file], testing=True)
success = success != True
self.assertTrue(success, "Sending an email notification did not produce an" " exception.")
else:
self.assertTrue(True, "Email is not sent when SEND_EMAIL is False.")
'specified in the local configuration file.')
commandLineArgs = argParser.parse_args()
if __name__ == '__main__':
processCommandLineArguments()
filename = COMMAND_LINE_ARGS.filename
success = True
anyFailure = False
connector = MSGDBConnector(testing = COMMAND_LINE_ARGS.testing)
conn = connector.connectDB()
cur = conn.cursor()
dbUtil = MSGDBUtil()
notifier = MSGNotifier()
msgBody = ''
configer = MSGConfiger()
if COMMAND_LINE_ARGS.testing:
dbName = configer.configOptionValue("Database", "testing_db_name")
else:
dbName = configer.configOptionValue("Database", "db_name")
msg = ("Loading Meter Location History data in file %s to database %s.\n" % (
filename, dbName))
sys.stderr.write(msg)
msgBody += msg
f = open(filename, "r")
reader = csv.reader(f)
class NewDataAggregator(object):
"""
Perform aggregation of new data for a set of predefined data types (self
.rawTypes).
"""
def __init__(self):
"""
Constructor.
"""
self.logger = SEKLogger(__name__, 'DEBUG')
self.aggregator = MSGDataAggregator()
self.notifier = MSGNotifier()
self.rawTypes = [x.name for x in list(MSGAggregationTypes)]
self.connector = MSGDBConnector()
self.conn = self.connector.connectDB()
self.cursor = self.conn.cursor()
self.dbUtil = MSGDBUtil()
def sendNewDataNotification(self, result = None, testing = False):
"""
Sending notification reporting on new data being available since the
last time new data was reported.
:param result: list of dicts containing aggregation results as
provided by MSGDataAggregator::aggregateNewData.
:param testing: Use testing mode when True.
"""
self.logger.log('result {}'.format(result), 'debug')
lastReportDate = self.notifier.lastReportDate(
MSGNotificationHistoryTypes.MSG_DATA_AGGREGATOR)
if not lastReportDate:
lastReportDate = "never"
if not result:
msgBody = '\nNew data has NOT been aggregated in {}. No result ' \
'was obtained. This is an error that should be ' \
'investigated.'.format(self.connector.dbName)
else:
msgBody = '\nNew data has been aggregated in {}.'.format(
self.connector.dbName)
msgBody += '\n\n'
for i in range(len(result)):
msgBody += 'The new data count for type {} is {} readings' \
'.\n'.format(result[i].keys()[0],
result[i].values()[0])
msgBody += '\n\n'
msgBody += 'The last report date was %s.' % lastReportDate
msgBody += '\n\n'
self.notifier.sendNotificationEmail(msgBody, testing = testing)
self.notifier.recordNotificationEvent(
MSGNotificationHistoryTypes.MSG_DATA_AGGREGATOR)
def aggregateNewData(self):
"""
:return: list of dicts obtained from
MSGDataAggregator::aggregateNewData.
"""
result = map(self.aggregator.aggregateNewData, self.rawTypes)
self.logger.log('result {}'.format(result))
return result
from msg_configer import MSGConfiger
from msg_notifier import MSGNotifier
from meco_plotting import MECOPlotting
from insertSingleMECOEnergyDataFile import Inserter
from sek.logger import SEKLogger
xmlGzCount = 0
xmlCount = 0
configer = MSGConfiger()
logger = SEKLogger(__name__, 'info')
binPath = MSGConfiger.configOptionValue(configer, "Executable Paths",
"msg_bin_path")
COMMAND_LINE_ARGS = None
global msgBody
notifier = MSGNotifier()
def processCommandLineArguments():
"""
Generate command-line arguments. Load them into global variable
COMMAND_LINE_ARGS.
"""
global COMMAND_LINE_ARGS
parser = argparse.ArgumentParser(
description='Perform recursive insertion of data contained in the '
'current directory to the MECO database specified in the '
'configuration file.')
parser.add_argument('--email',
action='store_true',
from msg_db_connector import MSGDBConnector
from msg_db_util import MSGDBUtil
from msg_notifier import MSGNotifier
from msg_configer import MSGConfiger
if __name__ == '__main__':
filename = sys.argv[1]
success = True
anyFailure = False
connector = MSGDBConnector()
conn = connector.connectDB()
cur = conn.cursor()
dbUtil = MSGDBUtil()
notifier = MSGNotifier()
msgBody = ''
configer = MSGConfiger()
dbName = configer.configOptionValue("Database", "db_name")
msg = ("Loading static location record data in file %s to database %s.\n" %
(filename, dbName))
sys.stderr.write(msg)
msgBody += msg
f = open(filename, "r")
reader = csv.reader(f)
# @todo verify column order
class MSGEgaugeNewDataChecker(object):
"""
Provide notification of newly loaded MSG eGauge data.
This uses notification type MSG_EGAUGE_SERVICE.
"""
def __init__(self):
"""
Constructor.
"""
print __name__
self.logger = SEKLogger(__name__)
self.connector = MSGDBConnector()
self.dbUtil = MSGDBUtil()
self.notifier = MSGNotifier()
self.configer = MSGConfiger()
def newDataCount(self):
"""
Measure the amount of new data that is present since the last time
new data was reported.
"""
cursor = self.connector.conn.cursor()
tableName = 'EgaugeEnergyAutoload'
lastTime = self.lastReportDate('MSG_EGAUGE_SERVICE')
if lastTime is None:
lastTime = '1900-01-01'
sql = """SELECT COUNT(*) FROM "%s" WHERE datetime > '%s'""" % (
tableName, lastTime)
success = self.dbUtil.executeSQL(cursor, sql)
if success:
rows = cursor.fetchall()
if not rows[0][0]:
return 0
else:
return rows[0][0]
else:
# @todo Raise an exception.
return None
def lastReportDate(self, notificationType):
"""
Get the last time a notification was reported.
:param notificationType: A string indicating the type of the
notification. It is stored in the event history.
:returns: datetime of last report date.
"""
cursor = self.connector.conn.cursor()
sql = """SELECT MAX("notificationTime") FROM "%s" WHERE
"notificationType" = '%s'""" % (NOTIFICATION_HISTORY_TABLE,
notificationType)
success = self.dbUtil.executeSQL(cursor, sql)
if success:
rows = cursor.fetchall()
if not rows[0][0]:
return None
else:
return rows[0][0]
else:
# @todo Raise an exception.
return None
def saveNotificationTime(self):
"""
Save the notification event to the notification history.
"""
cursor = self.connector.conn.cursor()
sql = """INSERT INTO "%s" ("notificationType", "notificationTime")
VALUES ('MSG_EGAUGE_SERVICE', NOW())""" % NOTIFICATION_HISTORY_TABLE
success = self.dbUtil.executeSQL(cursor, sql)
self.connector.conn.commit()
if not success:
# @todo Raise an exception.
self.logger.log(
'An error occurred while saving the notification time.')
def sendNewDataNotification(self, testing=False):
"""
Sending notification reporting on new data being available since the
last time new data was reported.
:param testing: Use testing mode when True.
"""
lastReportDate = self.lastReportDate('MSG_EGAUGE_SERVICE')
if not lastReportDate:
lastReportDate = "never"
msgBody = '\nNew MSG eGauge data has been loaded to %s.' % self\
.connector.dbName
msgBody += '\n\n'
msgBody += 'The new data count is %s readings.' % self.newDataCount()
msgBody += '\n\n'
msgBody += 'The last report date was %s.' % lastReportDate
msgBody += '\n\n'
self.notifier.sendNotificationEmail(msgBody, testing=testing)
self.saveNotificationTime()
import argparse
from sek.logger import SEKLogger
import gzip
from msg_noaa_weather_data_parser import MSGNOAAWeatherDataParser
from msg_noaa_weather_data_inserter import MSGNOAAWeatherDataInserter
from msg_db_connector import MSGDBConnector
from msg_time_util import MSGTimeUtil
from msg_noaa_weather_data_util import MSGWeatherDataUtil
configer = MSGConfiger()
logger = SEKLogger(__name__, 'info')
binPath = MSGConfiger.configOptionValue(configer, "Executable Paths",
"msg_bin_path")
COMMAND_LINE_ARGS = None
msgBody = ''
notifier = MSGNotifier()
dataParser = MSGNOAAWeatherDataParser()
inserter = MSGNOAAWeatherDataInserter()
timeUtil = MSGTimeUtil()
def processCommandLineArguments():
global COMMAND_LINE_ARGS
argParser = argparse.ArgumentParser(
description='Perform recursive insertion of compressed weather data'
' contained in the current directory to the MECO '
'database specified in the configuration file.')
argParser.add_argument('--email',
action='store_true',
default=False,
help='Send email notification if this flag is '
class NewDataAggregator(object):
"""
Perform aggregation of new data for a set of predefined data types (self
.rawTypes).
"""
def __init__(self):
"""
Constructor.
"""
self.logger = SEKLogger(__name__, 'DEBUG')
self.aggregator = MSGDataAggregator()
self.notifier = MSGNotifier()
self.rawTypes = [x.name for x in list(MSGAggregationTypes)]
self.connector = MSGDBConnector()
self.conn = self.connector.connectDB()
self.cursor = self.conn.cursor()
self.dbUtil = MSGDBUtil()
def sendNewDataNotification(self, result=None, testing=False):
"""
Sending notification reporting on new data being available since the
last time new data was reported.
:param result: list of dicts containing aggregation results as
provided by MSGDataAggregator::aggregateNewData.
:param testing: Use testing mode when True.
"""
self.logger.log('result {}'.format(result), 'debug')
lastReportDate = self.notifier.lastReportDate(
MSGNotificationHistoryTypes.MSG_DATA_AGGREGATOR)
if not lastReportDate:
lastReportDate = "never"
if not result:
msgBody = '\nNew data has NOT been aggregated in {}. No result ' \
'was obtained. This is an error that should be ' \
'investigated.'.format(self.connector.dbName)
else:
msgBody = '\nNew data has been aggregated in {}.'.format(
self.connector.dbName)
msgBody += '\n\n'
for i in range(len(result)):
msgBody += 'The new data count for type {} is {} readings' \
'.\n'.format(result[i].keys()[0],
result[i].values()[0])
msgBody += '\n\n'
msgBody += 'The last report date was %s.' % lastReportDate
msgBody += '\n\n'
self.notifier.sendNotificationEmail(msgBody, testing=testing)
self.notifier.recordNotificationEvent(
MSGNotificationHistoryTypes.MSG_DATA_AGGREGATOR)
def aggregateNewData(self):
"""
:return: list of dicts obtained from
MSGDataAggregator::aggregateNewData.
"""
result = map(self.aggregator.aggregateNewData, self.rawTypes)
self.logger.log('result {}'.format(result))
return result
parser.add_argument('--dbname', help='Database file to be uploaded.')
parser.add_argument('--fullpath',
help='Full path to database file to be uploaded.')
parser.add_argument('--testing', action='store_true', default=False)
COMMAND_LINE_ARGS = parser.parse_args()
if __name__ == '__main__':
logger = SEKLogger(__name__, 'INFO')
logger.log("Exporting DBs to cloud.")
processCommandLineArguments()
exporter = MSGDBExporter()
notifier = MSGNotifier()
exporter.logger.shouldRecord = True
startTime = time.time()
dbs = exporter.configer.configOptionValue('Export',
'dbs_to_export').split(',')
fileIDs = exporter.exportDBs(databases=dbs,
toCloud=True,
testing=COMMAND_LINE_ARGS.testing,
deleteOutdated=True)
wallTime = time.time() - startTime
wallTimeMin = int(wallTime / 60.0)
wallTimeSec = (wallTime - wallTimeMin * 60.0)
if len(fileIDs) == len(dbs):
class NewDataAggregator(object):
"""
Perform aggregation of new data for a set of predefined data types (self
.rawTypes).
"""
def __init__(self):
"""
Constructor.
"""
self.logger = MSGLogger(__name__, 'DEBUG')
self.aggregator = MSGDataAggregator()
self.notifier = MSGNotifier()
self.rawTypes = ['weather', 'egauge', 'circuit', 'irradiance']
self.connector = MSGDBConnector()
self.conn = self.connector.connectDB()
self.cursor = self.conn.cursor()
self.dbUtil = MSGDBUtil()
def lastReportDate(self, notificationType):
"""
Get the last time a notification was reported.
:param notificationType: string indicating the type of the
notification. It is stored in the event history.
:returns: datetime of last report date.
"""
cursor = self.cursor
sql = """SELECT MAX("notificationTime") FROM "{}" WHERE
"notificationType" = '{}'""".format(NOTIFICATION_HISTORY_TABLE,
notificationType)
success = self.dbUtil.executeSQL(cursor, sql)
if success:
rows = cursor.fetchall()
if not rows[0][0]:
return None
else:
return rows[0][0]
else:
raise Exception('Exception during getting last report date.')
def sendNewDataNotification(self, result = None, testing = False):
"""
Sending notification reporting on new data being available since the
last time new data was reported.
:param result: list of dicts containing aggregation results as
provided by MSGDataAggregator::aggregateNewData.
:param testing: Use testing mode when True.
"""
self.logger.log('result {}'.format(result), 'debug')
lastReportDate = self.lastReportDate(NOTIFICATION_HISTORY_TYPE)
if not lastReportDate:
lastReportDate = "never"
if not result:
msgBody = '\nNew data has NOT been aggregated in {}. No result ' \
'was obtained. This is an error that should be ' \
'investigated.'.format(self.connector.dbName)
else:
msgBody = '\nNew data has been aggregated in {}.'.format(
self.connector.dbName)
msgBody += '\n\n'
for i in range(len(result)):
msgBody += 'The new data count for type {} is {} readings' \
'.\n'.format(result[i].keys()[0],
result[i].values()[0])
msgBody += '\n\n'
msgBody += 'The last report date was %s.' % lastReportDate
msgBody += '\n\n'
self.notifier.sendNotificationEmail(msgBody, testing = testing)
self.saveNotificationTime()
def saveNotificationTime(self):
"""
Save a notification event to the notification history.
"""
cursor = self.cursor
sql = """INSERT INTO "{}" ("notificationType", "notificationTime")
VALUES ('{}', NOW())""".format(NOTIFICATION_HISTORY_TABLE,
NOTIFICATION_HISTORY_TYPE)
success = self.dbUtil.executeSQL(cursor, sql)
self.conn.commit()
if not success:
raise Exception('Exception while saving the notification time.')
def aggregateNewData(self):
"""
:return: list of dicts obtained from
MSGDataAggregator::aggregateNewData.
"""
result = map(self.aggregator.aggregateNewData, self.rawTypes)
self.logger.log('result {}'.format(result))
return result
elif 'ST534' in name:
return 'ST534'
else:
return False
if __name__ == '__main__':
processCommandLineArguments()
success = True
anyFailure = False
connector = MSGDBConnector(testing = commandLineArgs.testing)
conn = connector.connectDB()
cur = conn.cursor()
dbUtil = MSGDBUtil()
notifier = MSGNotifier()
msgBody = ''
configer = MSGConfiger()
if commandLineArgs.testing:
dbName = configer.configOptionValue("Database", "testing_db_name")
else:
dbName = "meco_v3"
for root, dirs, files in os.walk(os.getcwd()):
# For each file in the list of found files.
for name in files:
nameAndPath = join(root, name)
if not '.csv' in name.lower(): # Skip all non-CSV files.
continue
msg = ("Loading NREL xformer data in file %s to database %s.\n" % (
from msg_db_connector import MSGDBConnector
from msg_db_util import MSGDBUtil
from msg_notifier import MSGNotifier
from msg_configer import MSGConfiger
if __name__ == '__main__':
filename = sys.argv[1]
success = True
anyFailure = False
connector = MSGDBConnector()
conn = connector.connectDB()
cur = conn.cursor()
dbUtil = MSGDBUtil()
notifier = MSGNotifier()
msgBody = ''
configer = MSGConfiger()
dbName = configer.configOptionValue("Database", "db_name")
msg = ("Loading static location record data in file %s to database %s.\n" % (
filename, dbName))
sys.stderr.write(msg)
msgBody += msg
f = open(filename, "r")
reader = csv.reader(f)
# @todo verify column order
def setUp(self):
self.logger = SEKLogger(__name__)
self.notifier = MSGNotifier()
self.configer = MSGConfiger()
def setUp(self):
self.logger = SEKLogger(__name__)
self.notifier = MSGNotifier()
self.configer = MSGConfiger()
class MSGEgaugeNewDataChecker(object):
"""
Provide notification of newly loaded MSG eGauge data.
This uses notification type MSG_EGAUGE_SERVICE.
"""
def __init__(self):
"""
Constructor.
"""
print __name__
self.logger = SEKLogger(__name__)
self.connector = MSGDBConnector()
self.dbUtil = MSGDBUtil()
self.notifier = MSGNotifier()
self.configer = MSGConfiger()
def newDataCount(self):
"""
Measure the amount of new data that is present since the last time
new data was reported.
"""
cursor = self.connector.conn.cursor()
tableName = 'EgaugeEnergyAutoload'
lastTime = self.lastReportDate('MSG_EGAUGE_SERVICE')
if lastTime is None:
lastTime = '1900-01-01'
sql = """SELECT COUNT(*) FROM "%s" WHERE datetime > '%s'""" % (
tableName, lastTime)
success = self.dbUtil.executeSQL(cursor, sql)
if success:
rows = cursor.fetchall()
if not rows[0][0]:
return 0
else:
return rows[0][0]
else:
# @todo Raise an exception.
return None
def lastReportDate(self, notificationType):
"""
Get the last time a notification was reported.
:param notificationType: A string indicating the type of the
notification. It is stored in the event history.
:returns: datetime of last report date.
"""
cursor = self.connector.conn.cursor()
sql = """SELECT MAX("notificationTime") FROM "%s" WHERE
"notificationType" = '%s'""" % (
NOTIFICATION_HISTORY_TABLE, notificationType)
success = self.dbUtil.executeSQL(cursor, sql)
if success:
rows = cursor.fetchall()
if not rows[0][0]:
return None
else:
return rows[0][0]
else:
# @todo Raise an exception.
return None
def saveNotificationTime(self):
"""
Save the notification event to the notification history.
"""
cursor = self.connector.conn.cursor()
sql = """INSERT INTO "%s" ("notificationType", "notificationTime")
VALUES ('MSG_EGAUGE_SERVICE', NOW())""" % NOTIFICATION_HISTORY_TABLE
success = self.dbUtil.executeSQL(cursor, sql)
self.connector.conn.commit()
if not success:
# @todo Raise an exception.
self.logger.log(
'An error occurred while saving the notification time.')
def sendNewDataNotification(self, testing = False):
"""
Sending notification reporting on new data being available since the
last time new data was reported.
:param testing: Use testing mode when True.
"""
lastReportDate = self.lastReportDate('MSG_EGAUGE_SERVICE')
if not lastReportDate:
lastReportDate = "never"
msgBody = '\nNew MSG eGauge data has been loaded to %s.' % self\
.connector.dbName
msgBody += '\n\n'
msgBody += 'The new data count is %s readings.' % self.newDataCount()
msgBody += '\n\n'
msgBody += 'The last report date was %s.' % lastReportDate
msgBody += '\n\n'
self.notifier.sendNotificationEmail(msgBody, testing = testing)
self.saveNotificationTime()
from sek.logger import SEKLogger
import gzip
from msg_noaa_weather_data_parser import MSGNOAAWeatherDataParser
from msg_noaa_weather_data_inserter import MSGNOAAWeatherDataInserter
from msg_db_connector import MSGDBConnector
from msg_time_util import MSGTimeUtil
from msg_noaa_weather_data_util import MSGWeatherDataUtil
configer = MSGConfiger()
logger = SEKLogger(__name__, 'info')
binPath = MSGConfiger.configOptionValue(configer, "Executable Paths",
"msg_bin_path")
COMMAND_LINE_ARGS = None
msgBody = ''
notifier = MSGNotifier()
dataParser = MSGNOAAWeatherDataParser()
inserter = MSGNOAAWeatherDataInserter()
timeUtil = MSGTimeUtil()
def processCommandLineArguments():
global COMMAND_LINE_ARGS
argParser = argparse.ArgumentParser(
description = 'Perform recursive insertion of compressed weather data'
' contained in the current directory to the MECO '
'database specified in the configuration file.')
argParser.add_argument('--email', action = 'store_true', default = False,
help = 'Send email notification if this flag is '
'specified.')
argParser.add_argument('--testing', action = 'store_true', default = False,
from msg_configer import MSGConfiger
from msg_notifier import MSGNotifier
from meco_plotting import MECOPlotting
from insertSingleMECOEnergyDataFile import Inserter
from sek.logger import SEKLogger
xmlGzCount = 0
xmlCount = 0
configer = MSGConfiger()
logger = SEKLogger(__name__, 'info')
binPath = MSGConfiger.configOptionValue(configer, "Executable Paths",
"msg_bin_path")
COMMAND_LINE_ARGS = None
global msgBody
notifier = MSGNotifier()
def processCommandLineArguments():
"""
Generate command-line arguments. Load them into global variable
COMMAND_LINE_ARGS.
"""
global COMMAND_LINE_ARGS
parser = argparse.ArgumentParser(
description = 'Perform recursive insertion of data contained in the '
'current directory to the MECO database specified in the '
'configuration file.')
parser.add_argument('--email', action = 'store_true', default = False,
help = 'Send email notification if this flag is '
return 'ST534'
else:
return False
if __name__ == '__main__':
processCommandLineArguments()
success = True
anyFailure = False
connector = MSGDBConnector(testing=commandLineArgs.testing)
conn = connector.connectDB()
cur = conn.cursor()
dbUtil = MSGDBUtil()
notifier = MSGNotifier()
msgBody = ''
configer = MSGConfiger()
if commandLineArgs.testing:
dbName = configer.configOptionValue("Database", "testing_db_name")
else:
dbName = "meco_v3"
for root, dirs, files in os.walk(os.getcwd()):
# For each file in the list of found files.
for name in files:
nameAndPath = join(root, name)
if not '.csv' in name.lower(): # Skip all non-CSV files.
continue
msg = ("Loading NREL xformer data in file %s to database %s.\n" %