Example #1
0
    def __init__(self):
        """
        Constructor.
        """

        print __name__
        self.logger = SEKLogger(__name__)
        self.connector = MSGDBConnector()
        self.dbUtil = MSGDBUtil()
        self.notifier = MSGNotifier()
        self.configer = MSGConfiger()
Example #2
0
 def __init__(self):
     """
     Constructor.
     """
     self.logger = SEKLogger(__name__, 'DEBUG')
     self.aggregator = MSGDataAggregator()
     self.notifier = MSGNotifier()
     self.rawTypes = [x.name for x in list(MSGAggregationTypes)]
     self.connector = MSGDBConnector()
     self.conn = self.connector.connectDB()
     self.cursor = self.conn.cursor()
     self.dbUtil = MSGDBUtil()
    def __init__(self):
        """
        Constructor.
        """

        print __name__
        self.logger = SEKLogger(__name__)
        self.connector = MSGDBConnector()
        self.dbUtil = MSGDBUtil()
        self.notifier = MSGNotifier()
        self.configer = MSGConfiger()
 def __init__(self):
     """
     Constructor.
     """
     self.logger = SEKLogger(__name__, 'DEBUG')
     self.aggregator = MSGDataAggregator()
     self.notifier = MSGNotifier()
     self.rawTypes = [x.name for x in list(MSGAggregationTypes)]
     self.connector = MSGDBConnector()
     self.conn = self.connector.connectDB()
     self.cursor = self.conn.cursor()
     self.dbUtil = MSGDBUtil()
 def __init__(self):
     """
     Constructor.
     """
     self.logger = MSGLogger(__name__, 'DEBUG')
     self.aggregator = MSGDataAggregator()
     self.notifier = MSGNotifier()
     self.rawTypes = ['weather', 'egauge', 'circuit', 'irradiance']
     self.connector = MSGDBConnector()
     self.conn = self.connector.connectDB()
     self.cursor = self.conn.cursor()
     self.dbUtil = MSGDBUtil()
    def __init__(self,
                 exitOnError=True,
                 commitOnEveryInsert=False,
                 testing=False):
        """
        Constructor.

        :param testing: if True, the testing DB will be connected instead of
        the production DB.
        """

        self.logger = SEKLogger(__name__, 'info')
        self.configer = MSGConfiger()
        self.conn = MSGDBConnector().connectDB()
        self.cursor = self.conn.cursor()
        self.dbUtil = MSGDBUtil()
        self.notifier = MSGNotifier()
        self.mathUtil = MSGMathUtil()
        self.timeUtil = MSGTimeUtil()
        self.nextMinuteCrossing = {}
        self.nextMinuteCrossingWithoutSubkeys = None
        self.exitOnError = exitOnError
        self.commitOnEveryInsert = commitOnEveryInsert
        section = 'Aggregation'
        tableList = [
            'irradiance', 'agg_irradiance', 'weather', 'agg_weather',
            'circuit', 'agg_circuit', 'egauge', 'agg_egauge'
        ]
        self.dataParams = {
            'weather': ('agg_weather', 'timestamp', ''),
            'egauge': ('agg_egauge', 'datetime', 'egauge_id'),
            'circuit': ('agg_circuit', 'timestamp', 'circuit'),
            'irradiance': ('agg_irradiance', 'timestamp', 'sensor_id')
        }
        self.columns = {}

        # tables[datatype] gives the table name for datatype.
        self.tables = {
            t: self.configer.configOptionValue(section, '{}_table'.format(t))
            for t in tableList
        }

        for t in self.tables.keys():
            self.logger.log('t:{}'.format(t), 'DEBUG')
            try:
                self.columns[t] = self.dbUtil.columnsString(
                    self.cursor, self.tables[t])
            except TypeError as error:
                self.logger.log(
                    'Ignoring missing table: Error is {}.'.format(error),
                    'error')
Example #7
0
    def __init__(self):
        """
        Constructor.
        """
        self.logger = SEKLogger(__name__, 'DEBUG')
        self.aggregator = MSGDataAggregator(exitOnError=False,
                                            commitOnEveryInsert=True)
        self.notifier = MSGNotifier()

        # Available types are in ['weather', 'egauge', 'circuit', 'irradiance'].
        self.rawTypes = ['']
        self.connector = MSGDBConnector()
        self.conn = self.connector.connectDB()
        self.cursor = self.conn.cursor()
        self.dbUtil = MSGDBUtil()
Example #8
0
class TestMECONotifier(unittest.TestCase):
    """
    Unit tests for the MECO Notifier.
    """
    def setUp(self):
        self.logger = SEKLogger(__name__)
        self.notifier = MSGNotifier()
        self.configer = MSGConfiger()

    def tearDown(self):
        pass

    def testInit(self):
        self.assertIsNotNone(self.notifier, "Notifier has been initialized.")

    def testEmailServer(self):
        """
        Test connecting to the email server.
        """

        errorOccurred = False
        user = self.configer.configOptionValue('Notifications',
                                               'email_username')
        password = self.configer.configOptionValue('Notifications',
                                                   'email_password')

        server = smtplib.SMTP(
            self.configer.configOptionValue('Notifications',
                                            'smtp_server_and_port'))

        try:
            server.starttls()
        except smtplib.SMTPException as detail:
            self.logger.log("Exception: {}".format(detail))

        try:
            server.login(user, password)
        except smtplib.SMTPException as detail:
            self.logger.log("Exception: {}".format(detail))

        self.assertFalse(errorOccurred,
                         "No errors occurred during SMTP setup.")

    def testSendEmailNotification(self):
        """
        Send a test notification by email.
        """

        if SEND_EMAIL:
            success = self.notifier.sendNotificationEmail(
                'This is a message from testSendEmailNotification.',
                testing=True)
            self.assertTrue(
                success, "Sending an email notification did not produce an"
                " exception.")
        else:
            self.assertTrue(True,
                            "Email is not sent when SEND_EMAIL is False.")

    def testSendEmailAttachment(self):
        """
        Send a test notification with attachment by email.
        """

        if SEND_EMAIL:
            body = "Test message"
            testDataPath = self.configer.configOptionValue(
                'Testing', 'test_data_path')
            file = os.path.join(testDataPath, 'graph.png')
            success = self.notifier.sendMailWithAttachments(body, [file],
                                                            testing=True)
            success = (success != True)
            self.assertTrue(
                success, "Sending an email notification did not produce an"
                " exception.")
        else:
            self.assertTrue(True,
                            "Email is not sent when SEND_EMAIL is False.")
class TestMECONotifier(unittest.TestCase):
    """
    Unit tests for the MECO Notifier.
    """

    def setUp(self):
        self.logger = SEKLogger(__name__)
        self.notifier = MSGNotifier()
        self.configer = MSGConfiger()

    def tearDown(self):
        pass

    def testInit(self):
        self.assertIsNotNone(self.notifier, "Notifier has been initialized.")

    def testEmailServer(self):
        """
        Test connecting to the email server.
        """

        errorOccurred = False
        user = self.configer.configOptionValue("Notifications", "email_username")
        password = self.configer.configOptionValue("Notifications", "email_password")

        server = smtplib.SMTP(self.configer.configOptionValue("Notifications", "smtp_server_and_port"))

        try:
            server.starttls()
        except smtplib.SMTPException as detail:
            self.logger.log("Exception: {}".format(detail))

        try:
            server.login(user, password)
        except smtplib.SMTPException as detail:
            self.logger.log("Exception: {}".format(detail))

        self.assertFalse(errorOccurred, "No errors occurred during SMTP setup.")

    def testSendEmailNotification(self):
        """
        Send a test notification by email.
        """

        if SEND_EMAIL:
            success = self.notifier.sendNotificationEmail(
                "This is a message from testSendEmailNotification.", testing=True
            )
            self.assertTrue(success, "Sending an email notification did not produce an" " exception.")
        else:
            self.assertTrue(True, "Email is not sent when SEND_EMAIL is False.")

    def testSendEmailAttachment(self):
        """
        Send a test notification with attachment by email.
        """

        if SEND_EMAIL:
            body = "Test message"
            testDataPath = self.configer.configOptionValue("Testing", "test_data_path")
            file = os.path.join(testDataPath, "graph.png")
            success = self.notifier.sendMailWithAttachments(body, [file], testing=True)
            success = success != True
            self.assertTrue(success, "Sending an email notification did not produce an" " exception.")
        else:
            self.assertTrue(True, "Email is not sent when SEND_EMAIL is False.")
                                  'specified in the local configuration file.')
    commandLineArgs = argParser.parse_args()

if __name__ == '__main__':

    processCommandLineArguments()

    filename = COMMAND_LINE_ARGS.filename

    success = True
    anyFailure = False
    connector = MSGDBConnector(testing = COMMAND_LINE_ARGS.testing)
    conn = connector.connectDB()
    cur = conn.cursor()
    dbUtil = MSGDBUtil()
    notifier = MSGNotifier()
    msgBody = ''
    configer = MSGConfiger()

    if COMMAND_LINE_ARGS.testing:
        dbName = configer.configOptionValue("Database", "testing_db_name")
    else:
        dbName = configer.configOptionValue("Database", "db_name")

    msg = ("Loading Meter Location History data in file %s to database %s.\n" % (
        filename, dbName))
    sys.stderr.write(msg)
    msgBody += msg

    f = open(filename, "r")
    reader = csv.reader(f)
class NewDataAggregator(object):
    """
    Perform aggregation of new data for a set of predefined data types (self
    .rawTypes).
    """

    def __init__(self):
        """
        Constructor.
        """
        self.logger = SEKLogger(__name__, 'DEBUG')
        self.aggregator = MSGDataAggregator()
        self.notifier = MSGNotifier()
        self.rawTypes = [x.name for x in list(MSGAggregationTypes)]
        self.connector = MSGDBConnector()
        self.conn = self.connector.connectDB()
        self.cursor = self.conn.cursor()
        self.dbUtil = MSGDBUtil()


    def sendNewDataNotification(self, result = None, testing = False):
        """
        Sending notification reporting on new data being available since the
        last time new data was reported.

        :param result: list of dicts containing aggregation results as
        provided by MSGDataAggregator::aggregateNewData.
        :param testing: Use testing mode when True.
        """

        self.logger.log('result {}'.format(result), 'debug')

        lastReportDate = self.notifier.lastReportDate(
            MSGNotificationHistoryTypes.MSG_DATA_AGGREGATOR)

        if not lastReportDate:
            lastReportDate = "never"

        if not result:
            msgBody = '\nNew data has NOT been aggregated in {}. No result ' \
                      'was obtained. This is an error that should be ' \
                      'investigated.'.format(self.connector.dbName)
        else:
            msgBody = '\nNew data has been aggregated in {}.'.format(
                self.connector.dbName)
            msgBody += '\n\n'
            for i in range(len(result)):
                msgBody += 'The new data count for type {} is {} readings' \
                           '.\n'.format(result[i].keys()[0],
                                        result[i].values()[0])
            msgBody += '\n\n'
            msgBody += 'The last report date was %s.' % lastReportDate
            msgBody += '\n\n'
        self.notifier.sendNotificationEmail(msgBody, testing = testing)
        self.notifier.recordNotificationEvent(
            MSGNotificationHistoryTypes.MSG_DATA_AGGREGATOR)


    def aggregateNewData(self):
        """
        :return: list of dicts obtained from
        MSGDataAggregator::aggregateNewData.
        """

        result = map(self.aggregator.aggregateNewData, self.rawTypes)

        self.logger.log('result {}'.format(result))
        return result
Example #12
0
from msg_configer import MSGConfiger
from msg_notifier import MSGNotifier
from meco_plotting import MECOPlotting
from insertSingleMECOEnergyDataFile import Inserter
from sek.logger import SEKLogger

xmlGzCount = 0
xmlCount = 0
configer = MSGConfiger()
logger = SEKLogger(__name__, 'info')
binPath = MSGConfiger.configOptionValue(configer, "Executable Paths",
                                        "msg_bin_path")
COMMAND_LINE_ARGS = None
global msgBody
notifier = MSGNotifier()


def processCommandLineArguments():
    """
    Generate command-line arguments. Load them into global variable
    COMMAND_LINE_ARGS.
    """

    global COMMAND_LINE_ARGS
    parser = argparse.ArgumentParser(
        description='Perform recursive insertion of data contained in the '
        'current directory to the MECO database specified in the '
        'configuration file.')
    parser.add_argument('--email',
                        action='store_true',
Example #13
0
from msg_db_connector import MSGDBConnector
from msg_db_util import MSGDBUtil
from msg_notifier import MSGNotifier
from msg_configer import MSGConfiger

if __name__ == '__main__':

    filename = sys.argv[1]

    success = True
    anyFailure = False
    connector = MSGDBConnector()
    conn = connector.connectDB()
    cur = conn.cursor()
    dbUtil = MSGDBUtil()
    notifier = MSGNotifier()
    msgBody = ''
    configer = MSGConfiger()

    dbName = configer.configOptionValue("Database", "db_name")

    msg = ("Loading static location record data in file %s to database %s.\n" %
           (filename, dbName))
    sys.stderr.write(msg)
    msgBody += msg

    f = open(filename, "r")
    reader = csv.reader(f)

    # @todo verify column order
Example #14
0
class MSGEgaugeNewDataChecker(object):
    """
    Provide notification of newly loaded MSG eGauge data.

    This uses notification type MSG_EGAUGE_SERVICE.
    """
    def __init__(self):
        """
        Constructor.
        """

        print __name__
        self.logger = SEKLogger(__name__)
        self.connector = MSGDBConnector()
        self.dbUtil = MSGDBUtil()
        self.notifier = MSGNotifier()
        self.configer = MSGConfiger()

    def newDataCount(self):
        """
        Measure the amount of new data that is present since the last time
        new data was reported.
        """

        cursor = self.connector.conn.cursor()
        tableName = 'EgaugeEnergyAutoload'
        lastTime = self.lastReportDate('MSG_EGAUGE_SERVICE')
        if lastTime is None:
            lastTime = '1900-01-01'
        sql = """SELECT COUNT(*) FROM "%s" WHERE datetime > '%s'""" % (
            tableName, lastTime)

        success = self.dbUtil.executeSQL(cursor, sql)
        if success:
            rows = cursor.fetchall()

            if not rows[0][0]:
                return 0
            else:
                return rows[0][0]
        else:
            # @todo Raise an exception.
            return None

    def lastReportDate(self, notificationType):
        """
        Get the last time a notification was reported.

        :param notificationType: A string indicating the type of the
        notification. It is stored in the event history.
        :returns: datetime of last report date.
        """

        cursor = self.connector.conn.cursor()
        sql = """SELECT MAX("notificationTime") FROM "%s" WHERE
        "notificationType" = '%s'""" % (NOTIFICATION_HISTORY_TABLE,
                                        notificationType)

        success = self.dbUtil.executeSQL(cursor, sql)
        if success:
            rows = cursor.fetchall()

            if not rows[0][0]:
                return None
            else:
                return rows[0][0]
        else:
            # @todo Raise an exception.
            return None

    def saveNotificationTime(self):
        """
        Save the notification event to the notification history.
        """

        cursor = self.connector.conn.cursor()
        sql = """INSERT INTO "%s" ("notificationType", "notificationTime")
        VALUES ('MSG_EGAUGE_SERVICE', NOW())""" % NOTIFICATION_HISTORY_TABLE
        success = self.dbUtil.executeSQL(cursor, sql)
        self.connector.conn.commit()
        if not success:
            # @todo Raise an exception.
            self.logger.log(
                'An error occurred while saving the notification time.')

    def sendNewDataNotification(self, testing=False):
        """
        Sending notification reporting on new data being available since the
        last time new data was reported.

        :param testing: Use testing mode when True.
        """

        lastReportDate = self.lastReportDate('MSG_EGAUGE_SERVICE')

        if not lastReportDate:
            lastReportDate = "never"

        msgBody = '\nNew MSG eGauge data has been loaded to %s.' % self\
            .connector.dbName
        msgBody += '\n\n'
        msgBody += 'The new data count is %s readings.' % self.newDataCount()
        msgBody += '\n\n'
        msgBody += 'The last report date was %s.' % lastReportDate
        msgBody += '\n\n'
        self.notifier.sendNotificationEmail(msgBody, testing=testing)
        self.saveNotificationTime()
import argparse
from sek.logger import SEKLogger
import gzip
from msg_noaa_weather_data_parser import MSGNOAAWeatherDataParser
from msg_noaa_weather_data_inserter import MSGNOAAWeatherDataInserter
from msg_db_connector import MSGDBConnector
from msg_time_util import MSGTimeUtil
from msg_noaa_weather_data_util import MSGWeatherDataUtil

configer = MSGConfiger()
logger = SEKLogger(__name__, 'info')
binPath = MSGConfiger.configOptionValue(configer, "Executable Paths",
                                        "msg_bin_path")
COMMAND_LINE_ARGS = None
msgBody = ''
notifier = MSGNotifier()
dataParser = MSGNOAAWeatherDataParser()
inserter = MSGNOAAWeatherDataInserter()
timeUtil = MSGTimeUtil()


def processCommandLineArguments():
    global COMMAND_LINE_ARGS
    argParser = argparse.ArgumentParser(
        description='Perform recursive insertion of compressed weather data'
        ' contained in the current directory to the MECO '
        'database specified in the configuration file.')
    argParser.add_argument('--email',
                           action='store_true',
                           default=False,
                           help='Send email notification if this flag is '
Example #16
0
class NewDataAggregator(object):
    """
    Perform aggregation of new data for a set of predefined data types (self
    .rawTypes).
    """
    def __init__(self):
        """
        Constructor.
        """
        self.logger = SEKLogger(__name__, 'DEBUG')
        self.aggregator = MSGDataAggregator()
        self.notifier = MSGNotifier()
        self.rawTypes = [x.name for x in list(MSGAggregationTypes)]
        self.connector = MSGDBConnector()
        self.conn = self.connector.connectDB()
        self.cursor = self.conn.cursor()
        self.dbUtil = MSGDBUtil()

    def sendNewDataNotification(self, result=None, testing=False):
        """
        Sending notification reporting on new data being available since the
        last time new data was reported.

        :param result: list of dicts containing aggregation results as
        provided by MSGDataAggregator::aggregateNewData.
        :param testing: Use testing mode when True.
        """

        self.logger.log('result {}'.format(result), 'debug')

        lastReportDate = self.notifier.lastReportDate(
            MSGNotificationHistoryTypes.MSG_DATA_AGGREGATOR)

        if not lastReportDate:
            lastReportDate = "never"

        if not result:
            msgBody = '\nNew data has NOT been aggregated in {}. No result ' \
                      'was obtained. This is an error that should be ' \
                      'investigated.'.format(self.connector.dbName)
        else:
            msgBody = '\nNew data has been aggregated in {}.'.format(
                self.connector.dbName)
            msgBody += '\n\n'
            for i in range(len(result)):
                msgBody += 'The new data count for type {} is {} readings' \
                           '.\n'.format(result[i].keys()[0],
                                        result[i].values()[0])
            msgBody += '\n\n'
            msgBody += 'The last report date was %s.' % lastReportDate
            msgBody += '\n\n'
        self.notifier.sendNotificationEmail(msgBody, testing=testing)
        self.notifier.recordNotificationEvent(
            MSGNotificationHistoryTypes.MSG_DATA_AGGREGATOR)

    def aggregateNewData(self):
        """
        :return: list of dicts obtained from
        MSGDataAggregator::aggregateNewData.
        """

        result = map(self.aggregator.aggregateNewData, self.rawTypes)

        self.logger.log('result {}'.format(result))
        return result
Example #17
0
    parser.add_argument('--dbname', help='Database file to be uploaded.')
    parser.add_argument('--fullpath',
                        help='Full path to database file to be uploaded.')
    parser.add_argument('--testing', action='store_true', default=False)

    COMMAND_LINE_ARGS = parser.parse_args()


if __name__ == '__main__':
    logger = SEKLogger(__name__, 'INFO')

    logger.log("Exporting DBs to cloud.")
    processCommandLineArguments()

    exporter = MSGDBExporter()
    notifier = MSGNotifier()
    exporter.logger.shouldRecord = True

    startTime = time.time()
    dbs = exporter.configer.configOptionValue('Export',
                                              'dbs_to_export').split(',')
    fileIDs = exporter.exportDBs(databases=dbs,
                                 toCloud=True,
                                 testing=COMMAND_LINE_ARGS.testing,
                                 deleteOutdated=True)

    wallTime = time.time() - startTime
    wallTimeMin = int(wallTime / 60.0)
    wallTimeSec = (wallTime - wallTimeMin * 60.0)

    if len(fileIDs) == len(dbs):
class NewDataAggregator(object):
    """
    Perform aggregation of new data for a set of predefined data types (self
    .rawTypes).
    """

    def __init__(self):
        """
        Constructor.
        """
        self.logger = MSGLogger(__name__, 'DEBUG')
        self.aggregator = MSGDataAggregator()
        self.notifier = MSGNotifier()
        self.rawTypes = ['weather', 'egauge', 'circuit', 'irradiance']
        self.connector = MSGDBConnector()
        self.conn = self.connector.connectDB()
        self.cursor = self.conn.cursor()
        self.dbUtil = MSGDBUtil()


    def lastReportDate(self, notificationType):
        """
        Get the last time a notification was reported.

        :param notificationType: string indicating the type of the
        notification. It is stored in the event history.
        :returns: datetime of last report date.
        """

        cursor = self.cursor
        sql = """SELECT MAX("notificationTime") FROM "{}" WHERE
        "notificationType" = '{}'""".format(NOTIFICATION_HISTORY_TABLE,
                                            notificationType)

        success = self.dbUtil.executeSQL(cursor, sql)
        if success:
            rows = cursor.fetchall()

            if not rows[0][0]:
                return None
            else:
                return rows[0][0]
        else:
            raise Exception('Exception during getting last report date.')


    def sendNewDataNotification(self, result = None, testing = False):
        """
        Sending notification reporting on new data being available since the
        last time new data was reported.

        :param result: list of dicts containing aggregation results as
        provided by MSGDataAggregator::aggregateNewData.
        :param testing: Use testing mode when True.
        """

        self.logger.log('result {}'.format(result), 'debug')

        lastReportDate = self.lastReportDate(NOTIFICATION_HISTORY_TYPE)

        if not lastReportDate:
            lastReportDate = "never"

        if not result:
            msgBody = '\nNew data has NOT been aggregated in {}. No result ' \
                      'was obtained. This is an error that should be ' \
                      'investigated.'.format(self.connector.dbName)
        else:
            msgBody = '\nNew data has been aggregated in {}.'.format(
                self.connector.dbName)
            msgBody += '\n\n'
            for i in range(len(result)):
                msgBody += 'The new data count for type {} is {} readings' \
                           '.\n'.format(result[i].keys()[0],
                                      result[i].values()[0])
            msgBody += '\n\n'
            msgBody += 'The last report date was %s.' % lastReportDate
            msgBody += '\n\n'
        self.notifier.sendNotificationEmail(msgBody, testing = testing)
        self.saveNotificationTime()


    def saveNotificationTime(self):
        """
        Save a notification event to the notification history.
        """

        cursor = self.cursor
        sql = """INSERT INTO "{}" ("notificationType", "notificationTime")
        VALUES ('{}', NOW())""".format(NOTIFICATION_HISTORY_TABLE,
                                       NOTIFICATION_HISTORY_TYPE)
        success = self.dbUtil.executeSQL(cursor, sql)
        self.conn.commit()
        if not success:
            raise Exception('Exception while saving the notification time.')


    def aggregateNewData(self):
        """
        :return: list of dicts obtained from
        MSGDataAggregator::aggregateNewData.
        """

        result = map(self.aggregator.aggregateNewData, self.rawTypes)

        self.logger.log('result {}'.format(result))
        return result
    elif 'ST534' in name:
        return 'ST534'
    else:
        return False

if __name__ == '__main__':

    processCommandLineArguments()

    success = True
    anyFailure = False
    connector = MSGDBConnector(testing = commandLineArgs.testing)
    conn = connector.connectDB()
    cur = conn.cursor()
    dbUtil = MSGDBUtil()
    notifier = MSGNotifier()
    msgBody = ''
    configer = MSGConfiger()

    if commandLineArgs.testing:
        dbName = configer.configOptionValue("Database", "testing_db_name")
    else:
        dbName = "meco_v3"

    for root, dirs, files in os.walk(os.getcwd()):
        # For each file in the list of found files.
        for name in files:
            nameAndPath = join(root, name)
            if not '.csv' in name.lower(): # Skip all non-CSV files.
                continue
            msg = ("Loading NREL xformer data in file %s to database %s.\n" % (
from msg_db_connector import MSGDBConnector
from msg_db_util import MSGDBUtil
from msg_notifier import MSGNotifier
from msg_configer import MSGConfiger

if __name__ == '__main__':

    filename = sys.argv[1]

    success = True
    anyFailure = False
    connector = MSGDBConnector()
    conn = connector.connectDB()
    cur = conn.cursor()
    dbUtil = MSGDBUtil()
    notifier = MSGNotifier()
    msgBody = ''
    configer = MSGConfiger()

    dbName = configer.configOptionValue("Database", "db_name")

    msg = ("Loading static location record data in file %s to database %s.\n" % (
    filename, dbName))
    sys.stderr.write(msg)
    msgBody += msg

    f = open(filename, "r")
    reader = csv.reader(f)

    # @todo verify column order
 def setUp(self):
     self.logger = SEKLogger(__name__)
     self.notifier = MSGNotifier()
     self.configer = MSGConfiger()
Example #22
0
 def setUp(self):
     self.logger = SEKLogger(__name__)
     self.notifier = MSGNotifier()
     self.configer = MSGConfiger()
class MSGEgaugeNewDataChecker(object):
    """
    Provide notification of newly loaded MSG eGauge data.

    This uses notification type MSG_EGAUGE_SERVICE.
    """

    def __init__(self):
        """
        Constructor.
        """

        print __name__
        self.logger = SEKLogger(__name__)
        self.connector = MSGDBConnector()
        self.dbUtil = MSGDBUtil()
        self.notifier = MSGNotifier()
        self.configer = MSGConfiger()


    def newDataCount(self):
        """
        Measure the amount of new data that is present since the last time
        new data was reported.
        """

        cursor = self.connector.conn.cursor()
        tableName = 'EgaugeEnergyAutoload'
        lastTime = self.lastReportDate('MSG_EGAUGE_SERVICE')
        if lastTime is None:
            lastTime = '1900-01-01'
        sql = """SELECT COUNT(*) FROM "%s" WHERE datetime > '%s'""" % (
            tableName, lastTime)

        success = self.dbUtil.executeSQL(cursor, sql)
        if success:
            rows = cursor.fetchall()

            if not rows[0][0]:
                return 0
            else:
                return rows[0][0]
        else:
            # @todo Raise an exception.
            return None


    def lastReportDate(self, notificationType):
        """
        Get the last time a notification was reported.

        :param notificationType: A string indicating the type of the
        notification. It is stored in the event history.
        :returns: datetime of last report date.
        """

        cursor = self.connector.conn.cursor()
        sql = """SELECT MAX("notificationTime") FROM "%s" WHERE
        "notificationType" = '%s'""" % (
            NOTIFICATION_HISTORY_TABLE, notificationType)

        success = self.dbUtil.executeSQL(cursor, sql)
        if success:
            rows = cursor.fetchall()

            if not rows[0][0]:
                return None
            else:
                return rows[0][0]
        else:
            # @todo Raise an exception.
            return None


    def saveNotificationTime(self):
        """
        Save the notification event to the notification history.
        """

        cursor = self.connector.conn.cursor()
        sql = """INSERT INTO "%s" ("notificationType", "notificationTime")
        VALUES ('MSG_EGAUGE_SERVICE', NOW())""" % NOTIFICATION_HISTORY_TABLE
        success = self.dbUtil.executeSQL(cursor, sql)
        self.connector.conn.commit()
        if not success:
            # @todo Raise an exception.
            self.logger.log(
                'An error occurred while saving the notification time.')


    def sendNewDataNotification(self, testing = False):
        """
        Sending notification reporting on new data being available since the
        last time new data was reported.

        :param testing: Use testing mode when True.
        """

        lastReportDate = self.lastReportDate('MSG_EGAUGE_SERVICE')

        if not lastReportDate:
            lastReportDate = "never"

        msgBody = '\nNew MSG eGauge data has been loaded to %s.' % self\
            .connector.dbName
        msgBody += '\n\n'
        msgBody += 'The new data count is %s readings.' % self.newDataCount()
        msgBody += '\n\n'
        msgBody += 'The last report date was %s.' % lastReportDate
        msgBody += '\n\n'
        self.notifier.sendNotificationEmail(msgBody, testing = testing)
        self.saveNotificationTime()
from sek.logger import SEKLogger
import gzip
from msg_noaa_weather_data_parser import MSGNOAAWeatherDataParser
from msg_noaa_weather_data_inserter import MSGNOAAWeatherDataInserter
from msg_db_connector import MSGDBConnector
from msg_time_util import MSGTimeUtil
from msg_noaa_weather_data_util import MSGWeatherDataUtil


configer = MSGConfiger()
logger = SEKLogger(__name__, 'info')
binPath = MSGConfiger.configOptionValue(configer, "Executable Paths",
                                        "msg_bin_path")
COMMAND_LINE_ARGS = None
msgBody = ''
notifier = MSGNotifier()
dataParser = MSGNOAAWeatherDataParser()
inserter = MSGNOAAWeatherDataInserter()
timeUtil = MSGTimeUtil()


def processCommandLineArguments():
    global COMMAND_LINE_ARGS
    argParser = argparse.ArgumentParser(
        description = 'Perform recursive insertion of compressed weather data'
                      ' contained in the current directory to the MECO '
                      'database specified in the configuration file.')
    argParser.add_argument('--email', action = 'store_true', default = False,
                           help = 'Send email notification if this flag is '
                                  'specified.')
    argParser.add_argument('--testing', action = 'store_true', default = False,
from msg_configer import MSGConfiger
from msg_notifier import MSGNotifier
from meco_plotting import MECOPlotting
from insertSingleMECOEnergyDataFile import Inserter
from sek.logger import SEKLogger


xmlGzCount = 0
xmlCount = 0
configer = MSGConfiger()
logger = SEKLogger(__name__, 'info')
binPath = MSGConfiger.configOptionValue(configer, "Executable Paths",
                                        "msg_bin_path")
COMMAND_LINE_ARGS = None
global msgBody
notifier = MSGNotifier()


def processCommandLineArguments():
    """
    Generate command-line arguments. Load them into global variable
    COMMAND_LINE_ARGS.
    """

    global COMMAND_LINE_ARGS
    parser = argparse.ArgumentParser(
        description = 'Perform recursive insertion of data contained in the '
                      'current directory to the MECO database specified in the '
                      'configuration file.')
    parser.add_argument('--email', action = 'store_true', default = False,
                        help = 'Send email notification if this flag is '
        return 'ST534'
    else:
        return False


if __name__ == '__main__':

    processCommandLineArguments()

    success = True
    anyFailure = False
    connector = MSGDBConnector(testing=commandLineArgs.testing)
    conn = connector.connectDB()
    cur = conn.cursor()
    dbUtil = MSGDBUtil()
    notifier = MSGNotifier()
    msgBody = ''
    configer = MSGConfiger()

    if commandLineArgs.testing:
        dbName = configer.configOptionValue("Database", "testing_db_name")
    else:
        dbName = "meco_v3"

    for root, dirs, files in os.walk(os.getcwd()):
        # For each file in the list of found files.
        for name in files:
            nameAndPath = join(root, name)
            if not '.csv' in name.lower():  # Skip all non-CSV files.
                continue
            msg = ("Loading NREL xformer data in file %s to database %s.\n" %