Beispiel #1
0
    def __init__(self):
        """
        Constructor.
        """

        self._config = ConfigParser.ConfigParser()
        self.logger = SEKLogger(__name__, 'INFO')
        self.fileUtil = SEKFileUtil()

        # Define tables that will have data inserted. Data will only be inserted
        # to tables that are defined here.
        self.insertTables = ('MeterData', 'RegisterData', 'RegisterRead',
                             'Tier', 'Register', 'IntervalReadData',
                             'Interval', 'Reading', 'EventData', 'Event')

        # Check permissions on the config file. Refuse to run if the permissions
        # are not set appropriately.

        configFilePath = '~/.msg-data-operations.cfg'

        if self.fileUtil.isMoreThanOwnerReadableAndWritable(
                os.path.expanduser(configFilePath)):
            self.logger.log(
                "Configuration file permissions are too permissive. Operation "
                "will not continue.", 'error')
            sys.exit()

        try:
            self._config.read(['site.cfg', os.path.expanduser(configFilePath)])
        except:
            self.logger.log(
                "Critical error: The data in {} cannot be "
                "accessed successfully.".format(configFilePath), 'ERROR')
            sys.exit(-1)
Beispiel #2
0
    def setUp(self):
        self.logger = SEKLogger(__name__, 'DEBUG')
        self.configer = MSGConfiger()
        self.exporter = MSGDBExporter()
        self.testDir = 'db_exporter_test'
        self.uncompressedTestFilename = 'meco_v3_test_data.sql'
        self.compressedTestFilename = 'meco_v3_test_data.sql.gz'
        self.exportTestDataPath = self.configer.configOptionValue(
            'Testing', 'export_test_data_path')
        self.fileUtil = MSGFileUtil()
        self.fileChunks = []
        self.testDataFileID = ''
        self.pyUtil = MSGPythonUtil()
        self.timeUtil = MSGTimeUtil()

        conn = None
        try:
            conn = MSGDBConnector().connectDB()
        except Exception as detail:
            self.logger.log("Exception occurred: {}".format(detail), 'error')
            exit(-1)

        self.logger.log("conn = {}".format(conn), 'debug')
        self.assertIsNotNone(conn)

        # Create a temporary working directory.
        try:
            os.mkdir(self.testDir)
        except OSError as detail:
            self.logger.log(
                'Exception during creation of temp directory: %s' % detail,
                'ERROR')
    def __init__(self):
        """
        Constructor.
        """

        warnings.simplefilter('default')
        warnings.warn("This module is deprecated in favor of SEKNotifier.",
                      DeprecationWarning)

        self.config = MSGConfiger()
        self.logger = SEKLogger(__name__, 'info')
        self.connector = MSGDBConnector()
        self.conn = self.connector.connectDB()
        self.cursor = self.conn.cursor()
        self.dbUtil = MSGDBUtil()
        self.noticeTable = 'NotificationHistory'
        self.notificationHeader = "This is a message from the Hawaii Smart " \
                                  "Energy Project MSG Project notification " \
                                  "system.\n\n"

        self.noReplyNotice = '\n\nThis email account is not monitored. No ' \
                             'replies will originate from this ' \
                             'account.\n\nYou are receiving this message ' \
                             'because you are on the recipient list for ' \
                             'notifications for the Hawaii Smart Energy ' \
                             'Project.'
    def __init__(self):
        """
        Constructor.
        """

        self._config = ConfigParser.ConfigParser()
        self.logger = SEKLogger(__name__, 'INFO')
        self.fileUtil = SEKFileUtil()
        self.dbUtil = SEKDBUtil()
        self.cursor = None

        configFilePath = '~/.smart-inverter.cfg'

        if self.fileUtil.isMoreThanOwnerReadableAndWritable(
                os.path.expanduser(configFilePath)):
            self.logger.log(
                "Configuration file permissions are too permissive. Operation "
                "will not continue.", 'error')
            sys.exit(-1)

        try:
            self._config.read(['site.cfg', os.path.expanduser(configFilePath)])
        except:
            self.logger.log(
                "Critical error: The data in {} cannot be "
                "accessed successfully.".format(configFilePath), 'ERROR')
            sys.exit(-1)
Beispiel #5
0
    def __init__(self):
        """
        Constructor.
        """
        self.logger = SEKLogger(__name__, 'debug')

        self.cols = ["wban", "datetime", "datetime", "station_type",
                     "sky_condition", "sky_condition_flag", "visibility",
                     "visibility_flag", "weather_type", "weather_type_flag",
                     "dry_bulb_farenheit", "dry_bulb_farenheit_flag",
                     "dry_bulb_celsius", "dry_bulb_celsius_flag",
                     "wet_bulb_farenheit", "wet_bulb_farenheit_flag",
                     "wet_bulb_celsius", "wet_bulb_celsius_flag",
                     "dew_point_farenheit", "dew_point_farenheit_flag",
                     "dew_point_celsius", "dew_point_celsius_flag",
                     "relative_humidity", "relative_humidity_flag",
                     "wind_speed", "wind_speed_flag", "wind_direction",
                     "wind_direction_flag", "value_for_wind_character",
                     "value_for_wind_character_flag", "station_pressure",
                     "station_pressure_flag", "pressure_tendency",
                     "pressure_tendency_flag", "pressure_change",
                     "pressure_change_flag", "sea_level_pressure",
                     "sea_level_pressure_flag", "record_type",
                     "record_type_flag", "hourly_precip", "hourly_precip_flag",
                     "altimeter", "altimeter_flag"]
Beispiel #6
0
 def setUp(self):
     self.logger = SEKLogger(__name__, 'DEBUG')
     self.configer = SIConfiger()
     self.conn = SEKDBConnector(
         dbName=self.configer.configOptionValue('Database', 'db_name'),
         dbHost=self.configer.configOptionValue('Database', 'db_host'),
         dbPort=self.configer.configOptionValue('Database', 'db_port'),
         dbUsername=self.configer.configOptionValue('Database',
                                                    'db_username'),
         dbPassword=self.configer.configOptionValue(
             'Database', 'db_password')).connectDB()
     self.cursor = self.conn.cursor()
     self.dbUtil = SEKDBUtil()
     self.dataUtil = SIDataUtil()
     self.inserter = SingleFileLoader('data/test-meter/log.csv')
     self.data = '"2014-07-12 16:22:30",0,,,1187488464896.00,' \
                 '2322185846784.00,1134697381888.00,35184644096.00,' \
                 '290857353216.00,10133100822528.00,367.13,' \
                 '-17660932096.00,1078.01,17660934144.00,-7.86,1.80,8.06,' \
                 '-0.97,244.01,122.00,32.93,60.01,-7.09,1.42,7.24,8.06,' \
                 '3.34,8.35,-40.18,-5.68,40.52,516.72,403.12,0,' \
                 '8797179904.00,47518.67,0,86.03,50.23,4198.40,' \
                 '281475022848.00,2251868602368.00,0,6820.01,' \
                 '8796095488.00,0,178.83,188.30,0,620.07,505.19,' \
                 '288230389841920.02,12668.18,68729384.00,0,-3.68,-4.18,,' \
                 '1.00,0.79,,3.81,4.25,,-0.97,-0.98,,244.01,,,121.54,' \
                 '122.46,,31.28,34.59,'
     self.testMeterName = 'test-meter'
Beispiel #7
0
    def __init__(self):
        """
        Constructor.
        """

        self.logger = SEKLogger(__name__)
        self.configer = MSGConfiger()
        self.fileUtil = MSGFileUtil()
    def __init__(self):
        """
        Constructor.
        """

        self.logger = SEKLogger(__name__, 'DEBUG')
        self.cursor = MSGDBConnector().connectDB().cursor()
        self.dbUtil = MSGDBUtil()
    def __init__(self, testing=False):
        """
        Constructor.
        :param testing: True if testing mode is being used.
        """

        self.logger = SEKLogger(__name__, 'info')
        self.dbUtil = MSGDBUtil()
        self.dupeChecker = MSGWeatherDataDupeChecker()
Beispiel #10
0
    def __init__(self):
        """
        Constructor.
        """

        self.logger = SEKLogger(__name__, 'debug')
        self.mapper = MECOMapper()
        self.dupeChecker = MECODupeChecker()
        self.dbUtil = MSGDBUtil()
Beispiel #11
0
    def __init__(self, testing=False):
        """
        Constructor.

        :param testing: Flag for testing mode.
        """

        self.logger = SEKLogger(__name__, 'debug')
        self.dbUtil = MSGDBUtil()
Beispiel #12
0
 def setUp(self):
     """
     Constructor.
     """
     self.logger = SEKLogger(__name__, 'DEBUG')
     self.aggregator = MSGDataAggregator()
     self.testStart = '2014-01-02 11:59'
     self.testEnd = '2014-01-02 12:14'
     self.rawTypes = ['weather', 'egauge', 'circuit', 'irradiance']
Beispiel #13
0
    def __init__(self):
        """
        Constructor.
        """

        self.logger = SEKLogger(__name__, 'debug')
        self.mecoConfig = MSGConfiger()
        self.currentReadingID = 0
        self.dbUtil = MSGDBUtil()
    def __init__(self, testing=False):
        """
        Constructor.

        :param testing: Flag indicating if testing mode is on.
        """

        self.logger = SEKLogger(__name__)
        self.parser = MECOXMLParser(testing)
        self.configer = MSGConfiger()
Beispiel #15
0
    def __init__(self):
        """
        Constructor.
        """

        print __name__
        self.logger = SEKLogger(__name__)
        self.connector = MSGDBConnector()
        self.dbUtil = MSGDBUtil()
        self.notifier = MSGNotifier()
        self.configer = MSGConfiger()
Beispiel #16
0
 def __init__(self):
     """
     Constructor.
     """
     self.logger = SEKLogger(__name__, 'DEBUG')
     self.aggregator = MSGDataAggregator()
     self.notifier = MSGNotifier()
     self.rawTypes = [x.name for x in list(MSGAggregationTypes)]
     self.connector = MSGDBConnector()
     self.conn = self.connector.connectDB()
     self.cursor = self.conn.cursor()
     self.dbUtil = MSGDBUtil()
Beispiel #17
0
class MECODataAutoloader(object):
    """
    Provide automated loading of MECO energy data from exports in
    gzip-compressed XML source data.
    """
    def __init__(self):
        """
        Constructor.
        """

        self.logger = SEKLogger(__name__)
        self.configer = MSGConfiger()
        self.fileUtil = MSGFileUtil()

    def newDataExists(self):
        """
        Check the data autoload folder for the presence of new data.

        :returns: True if new data exists.
        """

        autoloadPath = self.configer.configOptionValue(
            'MECO Autoload', 'meco_autoload_new_data_path')
        if not self.fileUtil.validDirectory(autoloadPath):
            raise Exception('InvalidDirectory', '%s' % autoloadPath)

        patterns = ['*.gz']
        matchCnt = 0
        for root, dirs, filenames in os.walk(autoloadPath):
            for pat in patterns:
                for filename in fnmatch.filter(filenames, pat):
                    print filename
                    matchCnt += 1
        if matchCnt > 0:
            return True
        else:
            return False

    def loadNewData(self):
        """
        Load new data contained in the new data path.
        """

        autoloadPath = self.configer.configOptionValue(
            'MECO Autoload', 'meco_autoload_new_data_path')
        command = self.configer.configOptionValue('MECO Autoload',
                                                  'meco_autoload_command')
        os.chdir(autoloadPath)

        try:
            subprocess.check_call(command, shell=True)
        except subprocess.CalledProcessError, e:
            self.logger.log("An exception occurred: %s" % e, 'error')
    def __init__(self,
                 exitOnError=True,
                 commitOnEveryInsert=False,
                 testing=False):
        """
        Constructor.

        :param testing: if True, the testing DB will be connected instead of
        the production DB.
        """

        self.logger = SEKLogger(__name__, 'info')
        self.configer = MSGConfiger()
        self.conn = MSGDBConnector().connectDB()
        self.cursor = self.conn.cursor()
        self.dbUtil = MSGDBUtil()
        self.notifier = MSGNotifier()
        self.mathUtil = MSGMathUtil()
        self.timeUtil = MSGTimeUtil()
        self.nextMinuteCrossing = {}
        self.nextMinuteCrossingWithoutSubkeys = None
        self.exitOnError = exitOnError
        self.commitOnEveryInsert = commitOnEveryInsert
        section = 'Aggregation'
        tableList = [
            'irradiance', 'agg_irradiance', 'weather', 'agg_weather',
            'circuit', 'agg_circuit', 'egauge', 'agg_egauge'
        ]
        self.dataParams = {
            'weather': ('agg_weather', 'timestamp', ''),
            'egauge': ('agg_egauge', 'datetime', 'egauge_id'),
            'circuit': ('agg_circuit', 'timestamp', 'circuit'),
            'irradiance': ('agg_irradiance', 'timestamp', 'sensor_id')
        }
        self.columns = {}

        # tables[datatype] gives the table name for datatype.
        self.tables = {
            t: self.configer.configOptionValue(section, '{}_table'.format(t))
            for t in tableList
        }

        for t in self.tables.keys():
            self.logger.log('t:{}'.format(t), 'DEBUG')
            try:
                self.columns[t] = self.dbUtil.columnsString(
                    self.cursor, self.tables[t])
            except TypeError as error:
                self.logger.log(
                    'Ignoring missing table: Error is {}.'.format(error),
                    'error')
class MECODataAutoloader(object):
    """
    Provide automated loading of MECO energy data from exports in
    gzip-compressed XML source data.
    """

    def __init__(self):
        """
        Constructor.
        """

        self.logger = SEKLogger(__name__)
        self.configer = MSGConfiger()
        self.fileUtil = MSGFileUtil()

    def newDataExists(self):
        """
        Check the data autoload folder for the presence of new data.

        :returns: True if new data exists.
        """

        autoloadPath = self.configer.configOptionValue("MECO Autoload", "meco_autoload_new_data_path")
        if not self.fileUtil.validDirectory(autoloadPath):
            raise Exception("InvalidDirectory", "%s" % autoloadPath)

        patterns = ["*.gz"]
        matchCnt = 0
        for root, dirs, filenames in os.walk(autoloadPath):
            for pat in patterns:
                for filename in fnmatch.filter(filenames, pat):
                    print filename
                    matchCnt += 1
        if matchCnt > 0:
            return True
        else:
            return False

    def loadNewData(self):
        """
        Load new data contained in the new data path.
        """

        autoloadPath = self.configer.configOptionValue("MECO Autoload", "meco_autoload_new_data_path")
        command = self.configer.configOptionValue("MECO Autoload", "meco_autoload_command")
        os.chdir(autoloadPath)

        try:
            subprocess.check_call(command, shell=True)
        except subprocess.CalledProcessError, e:
            self.logger.log("An exception occurred: %s" % e, "error")
Beispiel #20
0
    def __init__(self):
        """
        Constructor.
        """

        self.logger = SEKLogger(__name__, 'DEBUG', useColor=False)
        self.timeUtil = MSGTimeUtil()
        self.configer = MSGConfiger()
        self.fileUtil = MSGFileUtil()
        self.pythonUtil = MSGPythonUtil()  # for debugging
        self.connector = MSGDBConnector()
        self.conn = self.connector.connectDB()
        self.cursor = self.conn.cursor()
        self.dbUtil = MSGDBUtil()
        self.notifier = SEKNotifier(
            connector=self.connector,
            dbUtil=self.dbUtil,
            user=self.configer.configOptionValue('Notifications',
                                                 'email_username'),
            password=self.configer.configOptionValue('Notifications',
                                                     'email_password'),
            fromaddr=self.configer.configOptionValue('Notifications',
                                                     'email_from_address'),
            toaddr=self.configer.configOptionValue('Notifications',
                                                   'email_recipients'),
            testing_toaddr=self.configer.configOptionValue(
                'Notifications', 'testing_email_recipients'),
            smtp_server_and_port=self.configer.configOptionValue(
                'Notifications', 'smtp_server_and_port'))

        # Google Drive parameters.
        self.clientID = self.configer.configOptionValue(
            'Export', 'google_api_client_id')
        self.clientSecret = self.configer.configOptionValue(
            'Export', 'google_api_client_secret')
        self.oauthScope = 'https://www.googleapis.com/auth/drive'
        self.oauthConsent = 'urn:ietf:wg:oauth:2.0:oob'
        self.googleAPICredentials = ''
        self.exportTempWorkPath = self.configer.configOptionValue(
            'Export', 'db_export_work_path')

        self.credentialPath = self.configer.configOptionValue(
            'Export', 'google_api_credentials_path')
        self.credentialStorage = Storage('{}/google_api_credentials'.format(
            self.credentialPath))

        self._driveService = None
        self._cloudFiles = None
        self.postAgent = 'Maui Smart Grid 1.0.0 DB Exporter'
        self.retryDelay = 10
        self.availableFilesURL = ''
    def __init__(self):
        """
        Constructor.
        """

        self._config = ConfigParser.ConfigParser()
        self.logger = SEKLogger(__name__, 'INFO')
        self.fileUtil = SEKFileUtil()
        self.dbUtil = SEKDBUtil()
        self.cursor = None

        configFilePath = '~/.smart-inverter.cfg'

        if self.fileUtil.isMoreThanOwnerReadableAndWritable(
                os.path.expanduser(configFilePath)):
            self.logger.log(
                "Configuration file permissions are too permissive. Operation "
                "will not continue.", 'error')
            sys.exit(-1)

        try:
            self._config.read(['site.cfg', os.path.expanduser(configFilePath)])
        except:
            self.logger.log("Critical error: The data in {} cannot be "
                            "accessed successfully.".format(configFilePath),
                            'ERROR')
            sys.exit(-1)
 def setUp(self):
     self.logger = SEKLogger(__name__,'DEBUG')
     self.configer = SIConfiger()
     self.conn = SEKDBConnector(
         dbName = self.configer.configOptionValue('Database', 'db_name'),
         dbHost = self.configer.configOptionValue('Database', 'db_host'),
         dbPort = self.configer.configOptionValue('Database', 'db_port'),
         dbUsername = self.configer.configOptionValue('Database',
                                                      'db_username'),
         dbPassword = self.configer.configOptionValue('Database',
                                                      'db_password')).connectDB()
     self.cursor = self.conn.cursor()
     self.dbUtil = SEKDBUtil()
     self.dataUtil = SIDataUtil()
     self.inserter = SingleFileLoader('data/test-meter/log.csv')
     self.data = '"2014-07-12 16:22:30",0,,,1187488464896.00,' \
                 '2322185846784.00,1134697381888.00,35184644096.00,' \
                 '290857353216.00,10133100822528.00,367.13,' \
                 '-17660932096.00,1078.01,17660934144.00,-7.86,1.80,8.06,' \
                 '-0.97,244.01,122.00,32.93,60.01,-7.09,1.42,7.24,8.06,' \
                 '3.34,8.35,-40.18,-5.68,40.52,516.72,403.12,0,' \
                 '8797179904.00,47518.67,0,86.03,50.23,4198.40,' \
                 '281475022848.00,2251868602368.00,0,6820.01,' \
                 '8796095488.00,0,178.83,188.30,0,620.07,505.19,' \
                 '288230389841920.02,12668.18,68729384.00,0,-3.68,-4.18,,' \
                 '1.00,0.79,,3.81,4.25,,-0.97,-0.98,,244.01,,,121.54,' \
                 '122.46,,31.28,34.59,'
     self.testMeterName = 'test-meter'
    def __init__(self):
        """
        Constructor.
        """

        self._config = ConfigParser.ConfigParser()
        self.logger = SEKLogger(__name__, 'INFO')
        self.fileUtil = SEKFileUtil()

        # Define tables that will have data inserted. Data will only be inserted
        # to tables that are defined here.
        self.insertTables = (
            'MeterData', 'RegisterData', 'RegisterRead', 'Tier', 'Register',
            'IntervalReadData', 'Interval', 'Reading', 'EventData', 'Event')

        # Check permissions on the config file. Refuse to run if the permissions
        # are not set appropriately.

        configFilePath = '~/.msg-data-operations.cfg'

        if self.fileUtil.isMoreThanOwnerReadableAndWritable(
                os.path.expanduser(configFilePath)):
            self.logger.log(
                "Configuration file permissions are too permissive. Operation "
                "will not continue.", 'error')
            sys.exit()

        try:
            self._config.read(['site.cfg', os.path.expanduser(configFilePath)])
        except:
            self.logger.log("Critical error: The data in {} cannot be "
                            "accessed successfully.".format(configFilePath),
                            'ERROR')
            sys.exit(-1)
    def setUp(self):
        self.logger = SEKLogger(__name__, 'DEBUG')
        self.configer = MSGConfiger()
        self.exporter = MSGDBExporter()
        self.testDir = 'db_exporter_test'
        self.uncompressedTestFilename = 'meco_v3_test_data.sql'
        self.compressedTestFilename = 'meco_v3_test_data.sql.gz'
        self.exportTestDataPath = self.configer.configOptionValue('Testing',
                                                                  'export_test_data_path')
        self.fileUtil = MSGFileUtil()
        self.fileChunks = []
        self.testDataFileID = ''
        self.pyUtil = MSGPythonUtil()
        self.timeUtil = MSGTimeUtil()

        conn = None
        try:
            conn = MSGDBConnector().connectDB()
        except Exception as detail:
            self.logger.log("Exception occurred: {}".format(detail), 'error')
            exit(-1)

        self.logger.log("conn = {}".format(conn), 'debug')
        self.assertIsNotNone(conn)

        # Create a temporary working directory.
        try:
            os.mkdir(self.testDir)
        except OSError as detail:
            self.logger.log(
                'Exception during creation of temp directory: %s' % detail,
                'ERROR')
    def __init__(self, dbName = '', dbHost = '', dbPort = '', dbUsername = '',
                 dbPassword = '', testing = False, logLevel = 'silent'):
        """
        Constructor.

        :param testing: Boolean indicating if Testing Mode is on. When
        testing mode is on, a connection to the testing database will be made
        instead of the production database. This is useful for unit testing.
        :param logLevel
        """

        self.logger = SEKLogger(__name__, logLevel)

        if testing:
            self.logger.log("Testing Mode is ON.")

        self.dbName = dbName
        self.dbHost = dbHost
        self.dbPort = dbPort
        self.dbPassword = dbPassword
        self.dbUsername = dbUsername

        self.logger.log(
            "Instantiating DB connector with database {}.".format(dbName))

        self.conn = self.connectDB()
        if not self.conn:
            self.logger.log('DB connection not available.', 'error')
            sys.exit(-1)

        try:
            self.dictCur = self.conn.cursor(
                cursor_factory = psycopg2.extras.DictCursor)
        except AttributeError as error:
            self.logger.log('Error while getting DictCursor: {}'.format(error))
 def __init__(self):
     """
     Constructor.
     """
     warnings.simplefilter('default')
     warnings.warn("This module is deprecated in favor of SEKFileUtil.",
                   DeprecationWarning)
     self.logger = SEKLogger(__name__, 'DEBUG')
    def __init__(self):
        """
        Constructor.
        """

        self.logger = SEKLogger(__name__)
        self.configer = MSGConfiger()
        self.fileUtil = MSGFileUtil()
class MSGWeatherDataDupeChecker(object):
    """
    Determine if a duplicate record exists based on the tuple

    (WBAN, Date, Time, StationType).
    """

    def __init__(self, testing = False):
        """
        Constructor.

        :param testing: Flag for testing mode.
        """

        self.logger = SEKLogger(__name__, 'debug')
        self.dbUtil = MSGDBUtil()


    def duplicateExists(self, dbCursor, wban, datetime, recordType):
        """
        Check for the existence of a duplicate record.

        :param dbCursor
        :param wban
        :param datetime
        :param recordType
        :returns: True if a duplicate record exists, otherwise False.
        """

        tableName = "WeatherNOAA"
        sql = """SELECT wban, datetime, record_type FROM \"%s\" WHERE
                 wban = '%s' AND datetime = '%s' AND record_type = '%s'""" % (
            tableName, wban, datetime, recordType)

        self.logger.log("sql=%s" % sql, 'debug')
        self.logger.log("wban=%s, datetime=%s, record_type=%s" % (
            wban, datetime, recordType), 'debug')

        self.dbUtil.executeSQL(dbCursor, sql)
        rows = dbCursor.fetchall()

        if len(rows) > 0:
            return True
        else:
            return False
 def setUp(self):
     """
     Constructor.
     """
     self.logger = SEKLogger(__name__, "DEBUG")
     self.aggregator = MSGDataAggregator()
     self.testStart = "2014-01-02 11:59"
     self.testEnd = "2014-01-02 12:14"
     self.rawTypes = ["weather", "egauge", "circuit", "irradiance"]
    def __init__(self):
        """
        Constructor.
        """

        self.logger = SEKLogger(__name__, 'debug')
        self.mecoConfig = MSGConfiger()
        self.currentReadingID = 0
        self.dbUtil = MSGDBUtil()
    def __init__(self, testing = False):
        """
        Constructor.
        :param testing: True if testing mode is being used.
        """

        self.logger = SEKLogger(__name__, 'info')
        self.dbUtil = MSGDBUtil()
        self.dupeChecker = MSGWeatherDataDupeChecker()
    def __init__(self):
        """
        Constructor.
        """

        self.logger = SEKLogger(__name__, 'debug')
        self.mapper = MECOMapper()
        self.dupeChecker = MECODupeChecker()
        self.dbUtil = MSGDBUtil()
    def __init__(self):
        """
        Constructor.

        A database connection is not maintained here to keep this class
        lightweight.
        """

        self.logger = SEKLogger(__name__, DEBUG)
        self.configer = MSGConfiger()
        self.url = self.configer.configOptionValue('Weather Data',
                                                   'weather_data_url')
        self.pattern = self.configer.configOptionValue('Weather Data',
                                                       'weather_data_pattern')
        self.fileList = []
        self.dateList = []  # List of dates corresponding weather data files.
        self.fillFileListAndDateList()
        self.dbUtil = MSGDBUtil()
    def __init__(self, testing = False):
        """
        Constructor.

        :param testing: Flag for testing mode.
        """

        self.logger = SEKLogger(__name__, 'debug')
        self.dbUtil = MSGDBUtil()
Beispiel #35
0
class MSGWeatherDataDupeChecker(object):
    """
    Determine if a duplicate record exists based on the tuple

    (WBAN, Date, Time, StationType).
    """
    def __init__(self, testing=False):
        """
        Constructor.

        :param testing: Flag for testing mode.
        """

        self.logger = SEKLogger(__name__, 'debug')
        self.dbUtil = MSGDBUtil()

    def duplicateExists(self, dbCursor, wban, datetime, recordType):
        """
        Check for the existence of a duplicate record.

        :param dbCursor
        :param wban
        :param datetime
        :param recordType
        :returns: True if a duplicate record exists, otherwise False.
        """

        tableName = "WeatherNOAA"
        sql = """SELECT wban, datetime, record_type FROM \"%s\" WHERE
                 wban = '%s' AND datetime = '%s' AND record_type = '%s'""" % (
            tableName, wban, datetime, recordType)

        self.logger.log("sql=%s" % sql, 'debug')
        self.logger.log(
            "wban=%s, datetime=%s, record_type=%s" %
            (wban, datetime, recordType), 'debug')

        self.dbUtil.executeSQL(dbCursor, sql)
        rows = dbCursor.fetchall()

        if len(rows) > 0:
            return True
        else:
            return False
    def setUp(self):
        self.i = MECODBInserter()

        # Connect to the testing database.
        self.connector = MSGDBConnector(testing=True)

        self.conn = self.connector.connectDB()
        self.lastSeqVal = None

        # Does this work having the dictCur be in another class?
        self.dictCur = self.connector.dictCur

        self.cursor = self.conn.cursor()
        self.deleter = MECODBDeleter()
        self.tableName = 'MeterData'
        self.columnName = 'meter_data_id'
        self.configer = MSGConfiger()
        self.logger = SEKLogger(__name__, 'debug')
        self.dbUtil = MSGDBUtil()
Beispiel #37
0
class MSGTimeUtilTester(unittest.TestCase):
    def setUp(self):
        self.logger = SEKLogger(__name__, 'debug')
        self.timeUtil = MSGTimeUtil()

    def test_concise_now(self):
        conciseNow = self.timeUtil.conciseNow()
        self.logger.log(conciseNow)
        pattern = '\d+-\d+-\d+_\d+'
        result = re.match(pattern, conciseNow)
        self.assertTrue(result is not None,
                        "Concise now matches the regex pattern.")

    def test_split_dates(self):
        start = dt(2014, 01, 07)
        end = dt(2014, 04, 04)
        print self.timeUtil.splitDates(start, end)
        self.assertEqual(len(self.timeUtil.splitDates(start, end)), 4,
                         'Unexpected date count.')
Beispiel #38
0
class SIDataUtilTester(unittest.TestCase):
    """
    """

    def setUp(self):
        self.dataUtil = SIDataUtil()
        self.logger = SEKLogger(__name__)


    def test_find_max_timestamp(self):
        filePath = 'data/test-meter/log.csv'
        self.assertEquals(self.dataUtil.maxTimeStamp(filePath),
                          datetime.strptime('2014-03-10 23:59:00',
                                            '%Y-%m-%d %H:%M:%S'))

    def test_find_max_timestamp_db(self):
        # @todo test with a static testing DB
        meter = '001EC6051A0D'
        self.logger.log(self.dataUtil.maxTimeStampDB(meter))
    def __init__(self, testing = False):
        """
        Constructor.

        :param testing: Flag indicating if testing mode is on.
        """

        self.logger = SEKLogger(__name__)
        self.parser = MECOXMLParser(testing)
        self.configer = MSGConfiger()
    def __init__(self):
        """
        Constructor.
        """

        self.logger = SEKLogger(__name__)
        self.viewPVReadingsinNonMLH = ''
        self.lastDateProcessed = None
        self.connector = MSGDBConnector()
        self.conn = self.connector.connectDB()
Beispiel #41
0
    def __init__(self, testing=False, logLevel='silent'):
        """
        Constructor.

        :param testing: Boolean indicating if Testing Mode is on. When
        testing mode is on, a connection to the testing database will be made
        instead of the production database. This is useful for unit testing.
        :param logLevel
        """

        self.logger = SEKLogger(__name__, logLevel)

        if testing:
            self.logger.log("Testing Mode is ON.")

        self.configer = MSGConfiger()
        self.dbPassword = self.configer.configOptionValue(
            "Database", 'db_password')
        self.dbHost = self.configer.configOptionValue("Database", 'db_host')
        self.dbPort = self.configer.configOptionValue("Database", 'db_port')

        if testing:
            self.dbName = self.configer.configOptionValue(
                "Database", 'testing_db_name')
        else:
            self.dbName = self.configer.configOptionValue(
                "Database", 'db_name')

        self.logger.log("Instantiating DB connector with database {}.".format(
            self.dbName))

        self.dbUsername = self.configer.configOptionValue(
            "Database", 'db_username')

        self.conn = self.connectDB()
        if not self.conn:
            raise Exception('DB connection not available.')

        try:
            self.dictCur = self.conn.cursor(
                cursor_factory=psycopg2.extras.DictCursor)
        except AttributeError as error:
            self.logger.log('Error while getting DictCursor: {}'.format(error))
    def __init__(self, filepath=''):
        """
        Constructor.

        :param testing: Flag indicating if testing mode is on.
        """

        self.logger = SEKLogger(__name__, DEBUG)
        self.configer = SIConfiger()
        self.dbUtil = SEKDBUtil()
        self.dataUtil = SIDataUtil()
        self.logger.log('making new db conn for filepath {}'.format(filepath),
                        SILENT)
        sys.stdout.flush()

        try:
            self.conn = SEKDBConnector(
                dbName=self.configer.configOptionValue('Database', 'db_name'),
                dbHost=self.configer.configOptionValue('Database', 'db_host'),
                dbPort=self.configer.configOptionValue('Database', 'db_port'),
                dbUsername=self.configer.configOptionValue(
                    'Database', 'db_username'),
                dbPassword=self.configer.configOptionValue(
                    'Database', 'db_password')).connectDB()
        except:
            raise Exception("Unable to get DB connection.")
        self.cursor = self.conn.cursor()
        self.exitOnError = False

        # An empty file path is used during creating of meter table entries.
        if filepath == '':
            self.filepath = None
            self.meterID = None
            self.meterDataTable = None
        else:
            self.filepath = filepath
            self.meterID = self.getOrMakeMeterID(self.meterName())
            assert self.meterID is not None
            self.meterDataTable = "MeterData_{}".format(self.meterName())
            # @todo Test existence of meter data table.
        self.timestampColumn = 0  # timestamp col in the raw data
        self.exceptionCount = 0
    def __init__(self):
        """
        Constructor.
        """

        print __name__
        self.logger = SEKLogger(__name__)
        self.connector = MSGDBConnector()
        self.dbUtil = MSGDBUtil()
        self.notifier = MSGNotifier()
        self.configer = MSGConfiger()
    def __init__(self):
        """
        Constructor.
        """

        self.logger = SEKLogger(__name__, 'DEBUG', useColor = False)
        self.timeUtil = MSGTimeUtil()
        self.configer = MSGConfiger()
        self.fileUtil = MSGFileUtil()
        self.pythonUtil = MSGPythonUtil()  # for debugging
        self.connector = MSGDBConnector()
        self.conn = self.connector.connectDB()
        self.cursor = self.conn.cursor()
        self.dbUtil = MSGDBUtil()
        self.notifier = SEKNotifier(connector = self.connector,
                                    dbUtil = self.dbUtil,
                                    user = self.configer.configOptionValue(
                                        'Notifications', 'email_username'),
                                    password = self.configer.configOptionValue(
                                        'Notifications', 'email_password'),
                                    fromaddr = self.configer.configOptionValue(
                                        'Notifications', 'email_from_address'),
                                    toaddr = self.configer.configOptionValue(
                                        'Notifications', 'email_recipients'),
                                    testing_toaddr =
                                    self.configer.configOptionValue(
                                        'Notifications',
                                        'testing_email_recipients'),
                                    smtp_server_and_port =
                                    self.configer.configOptionValue(
                                        'Notifications',
                                        'smtp_server_and_port'))

        # Google Drive parameters.
        self.clientID = self.configer.configOptionValue('Export',
                                                        'google_api_client_id')
        self.clientSecret = self.configer.configOptionValue('Export',
                                                            'google_api_client_secret')
        self.oauthScope = 'https://www.googleapis.com/auth/drive'
        self.oauthConsent = 'urn:ietf:wg:oauth:2.0:oob'
        self.googleAPICredentials = ''
        self.exportTempWorkPath = self.configer.configOptionValue('Export',
                                                                  'db_export_work_path')

        self.credentialPath = self.configer.configOptionValue('Export',
                                                              'google_api_credentials_path')
        self.credentialStorage = Storage(
            '{}/google_api_credentials'.format(self.credentialPath))

        self._driveService = None
        self._cloudFiles = None
        self.postAgent = 'Maui Smart Grid 1.0.0 DB Exporter'
        self.retryDelay = 10
        self.availableFilesURL = ''
Beispiel #45
0
class WeatherDataLoadingTester(unittest.TestCase):
    def setUp(self):
        self.weatherUtil = MSGWeatherDataUtil()
        self.logger = SEKLogger(__name__, 'DEBUG')
        self.dbConnector = MSGDBConnector()
        self.cursor = self.dbConnector.conn.cursor()
        self.configer = MSGConfiger()


    def testLoadDataSinceLastLoaded(self):
        """
        Data should be loaded since the last data present in the database.
        """
        pass


    def testRetrieveDataSinceLastLoaded(self):
        """
        Data since the last loaded date is retrieved.
        """
        pass


    def testGetLastLoadedDate(self):
        myDate = self.weatherUtil.getLastDateLoaded(self.cursor).strftime(
            "%Y-%m-%d %H:%M:%S")
        pattern = '^(\d+-\d+-\d+\s\d+:\d+:\d+)$'
        match = re.match(pattern, myDate)
        assert match and (match.group(1) == myDate), "Date format is valid."


    def testWeatherDataPattern(self):
        myPattern = self.configer.configOptionValue('Weather Data',
                                                    'weather_data_pattern')
        testString = """<A HREF="someURL">QCLCD201208.zip</A>"""

        match = re.match(myPattern, testString)
        self.logger.log("pattern = %s" % myPattern, 'info')
        if match:
            self.logger.log("match = %s" % match)
            self.logger.log("match group = %s" % match.group(1))
        else:
            self.logger.log("match not found")
        assert match and match.group(
            1) == 'QCLCD201208.zip', "Download filename was matched."


    def testWeatherDataURL(self):
        myURL = self.configer.configOptionValue('Weather Data',
                                                'weather_data_url')
        pass
Beispiel #46
0
    def __init__(self,
                 dbName='',
                 dbHost='',
                 dbPort='',
                 dbUsername='',
                 dbPassword='',
                 testing=False,
                 logLevel='silent'):
        """
        Constructor.

        :param testing: Boolean indicating if Testing Mode is on. When
        testing mode is on, a connection to the testing database will be made
        instead of the production database. This is useful for unit testing.
        :param logLevel
        """

        self.logger = SEKLogger(__name__, logLevel)

        if testing:
            self.logger.log("Testing Mode is ON.")

        self.dbName = dbName
        self.dbHost = dbHost
        self.dbPort = dbPort
        self.dbPassword = dbPassword
        self.dbUsername = dbUsername

        self.logger.log(
            "Instantiating DB connector with database {}.".format(dbName))

        self.conn = self.connectDB()
        if not self.conn:
            self.logger.log('DB connection not available.', 'error')
            sys.exit(-1)

        try:
            self.dictCur = self.conn.cursor(
                cursor_factory=psycopg2.extras.DictCursor)
        except AttributeError as error:
            self.logger.log('Error while getting DictCursor: {}'.format(error))
 def __init__(self):
     """
     Constructor.
     """
     self.logger = SEKLogger(__name__, 'DEBUG')
     self.aggregator = MSGDataAggregator()
     self.notifier = MSGNotifier()
     self.rawTypes = [x.name for x in list(MSGAggregationTypes)]
     self.connector = MSGDBConnector()
     self.conn = self.connector.connectDB()
     self.cursor = self.conn.cursor()
     self.dbUtil = MSGDBUtil()
class SIConfiger(object):
    """
    Supports site-level config for the Smart Grid PV Inverter project.
    The default path is ~/.smart-inverter.cfg.

    Usage:

        configer = SIConfiger()

    """
    def __init__(self):
        """
        Constructor.
        """

        self._config = ConfigParser.ConfigParser()
        self.logger = SEKLogger(__name__, 'INFO')
        self.fileUtil = SEKFileUtil()
        self.dbUtil = SEKDBUtil()
        self.cursor = None

        configFilePath = '~/.smart-inverter.cfg'

        if self.fileUtil.isMoreThanOwnerReadableAndWritable(
                os.path.expanduser(configFilePath)):
            self.logger.log(
                "Configuration file permissions are too permissive. Operation "
                "will not continue.", 'error')
            sys.exit(-1)

        try:
            self._config.read(['site.cfg', os.path.expanduser(configFilePath)])
        except:
            self.logger.log(
                "Critical error: The data in {} cannot be "
                "accessed successfully.".format(configFilePath), 'ERROR')
            sys.exit(-1)

    def configOptionValue(self, section, option):
        """
        Get a configuration value from the local configuration file.
        :param section: String of section in config file.
        :param option: String of option in config file.
        :returns: The value contained in the configuration file.
        """

        try:
            configValue = self._config.get(section, option)
            if configValue == "True":
                return True
            elif configValue == "False":
                return False
            else:
                return configValue
        except:
            self.logger.log(
                "Failed when getting configuration option {} in section {"
                "}.".format(option, section), 'error')
            sys.exit(-1)
    def __init__(self, exitOnError=True, commitOnEveryInsert=False, testing=False):
        """
        Constructor.

        :param testing: if True, the testing DB will be connected instead of
        the production DB.
        """

        self.logger = SEKLogger(__name__, "info")
        self.configer = MSGConfiger()
        self.conn = MSGDBConnector().connectDB()
        self.cursor = self.conn.cursor()
        self.dbUtil = MSGDBUtil()
        self.notifier = MSGNotifier()
        self.mathUtil = MSGMathUtil()
        self.timeUtil = MSGTimeUtil()
        self.nextMinuteCrossing = {}
        self.nextMinuteCrossingWithoutSubkeys = None
        self.exitOnError = exitOnError
        self.commitOnEveryInsert = commitOnEveryInsert
        section = "Aggregation"
        tableList = [
            "irradiance",
            "agg_irradiance",
            "weather",
            "agg_weather",
            "circuit",
            "agg_circuit",
            "egauge",
            "agg_egauge",
        ]
        self.dataParams = {
            "weather": ("agg_weather", "timestamp", ""),
            "egauge": ("agg_egauge", "datetime", "egauge_id"),
            "circuit": ("agg_circuit", "timestamp", "circuit"),
            "irradiance": ("agg_irradiance", "timestamp", "sensor_id"),
        }
        self.columns = {}

        # tables[datatype] gives the table name for datatype.
        self.tables = {t: self.configer.configOptionValue(section, "{}_table".format(t)) for t in tableList}

        for t in self.tables.keys():
            self.logger.log("t:{}".format(t), "DEBUG")
            try:
                self.columns[t] = self.dbUtil.columnsString(self.cursor, self.tables[t])
            except TypeError as error:
                self.logger.log("Ignoring missing table: Error is {}.".format(error), "error")
    def __init__(self):
        """
        Constructor.

        A database connection is not maintained here to keep this class
        lightweight.
        """

        self.logger = SEKLogger(__name__, DEBUG)
        self.configer = MSGConfiger()
        self.url = self.configer.configOptionValue('Weather Data',
                                                   'weather_data_url')
        self.pattern = self.configer.configOptionValue('Weather Data',
                                                       'weather_data_pattern')
        self.fileList = []
        self.dateList = [] # List of dates corresponding weather data files.
        self.fillFileListAndDateList()
        self.dbUtil = MSGDBUtil()
    def setUp(self):
        self.i = MECODBInserter()

        # Connect to the testing database.
        self.connector = MSGDBConnector(testing = True)

        self.conn = self.connector.connectDB()
        self.lastSeqVal = None

        # Does this work having the dictCur be in another class?
        self.dictCur = self.connector.dictCur

        self.cursor = self.conn.cursor()
        self.deleter = MECODBDeleter()
        self.tableName = 'MeterData'
        self.columnName = 'meter_data_id'
        self.configer = MSGConfiger()
        self.logger = SEKLogger(__name__, 'debug')
        self.dbUtil = MSGDBUtil()
    def __init__(self, testing = False, logLevel = 'silent'):
        """
        Constructor.

        :param testing: Boolean indicating if Testing Mode is on. When
        testing mode is on, a connection to the testing database will be made
        instead of the production database. This is useful for unit testing.
        :param logLevel
        """

        self.logger = SEKLogger(__name__, logLevel)

        if testing:
            self.logger.log("Testing Mode is ON.")

        self.configer = MSGConfiger()
        self.dbPassword = self.configer.configOptionValue("Database",
                                                          'db_password')
        self.dbHost = self.configer.configOptionValue("Database", 'db_host')
        self.dbPort = self.configer.configOptionValue("Database", 'db_port')

        if testing:
            self.dbName = self.configer.configOptionValue("Database",
                                                          'testing_db_name')
        else:
            self.dbName = self.configer.configOptionValue("Database", 'db_name')

        self.logger.log(
            "Instantiating DB connector with database {}.".format(self.dbName))

        self.dbUsername = self.configer.configOptionValue("Database",
                                                          'db_username')

        self.conn = self.connectDB()
        if not self.conn:
            raise Exception('DB connection not available.')

        try:
            self.dictCur = self.conn.cursor(
                cursor_factory = psycopg2.extras.DictCursor)
        except AttributeError as error:
            self.logger.log('Error while getting DictCursor: {}'.format(error))
    def __init__(self, filepath = ''):
        """
        Constructor.

        :param testing: Flag indicating if testing mode is on.
        """

        self.logger = SEKLogger(__name__, DEBUG)
        self.configer = SIConfiger()
        self.dbUtil = SEKDBUtil()
        self.dataUtil = SIDataUtil()
        self.logger.log('making new db conn for filepath {}'.format(filepath), SILENT)
        sys.stdout.flush()

        try:
            self.conn = SEKDBConnector(
                dbName = self.configer.configOptionValue('Database', 'db_name'),
                dbHost = self.configer.configOptionValue('Database', 'db_host'),
                dbPort = self.configer.configOptionValue('Database', 'db_port'),
                dbUsername = self.configer.configOptionValue('Database',
                                                             'db_username'),
                dbPassword = self.configer.configOptionValue('Database',
                                                             'db_password')).connectDB()
        except:
            raise Exception("Unable to get DB connection.")
        self.cursor = self.conn.cursor()
        self.exitOnError = False

        # An empty file path is used during creating of meter table entries.
        if filepath == '':
            self.filepath = None
            self.meterID = None
            self.meterDataTable = None
        else:
            self.filepath = filepath
            self.meterID = self.getOrMakeMeterID(self.meterName())
            assert self.meterID is not None
            self.meterDataTable = "MeterData_{}".format(self.meterName())
            # @todo Test existence of meter data table.
        self.timestampColumn = 0 # timestamp col in the raw data
        self.exceptionCount = 0
class SEKDBConnector(object):
    """
    Make and manage a connection to a PostgreSQL database.

    Usage:

        conn = SEKDBConnector().connectDB()
        cursor = conn.cursor()

    """


    def __init__(self, dbName = '', dbHost = '', dbPort = '', dbUsername = '',
                 dbPassword = '', testing = False, logLevel = 'silent'):
        """
        Constructor.

        :param testing: Boolean indicating if Testing Mode is on. When
        testing mode is on, a connection to the testing database will be made
        instead of the production database. This is useful for unit testing.
        :param logLevel
        """

        self.logger = SEKLogger(__name__, logLevel)

        if testing:
            self.logger.log("Testing Mode is ON.")

        self.dbName = dbName
        self.dbHost = dbHost
        self.dbPort = dbPort
        self.dbPassword = dbPassword
        self.dbUsername = dbUsername

        self.logger.log(
            "Instantiating DB connector with database {}.".format(dbName))

        self.conn = self.connectDB()
        if not self.conn:
            self.logger.log('DB connection not available.', 'error')
            sys.exit(-1)

        try:
            self.dictCur = self.conn.cursor(
                cursor_factory = psycopg2.extras.DictCursor)
        except AttributeError as error:
            self.logger.log('Error while getting DictCursor: {}'.format(error))


    def connectDB(self):
        """
        Make the DB connection.
        :returns: DB connection object if successful, otherwise None.
        """

        # @todo Make this method private since the init makes the connection.

        conn = None

        try:
            conn = psycopg2.connect(
                "dbname='{0}' user='******' host='{2}' port='{3}' password='******'".format(self.dbName, self.dbUsername, self.dbHost,
                             self.dbPort, self.dbPassword))
        except Exception as detail:
            self.logger.log(
                "Failed to connect to the database {}: {}.".format(self.dbName,
                                                                   detail),
                'error')
            sys.exit(-1)

        self.logger.log(
            "Opened DB connection to database {}.".format(self.dbName))
        return conn


    def closeDB(self, conn):
        """
        Close a database connection.
        """

        self.logger.log("Closing database {}.".format(self.dbName))
        conn.close()


    def __del__(self):
        """
        Destructor.

        Close the database connection.
        """

        self.logger.log(
            "Closing the DB connection to database {}.".format(self.dbName))
        self.conn.close()