def __init__(self):
"""
Constructor.
"""
self._config = ConfigParser.ConfigParser()
self.logger = SEKLogger(__name__, 'INFO')
self.fileUtil = SEKFileUtil()
# Define tables that will have data inserted. Data will only be inserted
# to tables that are defined here.
self.insertTables = ('MeterData', 'RegisterData', 'RegisterRead',
'Tier', 'Register', 'IntervalReadData',
'Interval', 'Reading', 'EventData', 'Event')
# Check permissions on the config file. Refuse to run if the permissions
# are not set appropriately.
configFilePath = '~/.msg-data-operations.cfg'
if self.fileUtil.isMoreThanOwnerReadableAndWritable(
os.path.expanduser(configFilePath)):
self.logger.log(
"Configuration file permissions are too permissive. Operation "
"will not continue.", 'error')
sys.exit()
try:
self._config.read(['site.cfg', os.path.expanduser(configFilePath)])
except:
self.logger.log(
"Critical error: The data in {} cannot be "
"accessed successfully.".format(configFilePath), 'ERROR')
sys.exit(-1)
def setUp(self):
self.logger = SEKLogger(__name__, 'DEBUG')
self.configer = MSGConfiger()
self.exporter = MSGDBExporter()
self.testDir = 'db_exporter_test'
self.uncompressedTestFilename = 'meco_v3_test_data.sql'
self.compressedTestFilename = 'meco_v3_test_data.sql.gz'
self.exportTestDataPath = self.configer.configOptionValue(
'Testing', 'export_test_data_path')
self.fileUtil = MSGFileUtil()
self.fileChunks = []
self.testDataFileID = ''
self.pyUtil = MSGPythonUtil()
self.timeUtil = MSGTimeUtil()
conn = None
try:
conn = MSGDBConnector().connectDB()
except Exception as detail:
self.logger.log("Exception occurred: {}".format(detail), 'error')
exit(-1)
self.logger.log("conn = {}".format(conn), 'debug')
self.assertIsNotNone(conn)
# Create a temporary working directory.
try:
os.mkdir(self.testDir)
except OSError as detail:
self.logger.log(
'Exception during creation of temp directory: %s' % detail,
'ERROR')
def __init__(self):
"""
Constructor.
"""
warnings.simplefilter('default')
warnings.warn("This module is deprecated in favor of SEKNotifier.",
DeprecationWarning)
self.config = MSGConfiger()
self.logger = SEKLogger(__name__, 'info')
self.connector = MSGDBConnector()
self.conn = self.connector.connectDB()
self.cursor = self.conn.cursor()
self.dbUtil = MSGDBUtil()
self.noticeTable = 'NotificationHistory'
self.notificationHeader = "This is a message from the Hawaii Smart " \
"Energy Project MSG Project notification " \
"system.\n\n"
self.noReplyNotice = '\n\nThis email account is not monitored. No ' \
'replies will originate from this ' \
'account.\n\nYou are receiving this message ' \
'because you are on the recipient list for ' \
'notifications for the Hawaii Smart Energy ' \
'Project.'
def __init__(self):
"""
Constructor.
"""
self._config = ConfigParser.ConfigParser()
self.logger = SEKLogger(__name__, 'INFO')
self.fileUtil = SEKFileUtil()
self.dbUtil = SEKDBUtil()
self.cursor = None
configFilePath = '~/.smart-inverter.cfg'
if self.fileUtil.isMoreThanOwnerReadableAndWritable(
os.path.expanduser(configFilePath)):
self.logger.log(
"Configuration file permissions are too permissive. Operation "
"will not continue.", 'error')
sys.exit(-1)
try:
self._config.read(['site.cfg', os.path.expanduser(configFilePath)])
except:
self.logger.log(
"Critical error: The data in {} cannot be "
"accessed successfully.".format(configFilePath), 'ERROR')
sys.exit(-1)
def __init__(self):
"""
Constructor.
"""
self.logger = SEKLogger(__name__, 'debug')
self.cols = ["wban", "datetime", "datetime", "station_type",
"sky_condition", "sky_condition_flag", "visibility",
"visibility_flag", "weather_type", "weather_type_flag",
"dry_bulb_farenheit", "dry_bulb_farenheit_flag",
"dry_bulb_celsius", "dry_bulb_celsius_flag",
"wet_bulb_farenheit", "wet_bulb_farenheit_flag",
"wet_bulb_celsius", "wet_bulb_celsius_flag",
"dew_point_farenheit", "dew_point_farenheit_flag",
"dew_point_celsius", "dew_point_celsius_flag",
"relative_humidity", "relative_humidity_flag",
"wind_speed", "wind_speed_flag", "wind_direction",
"wind_direction_flag", "value_for_wind_character",
"value_for_wind_character_flag", "station_pressure",
"station_pressure_flag", "pressure_tendency",
"pressure_tendency_flag", "pressure_change",
"pressure_change_flag", "sea_level_pressure",
"sea_level_pressure_flag", "record_type",
"record_type_flag", "hourly_precip", "hourly_precip_flag",
"altimeter", "altimeter_flag"]
def setUp(self):
self.logger = SEKLogger(__name__, 'DEBUG')
self.configer = SIConfiger()
self.conn = SEKDBConnector(
dbName=self.configer.configOptionValue('Database', 'db_name'),
dbHost=self.configer.configOptionValue('Database', 'db_host'),
dbPort=self.configer.configOptionValue('Database', 'db_port'),
dbUsername=self.configer.configOptionValue('Database',
'db_username'),
dbPassword=self.configer.configOptionValue(
'Database', 'db_password')).connectDB()
self.cursor = self.conn.cursor()
self.dbUtil = SEKDBUtil()
self.dataUtil = SIDataUtil()
self.inserter = SingleFileLoader('data/test-meter/log.csv')
self.data = '"2014-07-12 16:22:30",0,,,1187488464896.00,' \
'2322185846784.00,1134697381888.00,35184644096.00,' \
'290857353216.00,10133100822528.00,367.13,' \
'-17660932096.00,1078.01,17660934144.00,-7.86,1.80,8.06,' \
'-0.97,244.01,122.00,32.93,60.01,-7.09,1.42,7.24,8.06,' \
'3.34,8.35,-40.18,-5.68,40.52,516.72,403.12,0,' \
'8797179904.00,47518.67,0,86.03,50.23,4198.40,' \
'281475022848.00,2251868602368.00,0,6820.01,' \
'8796095488.00,0,178.83,188.30,0,620.07,505.19,' \
'288230389841920.02,12668.18,68729384.00,0,-3.68,-4.18,,' \
'1.00,0.79,,3.81,4.25,,-0.97,-0.98,,244.01,,,121.54,' \
'122.46,,31.28,34.59,'
self.testMeterName = 'test-meter'
def __init__(self):
"""
Constructor.
"""
self.logger = SEKLogger(__name__)
self.configer = MSGConfiger()
self.fileUtil = MSGFileUtil()
def __init__(self):
"""
Constructor.
"""
self.logger = SEKLogger(__name__, 'DEBUG')
self.cursor = MSGDBConnector().connectDB().cursor()
self.dbUtil = MSGDBUtil()
def __init__(self, testing=False):
"""
Constructor.
:param testing: True if testing mode is being used.
"""
self.logger = SEKLogger(__name__, 'info')
self.dbUtil = MSGDBUtil()
self.dupeChecker = MSGWeatherDataDupeChecker()
def __init__(self):
"""
Constructor.
"""
self.logger = SEKLogger(__name__, 'debug')
self.mapper = MECOMapper()
self.dupeChecker = MECODupeChecker()
self.dbUtil = MSGDBUtil()
def __init__(self, testing=False):
"""
Constructor.
:param testing: Flag for testing mode.
"""
self.logger = SEKLogger(__name__, 'debug')
self.dbUtil = MSGDBUtil()
def setUp(self):
"""
Constructor.
"""
self.logger = SEKLogger(__name__, 'DEBUG')
self.aggregator = MSGDataAggregator()
self.testStart = '2014-01-02 11:59'
self.testEnd = '2014-01-02 12:14'
self.rawTypes = ['weather', 'egauge', 'circuit', 'irradiance']
def __init__(self):
"""
Constructor.
"""
self.logger = SEKLogger(__name__, 'debug')
self.mecoConfig = MSGConfiger()
self.currentReadingID = 0
self.dbUtil = MSGDBUtil()
def __init__(self, testing=False):
"""
Constructor.
:param testing: Flag indicating if testing mode is on.
"""
self.logger = SEKLogger(__name__)
self.parser = MECOXMLParser(testing)
self.configer = MSGConfiger()
def __init__(self):
"""
Constructor.
"""
print __name__
self.logger = SEKLogger(__name__)
self.connector = MSGDBConnector()
self.dbUtil = MSGDBUtil()
self.notifier = MSGNotifier()
self.configer = MSGConfiger()
def __init__(self):
"""
Constructor.
"""
self.logger = SEKLogger(__name__, 'DEBUG')
self.aggregator = MSGDataAggregator()
self.notifier = MSGNotifier()
self.rawTypes = [x.name for x in list(MSGAggregationTypes)]
self.connector = MSGDBConnector()
self.conn = self.connector.connectDB()
self.cursor = self.conn.cursor()
self.dbUtil = MSGDBUtil()
class MECODataAutoloader(object):
"""
Provide automated loading of MECO energy data from exports in
gzip-compressed XML source data.
"""
def __init__(self):
"""
Constructor.
"""
self.logger = SEKLogger(__name__)
self.configer = MSGConfiger()
self.fileUtil = MSGFileUtil()
def newDataExists(self):
"""
Check the data autoload folder for the presence of new data.
:returns: True if new data exists.
"""
autoloadPath = self.configer.configOptionValue(
'MECO Autoload', 'meco_autoload_new_data_path')
if not self.fileUtil.validDirectory(autoloadPath):
raise Exception('InvalidDirectory', '%s' % autoloadPath)
patterns = ['*.gz']
matchCnt = 0
for root, dirs, filenames in os.walk(autoloadPath):
for pat in patterns:
for filename in fnmatch.filter(filenames, pat):
print filename
matchCnt += 1
if matchCnt > 0:
return True
else:
return False
def loadNewData(self):
"""
Load new data contained in the new data path.
"""
autoloadPath = self.configer.configOptionValue(
'MECO Autoload', 'meco_autoload_new_data_path')
command = self.configer.configOptionValue('MECO Autoload',
'meco_autoload_command')
os.chdir(autoloadPath)
try:
subprocess.check_call(command, shell=True)
except subprocess.CalledProcessError, e:
self.logger.log("An exception occurred: %s" % e, 'error')
def __init__(self,
exitOnError=True,
commitOnEveryInsert=False,
testing=False):
"""
Constructor.
:param testing: if True, the testing DB will be connected instead of
the production DB.
"""
self.logger = SEKLogger(__name__, 'info')
self.configer = MSGConfiger()
self.conn = MSGDBConnector().connectDB()
self.cursor = self.conn.cursor()
self.dbUtil = MSGDBUtil()
self.notifier = MSGNotifier()
self.mathUtil = MSGMathUtil()
self.timeUtil = MSGTimeUtil()
self.nextMinuteCrossing = {}
self.nextMinuteCrossingWithoutSubkeys = None
self.exitOnError = exitOnError
self.commitOnEveryInsert = commitOnEveryInsert
section = 'Aggregation'
tableList = [
'irradiance', 'agg_irradiance', 'weather', 'agg_weather',
'circuit', 'agg_circuit', 'egauge', 'agg_egauge'
]
self.dataParams = {
'weather': ('agg_weather', 'timestamp', ''),
'egauge': ('agg_egauge', 'datetime', 'egauge_id'),
'circuit': ('agg_circuit', 'timestamp', 'circuit'),
'irradiance': ('agg_irradiance', 'timestamp', 'sensor_id')
}
self.columns = {}
# tables[datatype] gives the table name for datatype.
self.tables = {
t: self.configer.configOptionValue(section, '{}_table'.format(t))
for t in tableList
}
for t in self.tables.keys():
self.logger.log('t:{}'.format(t), 'DEBUG')
try:
self.columns[t] = self.dbUtil.columnsString(
self.cursor, self.tables[t])
except TypeError as error:
self.logger.log(
'Ignoring missing table: Error is {}.'.format(error),
'error')
class MECODataAutoloader(object):
"""
Provide automated loading of MECO energy data from exports in
gzip-compressed XML source data.
"""
def __init__(self):
"""
Constructor.
"""
self.logger = SEKLogger(__name__)
self.configer = MSGConfiger()
self.fileUtil = MSGFileUtil()
def newDataExists(self):
"""
Check the data autoload folder for the presence of new data.
:returns: True if new data exists.
"""
autoloadPath = self.configer.configOptionValue("MECO Autoload", "meco_autoload_new_data_path")
if not self.fileUtil.validDirectory(autoloadPath):
raise Exception("InvalidDirectory", "%s" % autoloadPath)
patterns = ["*.gz"]
matchCnt = 0
for root, dirs, filenames in os.walk(autoloadPath):
for pat in patterns:
for filename in fnmatch.filter(filenames, pat):
print filename
matchCnt += 1
if matchCnt > 0:
return True
else:
return False
def loadNewData(self):
"""
Load new data contained in the new data path.
"""
autoloadPath = self.configer.configOptionValue("MECO Autoload", "meco_autoload_new_data_path")
command = self.configer.configOptionValue("MECO Autoload", "meco_autoload_command")
os.chdir(autoloadPath)
try:
subprocess.check_call(command, shell=True)
except subprocess.CalledProcessError, e:
self.logger.log("An exception occurred: %s" % e, "error")
def __init__(self):
"""
Constructor.
"""
self.logger = SEKLogger(__name__, 'DEBUG', useColor=False)
self.timeUtil = MSGTimeUtil()
self.configer = MSGConfiger()
self.fileUtil = MSGFileUtil()
self.pythonUtil = MSGPythonUtil() # for debugging
self.connector = MSGDBConnector()
self.conn = self.connector.connectDB()
self.cursor = self.conn.cursor()
self.dbUtil = MSGDBUtil()
self.notifier = SEKNotifier(
connector=self.connector,
dbUtil=self.dbUtil,
user=self.configer.configOptionValue('Notifications',
'email_username'),
password=self.configer.configOptionValue('Notifications',
'email_password'),
fromaddr=self.configer.configOptionValue('Notifications',
'email_from_address'),
toaddr=self.configer.configOptionValue('Notifications',
'email_recipients'),
testing_toaddr=self.configer.configOptionValue(
'Notifications', 'testing_email_recipients'),
smtp_server_and_port=self.configer.configOptionValue(
'Notifications', 'smtp_server_and_port'))
# Google Drive parameters.
self.clientID = self.configer.configOptionValue(
'Export', 'google_api_client_id')
self.clientSecret = self.configer.configOptionValue(
'Export', 'google_api_client_secret')
self.oauthScope = 'https://www.googleapis.com/auth/drive'
self.oauthConsent = 'urn:ietf:wg:oauth:2.0:oob'
self.googleAPICredentials = ''
self.exportTempWorkPath = self.configer.configOptionValue(
'Export', 'db_export_work_path')
self.credentialPath = self.configer.configOptionValue(
'Export', 'google_api_credentials_path')
self.credentialStorage = Storage('{}/google_api_credentials'.format(
self.credentialPath))
self._driveService = None
self._cloudFiles = None
self.postAgent = 'Maui Smart Grid 1.0.0 DB Exporter'
self.retryDelay = 10
self.availableFilesURL = ''
def __init__(self):
"""
Constructor.
"""
self._config = ConfigParser.ConfigParser()
self.logger = SEKLogger(__name__, 'INFO')
self.fileUtil = SEKFileUtil()
self.dbUtil = SEKDBUtil()
self.cursor = None
configFilePath = '~/.smart-inverter.cfg'
if self.fileUtil.isMoreThanOwnerReadableAndWritable(
os.path.expanduser(configFilePath)):
self.logger.log(
"Configuration file permissions are too permissive. Operation "
"will not continue.", 'error')
sys.exit(-1)
try:
self._config.read(['site.cfg', os.path.expanduser(configFilePath)])
except:
self.logger.log("Critical error: The data in {} cannot be "
"accessed successfully.".format(configFilePath),
'ERROR')
sys.exit(-1)
def setUp(self):
self.logger = SEKLogger(__name__,'DEBUG')
self.configer = SIConfiger()
self.conn = SEKDBConnector(
dbName = self.configer.configOptionValue('Database', 'db_name'),
dbHost = self.configer.configOptionValue('Database', 'db_host'),
dbPort = self.configer.configOptionValue('Database', 'db_port'),
dbUsername = self.configer.configOptionValue('Database',
'db_username'),
dbPassword = self.configer.configOptionValue('Database',
'db_password')).connectDB()
self.cursor = self.conn.cursor()
self.dbUtil = SEKDBUtil()
self.dataUtil = SIDataUtil()
self.inserter = SingleFileLoader('data/test-meter/log.csv')
self.data = '"2014-07-12 16:22:30",0,,,1187488464896.00,' \
'2322185846784.00,1134697381888.00,35184644096.00,' \
'290857353216.00,10133100822528.00,367.13,' \
'-17660932096.00,1078.01,17660934144.00,-7.86,1.80,8.06,' \
'-0.97,244.01,122.00,32.93,60.01,-7.09,1.42,7.24,8.06,' \
'3.34,8.35,-40.18,-5.68,40.52,516.72,403.12,0,' \
'8797179904.00,47518.67,0,86.03,50.23,4198.40,' \
'281475022848.00,2251868602368.00,0,6820.01,' \
'8796095488.00,0,178.83,188.30,0,620.07,505.19,' \
'288230389841920.02,12668.18,68729384.00,0,-3.68,-4.18,,' \
'1.00,0.79,,3.81,4.25,,-0.97,-0.98,,244.01,,,121.54,' \
'122.46,,31.28,34.59,'
self.testMeterName = 'test-meter'
def __init__(self):
"""
Constructor.
"""
self._config = ConfigParser.ConfigParser()
self.logger = SEKLogger(__name__, 'INFO')
self.fileUtil = SEKFileUtil()
# Define tables that will have data inserted. Data will only be inserted
# to tables that are defined here.
self.insertTables = (
'MeterData', 'RegisterData', 'RegisterRead', 'Tier', 'Register',
'IntervalReadData', 'Interval', 'Reading', 'EventData', 'Event')
# Check permissions on the config file. Refuse to run if the permissions
# are not set appropriately.
configFilePath = '~/.msg-data-operations.cfg'
if self.fileUtil.isMoreThanOwnerReadableAndWritable(
os.path.expanduser(configFilePath)):
self.logger.log(
"Configuration file permissions are too permissive. Operation "
"will not continue.", 'error')
sys.exit()
try:
self._config.read(['site.cfg', os.path.expanduser(configFilePath)])
except:
self.logger.log("Critical error: The data in {} cannot be "
"accessed successfully.".format(configFilePath),
'ERROR')
sys.exit(-1)
def setUp(self):
self.logger = SEKLogger(__name__, 'DEBUG')
self.configer = MSGConfiger()
self.exporter = MSGDBExporter()
self.testDir = 'db_exporter_test'
self.uncompressedTestFilename = 'meco_v3_test_data.sql'
self.compressedTestFilename = 'meco_v3_test_data.sql.gz'
self.exportTestDataPath = self.configer.configOptionValue('Testing',
'export_test_data_path')
self.fileUtil = MSGFileUtil()
self.fileChunks = []
self.testDataFileID = ''
self.pyUtil = MSGPythonUtil()
self.timeUtil = MSGTimeUtil()
conn = None
try:
conn = MSGDBConnector().connectDB()
except Exception as detail:
self.logger.log("Exception occurred: {}".format(detail), 'error')
exit(-1)
self.logger.log("conn = {}".format(conn), 'debug')
self.assertIsNotNone(conn)
# Create a temporary working directory.
try:
os.mkdir(self.testDir)
except OSError as detail:
self.logger.log(
'Exception during creation of temp directory: %s' % detail,
'ERROR')
def __init__(self, dbName = '', dbHost = '', dbPort = '', dbUsername = '',
dbPassword = '', testing = False, logLevel = 'silent'):
"""
Constructor.
:param testing: Boolean indicating if Testing Mode is on. When
testing mode is on, a connection to the testing database will be made
instead of the production database. This is useful for unit testing.
:param logLevel
"""
self.logger = SEKLogger(__name__, logLevel)
if testing:
self.logger.log("Testing Mode is ON.")
self.dbName = dbName
self.dbHost = dbHost
self.dbPort = dbPort
self.dbPassword = dbPassword
self.dbUsername = dbUsername
self.logger.log(
"Instantiating DB connector with database {}.".format(dbName))
self.conn = self.connectDB()
if not self.conn:
self.logger.log('DB connection not available.', 'error')
sys.exit(-1)
try:
self.dictCur = self.conn.cursor(
cursor_factory = psycopg2.extras.DictCursor)
except AttributeError as error:
self.logger.log('Error while getting DictCursor: {}'.format(error))
def __init__(self):
"""
Constructor.
"""
warnings.simplefilter('default')
warnings.warn("This module is deprecated in favor of SEKFileUtil.",
DeprecationWarning)
self.logger = SEKLogger(__name__, 'DEBUG')
def __init__(self):
"""
Constructor.
"""
self.logger = SEKLogger(__name__)
self.configer = MSGConfiger()
self.fileUtil = MSGFileUtil()
class MSGWeatherDataDupeChecker(object):
"""
Determine if a duplicate record exists based on the tuple
(WBAN, Date, Time, StationType).
"""
def __init__(self, testing = False):
"""
Constructor.
:param testing: Flag for testing mode.
"""
self.logger = SEKLogger(__name__, 'debug')
self.dbUtil = MSGDBUtil()
def duplicateExists(self, dbCursor, wban, datetime, recordType):
"""
Check for the existence of a duplicate record.
:param dbCursor
:param wban
:param datetime
:param recordType
:returns: True if a duplicate record exists, otherwise False.
"""
tableName = "WeatherNOAA"
sql = """SELECT wban, datetime, record_type FROM \"%s\" WHERE
wban = '%s' AND datetime = '%s' AND record_type = '%s'""" % (
tableName, wban, datetime, recordType)
self.logger.log("sql=%s" % sql, 'debug')
self.logger.log("wban=%s, datetime=%s, record_type=%s" % (
wban, datetime, recordType), 'debug')
self.dbUtil.executeSQL(dbCursor, sql)
rows = dbCursor.fetchall()
if len(rows) > 0:
return True
else:
return False
def setUp(self):
"""
Constructor.
"""
self.logger = SEKLogger(__name__, "DEBUG")
self.aggregator = MSGDataAggregator()
self.testStart = "2014-01-02 11:59"
self.testEnd = "2014-01-02 12:14"
self.rawTypes = ["weather", "egauge", "circuit", "irradiance"]
def __init__(self):
"""
Constructor.
"""
self.logger = SEKLogger(__name__, 'debug')
self.mecoConfig = MSGConfiger()
self.currentReadingID = 0
self.dbUtil = MSGDBUtil()
def __init__(self, testing = False):
"""
Constructor.
:param testing: True if testing mode is being used.
"""
self.logger = SEKLogger(__name__, 'info')
self.dbUtil = MSGDBUtil()
self.dupeChecker = MSGWeatherDataDupeChecker()
def __init__(self):
"""
Constructor.
"""
self.logger = SEKLogger(__name__, 'debug')
self.mapper = MECOMapper()
self.dupeChecker = MECODupeChecker()
self.dbUtil = MSGDBUtil()
def __init__(self):
"""
Constructor.
A database connection is not maintained here to keep this class
lightweight.
"""
self.logger = SEKLogger(__name__, DEBUG)
self.configer = MSGConfiger()
self.url = self.configer.configOptionValue('Weather Data',
'weather_data_url')
self.pattern = self.configer.configOptionValue('Weather Data',
'weather_data_pattern')
self.fileList = []
self.dateList = [] # List of dates corresponding weather data files.
self.fillFileListAndDateList()
self.dbUtil = MSGDBUtil()
def __init__(self, testing = False):
"""
Constructor.
:param testing: Flag for testing mode.
"""
self.logger = SEKLogger(__name__, 'debug')
self.dbUtil = MSGDBUtil()
class MSGWeatherDataDupeChecker(object):
"""
Determine if a duplicate record exists based on the tuple
(WBAN, Date, Time, StationType).
"""
def __init__(self, testing=False):
"""
Constructor.
:param testing: Flag for testing mode.
"""
self.logger = SEKLogger(__name__, 'debug')
self.dbUtil = MSGDBUtil()
def duplicateExists(self, dbCursor, wban, datetime, recordType):
"""
Check for the existence of a duplicate record.
:param dbCursor
:param wban
:param datetime
:param recordType
:returns: True if a duplicate record exists, otherwise False.
"""
tableName = "WeatherNOAA"
sql = """SELECT wban, datetime, record_type FROM \"%s\" WHERE
wban = '%s' AND datetime = '%s' AND record_type = '%s'""" % (
tableName, wban, datetime, recordType)
self.logger.log("sql=%s" % sql, 'debug')
self.logger.log(
"wban=%s, datetime=%s, record_type=%s" %
(wban, datetime, recordType), 'debug')
self.dbUtil.executeSQL(dbCursor, sql)
rows = dbCursor.fetchall()
if len(rows) > 0:
return True
else:
return False
def setUp(self):
self.i = MECODBInserter()
# Connect to the testing database.
self.connector = MSGDBConnector(testing=True)
self.conn = self.connector.connectDB()
self.lastSeqVal = None
# Does this work having the dictCur be in another class?
self.dictCur = self.connector.dictCur
self.cursor = self.conn.cursor()
self.deleter = MECODBDeleter()
self.tableName = 'MeterData'
self.columnName = 'meter_data_id'
self.configer = MSGConfiger()
self.logger = SEKLogger(__name__, 'debug')
self.dbUtil = MSGDBUtil()
class MSGTimeUtilTester(unittest.TestCase):
def setUp(self):
self.logger = SEKLogger(__name__, 'debug')
self.timeUtil = MSGTimeUtil()
def test_concise_now(self):
conciseNow = self.timeUtil.conciseNow()
self.logger.log(conciseNow)
pattern = '\d+-\d+-\d+_\d+'
result = re.match(pattern, conciseNow)
self.assertTrue(result is not None,
"Concise now matches the regex pattern.")
def test_split_dates(self):
start = dt(2014, 01, 07)
end = dt(2014, 04, 04)
print self.timeUtil.splitDates(start, end)
self.assertEqual(len(self.timeUtil.splitDates(start, end)), 4,
'Unexpected date count.')
class SIDataUtilTester(unittest.TestCase):
"""
"""
def setUp(self):
self.dataUtil = SIDataUtil()
self.logger = SEKLogger(__name__)
def test_find_max_timestamp(self):
filePath = 'data/test-meter/log.csv'
self.assertEquals(self.dataUtil.maxTimeStamp(filePath),
datetime.strptime('2014-03-10 23:59:00',
'%Y-%m-%d %H:%M:%S'))
def test_find_max_timestamp_db(self):
# @todo test with a static testing DB
meter = '001EC6051A0D'
self.logger.log(self.dataUtil.maxTimeStampDB(meter))
def __init__(self, testing = False):
"""
Constructor.
:param testing: Flag indicating if testing mode is on.
"""
self.logger = SEKLogger(__name__)
self.parser = MECOXMLParser(testing)
self.configer = MSGConfiger()
def __init__(self):
"""
Constructor.
"""
self.logger = SEKLogger(__name__)
self.viewPVReadingsinNonMLH = ''
self.lastDateProcessed = None
self.connector = MSGDBConnector()
self.conn = self.connector.connectDB()
def __init__(self, testing=False, logLevel='silent'):
"""
Constructor.
:param testing: Boolean indicating if Testing Mode is on. When
testing mode is on, a connection to the testing database will be made
instead of the production database. This is useful for unit testing.
:param logLevel
"""
self.logger = SEKLogger(__name__, logLevel)
if testing:
self.logger.log("Testing Mode is ON.")
self.configer = MSGConfiger()
self.dbPassword = self.configer.configOptionValue(
"Database", 'db_password')
self.dbHost = self.configer.configOptionValue("Database", 'db_host')
self.dbPort = self.configer.configOptionValue("Database", 'db_port')
if testing:
self.dbName = self.configer.configOptionValue(
"Database", 'testing_db_name')
else:
self.dbName = self.configer.configOptionValue(
"Database", 'db_name')
self.logger.log("Instantiating DB connector with database {}.".format(
self.dbName))
self.dbUsername = self.configer.configOptionValue(
"Database", 'db_username')
self.conn = self.connectDB()
if not self.conn:
raise Exception('DB connection not available.')
try:
self.dictCur = self.conn.cursor(
cursor_factory=psycopg2.extras.DictCursor)
except AttributeError as error:
self.logger.log('Error while getting DictCursor: {}'.format(error))
def __init__(self, filepath=''):
"""
Constructor.
:param testing: Flag indicating if testing mode is on.
"""
self.logger = SEKLogger(__name__, DEBUG)
self.configer = SIConfiger()
self.dbUtil = SEKDBUtil()
self.dataUtil = SIDataUtil()
self.logger.log('making new db conn for filepath {}'.format(filepath),
SILENT)
sys.stdout.flush()
try:
self.conn = SEKDBConnector(
dbName=self.configer.configOptionValue('Database', 'db_name'),
dbHost=self.configer.configOptionValue('Database', 'db_host'),
dbPort=self.configer.configOptionValue('Database', 'db_port'),
dbUsername=self.configer.configOptionValue(
'Database', 'db_username'),
dbPassword=self.configer.configOptionValue(
'Database', 'db_password')).connectDB()
except:
raise Exception("Unable to get DB connection.")
self.cursor = self.conn.cursor()
self.exitOnError = False
# An empty file path is used during creating of meter table entries.
if filepath == '':
self.filepath = None
self.meterID = None
self.meterDataTable = None
else:
self.filepath = filepath
self.meterID = self.getOrMakeMeterID(self.meterName())
assert self.meterID is not None
self.meterDataTable = "MeterData_{}".format(self.meterName())
# @todo Test existence of meter data table.
self.timestampColumn = 0 # timestamp col in the raw data
self.exceptionCount = 0
def __init__(self):
"""
Constructor.
"""
print __name__
self.logger = SEKLogger(__name__)
self.connector = MSGDBConnector()
self.dbUtil = MSGDBUtil()
self.notifier = MSGNotifier()
self.configer = MSGConfiger()
def __init__(self):
"""
Constructor.
"""
self.logger = SEKLogger(__name__, 'DEBUG', useColor = False)
self.timeUtil = MSGTimeUtil()
self.configer = MSGConfiger()
self.fileUtil = MSGFileUtil()
self.pythonUtil = MSGPythonUtil() # for debugging
self.connector = MSGDBConnector()
self.conn = self.connector.connectDB()
self.cursor = self.conn.cursor()
self.dbUtil = MSGDBUtil()
self.notifier = SEKNotifier(connector = self.connector,
dbUtil = self.dbUtil,
user = self.configer.configOptionValue(
'Notifications', 'email_username'),
password = self.configer.configOptionValue(
'Notifications', 'email_password'),
fromaddr = self.configer.configOptionValue(
'Notifications', 'email_from_address'),
toaddr = self.configer.configOptionValue(
'Notifications', 'email_recipients'),
testing_toaddr =
self.configer.configOptionValue(
'Notifications',
'testing_email_recipients'),
smtp_server_and_port =
self.configer.configOptionValue(
'Notifications',
'smtp_server_and_port'))
# Google Drive parameters.
self.clientID = self.configer.configOptionValue('Export',
'google_api_client_id')
self.clientSecret = self.configer.configOptionValue('Export',
'google_api_client_secret')
self.oauthScope = 'https://www.googleapis.com/auth/drive'
self.oauthConsent = 'urn:ietf:wg:oauth:2.0:oob'
self.googleAPICredentials = ''
self.exportTempWorkPath = self.configer.configOptionValue('Export',
'db_export_work_path')
self.credentialPath = self.configer.configOptionValue('Export',
'google_api_credentials_path')
self.credentialStorage = Storage(
'{}/google_api_credentials'.format(self.credentialPath))
self._driveService = None
self._cloudFiles = None
self.postAgent = 'Maui Smart Grid 1.0.0 DB Exporter'
self.retryDelay = 10
self.availableFilesURL = ''
class WeatherDataLoadingTester(unittest.TestCase):
def setUp(self):
self.weatherUtil = MSGWeatherDataUtil()
self.logger = SEKLogger(__name__, 'DEBUG')
self.dbConnector = MSGDBConnector()
self.cursor = self.dbConnector.conn.cursor()
self.configer = MSGConfiger()
def testLoadDataSinceLastLoaded(self):
"""
Data should be loaded since the last data present in the database.
"""
pass
def testRetrieveDataSinceLastLoaded(self):
"""
Data since the last loaded date is retrieved.
"""
pass
def testGetLastLoadedDate(self):
myDate = self.weatherUtil.getLastDateLoaded(self.cursor).strftime(
"%Y-%m-%d %H:%M:%S")
pattern = '^(\d+-\d+-\d+\s\d+:\d+:\d+)$'
match = re.match(pattern, myDate)
assert match and (match.group(1) == myDate), "Date format is valid."
def testWeatherDataPattern(self):
myPattern = self.configer.configOptionValue('Weather Data',
'weather_data_pattern')
testString = """<A HREF="someURL">QCLCD201208.zip</A>"""
match = re.match(myPattern, testString)
self.logger.log("pattern = %s" % myPattern, 'info')
if match:
self.logger.log("match = %s" % match)
self.logger.log("match group = %s" % match.group(1))
else:
self.logger.log("match not found")
assert match and match.group(
1) == 'QCLCD201208.zip', "Download filename was matched."
def testWeatherDataURL(self):
myURL = self.configer.configOptionValue('Weather Data',
'weather_data_url')
pass
def __init__(self,
dbName='',
dbHost='',
dbPort='',
dbUsername='',
dbPassword='',
testing=False,
logLevel='silent'):
"""
Constructor.
:param testing: Boolean indicating if Testing Mode is on. When
testing mode is on, a connection to the testing database will be made
instead of the production database. This is useful for unit testing.
:param logLevel
"""
self.logger = SEKLogger(__name__, logLevel)
if testing:
self.logger.log("Testing Mode is ON.")
self.dbName = dbName
self.dbHost = dbHost
self.dbPort = dbPort
self.dbPassword = dbPassword
self.dbUsername = dbUsername
self.logger.log(
"Instantiating DB connector with database {}.".format(dbName))
self.conn = self.connectDB()
if not self.conn:
self.logger.log('DB connection not available.', 'error')
sys.exit(-1)
try:
self.dictCur = self.conn.cursor(
cursor_factory=psycopg2.extras.DictCursor)
except AttributeError as error:
self.logger.log('Error while getting DictCursor: {}'.format(error))
def __init__(self):
"""
Constructor.
"""
self.logger = SEKLogger(__name__, 'DEBUG')
self.aggregator = MSGDataAggregator()
self.notifier = MSGNotifier()
self.rawTypes = [x.name for x in list(MSGAggregationTypes)]
self.connector = MSGDBConnector()
self.conn = self.connector.connectDB()
self.cursor = self.conn.cursor()
self.dbUtil = MSGDBUtil()
class SIConfiger(object):
"""
Supports site-level config for the Smart Grid PV Inverter project.
The default path is ~/.smart-inverter.cfg.
Usage:
configer = SIConfiger()
"""
def __init__(self):
"""
Constructor.
"""
self._config = ConfigParser.ConfigParser()
self.logger = SEKLogger(__name__, 'INFO')
self.fileUtil = SEKFileUtil()
self.dbUtil = SEKDBUtil()
self.cursor = None
configFilePath = '~/.smart-inverter.cfg'
if self.fileUtil.isMoreThanOwnerReadableAndWritable(
os.path.expanduser(configFilePath)):
self.logger.log(
"Configuration file permissions are too permissive. Operation "
"will not continue.", 'error')
sys.exit(-1)
try:
self._config.read(['site.cfg', os.path.expanduser(configFilePath)])
except:
self.logger.log(
"Critical error: The data in {} cannot be "
"accessed successfully.".format(configFilePath), 'ERROR')
sys.exit(-1)
def configOptionValue(self, section, option):
"""
Get a configuration value from the local configuration file.
:param section: String of section in config file.
:param option: String of option in config file.
:returns: The value contained in the configuration file.
"""
try:
configValue = self._config.get(section, option)
if configValue == "True":
return True
elif configValue == "False":
return False
else:
return configValue
except:
self.logger.log(
"Failed when getting configuration option {} in section {"
"}.".format(option, section), 'error')
sys.exit(-1)
def __init__(self, exitOnError=True, commitOnEveryInsert=False, testing=False):
"""
Constructor.
:param testing: if True, the testing DB will be connected instead of
the production DB.
"""
self.logger = SEKLogger(__name__, "info")
self.configer = MSGConfiger()
self.conn = MSGDBConnector().connectDB()
self.cursor = self.conn.cursor()
self.dbUtil = MSGDBUtil()
self.notifier = MSGNotifier()
self.mathUtil = MSGMathUtil()
self.timeUtil = MSGTimeUtil()
self.nextMinuteCrossing = {}
self.nextMinuteCrossingWithoutSubkeys = None
self.exitOnError = exitOnError
self.commitOnEveryInsert = commitOnEveryInsert
section = "Aggregation"
tableList = [
"irradiance",
"agg_irradiance",
"weather",
"agg_weather",
"circuit",
"agg_circuit",
"egauge",
"agg_egauge",
]
self.dataParams = {
"weather": ("agg_weather", "timestamp", ""),
"egauge": ("agg_egauge", "datetime", "egauge_id"),
"circuit": ("agg_circuit", "timestamp", "circuit"),
"irradiance": ("agg_irradiance", "timestamp", "sensor_id"),
}
self.columns = {}
# tables[datatype] gives the table name for datatype.
self.tables = {t: self.configer.configOptionValue(section, "{}_table".format(t)) for t in tableList}
for t in self.tables.keys():
self.logger.log("t:{}".format(t), "DEBUG")
try:
self.columns[t] = self.dbUtil.columnsString(self.cursor, self.tables[t])
except TypeError as error:
self.logger.log("Ignoring missing table: Error is {}.".format(error), "error")
def __init__(self):
"""
Constructor.
A database connection is not maintained here to keep this class
lightweight.
"""
self.logger = SEKLogger(__name__, DEBUG)
self.configer = MSGConfiger()
self.url = self.configer.configOptionValue('Weather Data',
'weather_data_url')
self.pattern = self.configer.configOptionValue('Weather Data',
'weather_data_pattern')
self.fileList = []
self.dateList = [] # List of dates corresponding weather data files.
self.fillFileListAndDateList()
self.dbUtil = MSGDBUtil()
def setUp(self):
self.i = MECODBInserter()
# Connect to the testing database.
self.connector = MSGDBConnector(testing = True)
self.conn = self.connector.connectDB()
self.lastSeqVal = None
# Does this work having the dictCur be in another class?
self.dictCur = self.connector.dictCur
self.cursor = self.conn.cursor()
self.deleter = MECODBDeleter()
self.tableName = 'MeterData'
self.columnName = 'meter_data_id'
self.configer = MSGConfiger()
self.logger = SEKLogger(__name__, 'debug')
self.dbUtil = MSGDBUtil()
def __init__(self, testing = False, logLevel = 'silent'):
"""
Constructor.
:param testing: Boolean indicating if Testing Mode is on. When
testing mode is on, a connection to the testing database will be made
instead of the production database. This is useful for unit testing.
:param logLevel
"""
self.logger = SEKLogger(__name__, logLevel)
if testing:
self.logger.log("Testing Mode is ON.")
self.configer = MSGConfiger()
self.dbPassword = self.configer.configOptionValue("Database",
'db_password')
self.dbHost = self.configer.configOptionValue("Database", 'db_host')
self.dbPort = self.configer.configOptionValue("Database", 'db_port')
if testing:
self.dbName = self.configer.configOptionValue("Database",
'testing_db_name')
else:
self.dbName = self.configer.configOptionValue("Database", 'db_name')
self.logger.log(
"Instantiating DB connector with database {}.".format(self.dbName))
self.dbUsername = self.configer.configOptionValue("Database",
'db_username')
self.conn = self.connectDB()
if not self.conn:
raise Exception('DB connection not available.')
try:
self.dictCur = self.conn.cursor(
cursor_factory = psycopg2.extras.DictCursor)
except AttributeError as error:
self.logger.log('Error while getting DictCursor: {}'.format(error))
def __init__(self, filepath = ''):
"""
Constructor.
:param testing: Flag indicating if testing mode is on.
"""
self.logger = SEKLogger(__name__, DEBUG)
self.configer = SIConfiger()
self.dbUtil = SEKDBUtil()
self.dataUtil = SIDataUtil()
self.logger.log('making new db conn for filepath {}'.format(filepath), SILENT)
sys.stdout.flush()
try:
self.conn = SEKDBConnector(
dbName = self.configer.configOptionValue('Database', 'db_name'),
dbHost = self.configer.configOptionValue('Database', 'db_host'),
dbPort = self.configer.configOptionValue('Database', 'db_port'),
dbUsername = self.configer.configOptionValue('Database',
'db_username'),
dbPassword = self.configer.configOptionValue('Database',
'db_password')).connectDB()
except:
raise Exception("Unable to get DB connection.")
self.cursor = self.conn.cursor()
self.exitOnError = False
# An empty file path is used during creating of meter table entries.
if filepath == '':
self.filepath = None
self.meterID = None
self.meterDataTable = None
else:
self.filepath = filepath
self.meterID = self.getOrMakeMeterID(self.meterName())
assert self.meterID is not None
self.meterDataTable = "MeterData_{}".format(self.meterName())
# @todo Test existence of meter data table.
self.timestampColumn = 0 # timestamp col in the raw data
self.exceptionCount = 0
class SEKDBConnector(object):
"""
Make and manage a connection to a PostgreSQL database.
Usage:
conn = SEKDBConnector().connectDB()
cursor = conn.cursor()
"""
def __init__(self, dbName = '', dbHost = '', dbPort = '', dbUsername = '',
dbPassword = '', testing = False, logLevel = 'silent'):
"""
Constructor.
:param testing: Boolean indicating if Testing Mode is on. When
testing mode is on, a connection to the testing database will be made
instead of the production database. This is useful for unit testing.
:param logLevel
"""
self.logger = SEKLogger(__name__, logLevel)
if testing:
self.logger.log("Testing Mode is ON.")
self.dbName = dbName
self.dbHost = dbHost
self.dbPort = dbPort
self.dbPassword = dbPassword
self.dbUsername = dbUsername
self.logger.log(
"Instantiating DB connector with database {}.".format(dbName))
self.conn = self.connectDB()
if not self.conn:
self.logger.log('DB connection not available.', 'error')
sys.exit(-1)
try:
self.dictCur = self.conn.cursor(
cursor_factory = psycopg2.extras.DictCursor)
except AttributeError as error:
self.logger.log('Error while getting DictCursor: {}'.format(error))
def connectDB(self):
"""
Make the DB connection.
:returns: DB connection object if successful, otherwise None.
"""
# @todo Make this method private since the init makes the connection.
conn = None
try:
conn = psycopg2.connect(
"dbname='{0}' user='******' host='{2}' port='{3}' password='******'".format(self.dbName, self.dbUsername, self.dbHost,
self.dbPort, self.dbPassword))
except Exception as detail:
self.logger.log(
"Failed to connect to the database {}: {}.".format(self.dbName,
detail),
'error')
sys.exit(-1)
self.logger.log(
"Opened DB connection to database {}.".format(self.dbName))
return conn
def closeDB(self, conn):
"""
Close a database connection.
"""
self.logger.log("Closing database {}.".format(self.dbName))
conn.close()
def __del__(self):
"""
Destructor.
Close the database connection.
"""
self.logger.log(
"Closing the DB connection to database {}.".format(self.dbName))
self.conn.close()
