def setUp(self):
self.logger = SEKLogger(__name__, 'DEBUG')
self.configer = MSGConfiger()
self.exporter = MSGDBExporter()
self.testDir = 'db_exporter_test'
self.uncompressedTestFilename = 'meco_v3_test_data.sql'
self.compressedTestFilename = 'meco_v3_test_data.sql.gz'
self.exportTestDataPath = self.configer.configOptionValue(
'Testing', 'export_test_data_path')
self.fileUtil = MSGFileUtil()
self.fileChunks = []
self.testDataFileID = ''
self.pyUtil = MSGPythonUtil()
self.timeUtil = MSGTimeUtil()
conn = None
try:
conn = MSGDBConnector().connectDB()
except Exception as detail:
self.logger.log("Exception occurred: {}".format(detail), 'error')
exit(-1)
self.logger.log("conn = {}".format(conn), 'debug')
self.assertIsNotNone(conn)
# Create a temporary working directory.
try:
os.mkdir(self.testDir)
except OSError as detail:
self.logger.log(
'Exception during creation of temp directory: %s' % detail,
'ERROR')
def __init__(self):
"""
Constructor.
"""
warnings.simplefilter('default')
warnings.warn("This module is deprecated in favor of SEKNotifier.",
DeprecationWarning)
self.config = MSGConfiger()
self.logger = SEKLogger(__name__, 'info')
self.connector = MSGDBConnector()
self.conn = self.connector.connectDB()
self.cursor = self.conn.cursor()
self.dbUtil = MSGDBUtil()
self.noticeTable = 'NotificationHistory'
self.notificationHeader = "This is a message from the Hawaii Smart " \
"Energy Project MSG Project notification " \
"system.\n\n"
self.noReplyNotice = '\n\nThis email account is not monitored. No ' \
'replies will originate from this ' \
'account.\n\nYou are receiving this message ' \
'because you are on the recipient list for ' \
'notifications for the Hawaii Smart Energy ' \
'Project.'
def __init__(self):
"""
Constructor.
"""
self.logger = SEKLogger(__name__)
self.configer = MSGConfiger()
self.fileUtil = MSGFileUtil()
def __init__(self):
"""
Constructor.
"""
self.logger = SEKLogger(__name__, 'debug')
self.mecoConfig = MSGConfiger()
self.currentReadingID = 0
self.dbUtil = MSGDBUtil()
def __init__(self, testing=False):
"""
Constructor.
:param testing: Flag indicating if testing mode is on.
"""
self.logger = SEKLogger(__name__)
self.parser = MECOXMLParser(testing)
self.configer = MSGConfiger()
def __init__(self):
"""
Constructor.
"""
print __name__
self.logger = SEKLogger(__name__)
self.connector = MSGDBConnector()
self.dbUtil = MSGDBUtil()
self.notifier = MSGNotifier()
self.configer = MSGConfiger()
def __init__(self,
exitOnError=True,
commitOnEveryInsert=False,
testing=False):
"""
Constructor.
:param testing: if True, the testing DB will be connected instead of
the production DB.
"""
self.logger = SEKLogger(__name__, 'info')
self.configer = MSGConfiger()
self.conn = MSGDBConnector().connectDB()
self.cursor = self.conn.cursor()
self.dbUtil = MSGDBUtil()
self.notifier = MSGNotifier()
self.mathUtil = MSGMathUtil()
self.timeUtil = MSGTimeUtil()
self.nextMinuteCrossing = {}
self.nextMinuteCrossingWithoutSubkeys = None
self.exitOnError = exitOnError
self.commitOnEveryInsert = commitOnEveryInsert
section = 'Aggregation'
tableList = [
'irradiance', 'agg_irradiance', 'weather', 'agg_weather',
'circuit', 'agg_circuit', 'egauge', 'agg_egauge'
]
self.dataParams = {
'weather': ('agg_weather', 'timestamp', ''),
'egauge': ('agg_egauge', 'datetime', 'egauge_id'),
'circuit': ('agg_circuit', 'timestamp', 'circuit'),
'irradiance': ('agg_irradiance', 'timestamp', 'sensor_id')
}
self.columns = {}
# tables[datatype] gives the table name for datatype.
self.tables = {
t: self.configer.configOptionValue(section, '{}_table'.format(t))
for t in tableList
}
for t in self.tables.keys():
self.logger.log('t:{}'.format(t), 'DEBUG')
try:
self.columns[t] = self.dbUtil.columnsString(
self.cursor, self.tables[t])
except TypeError as error:
self.logger.log(
'Ignoring missing table: Error is {}.'.format(error),
'error')
def __init__(self):
"""
Constructor.
"""
self.logger = SEKLogger(__name__, 'DEBUG', useColor=False)
self.timeUtil = MSGTimeUtil()
self.configer = MSGConfiger()
self.fileUtil = MSGFileUtil()
self.pythonUtil = MSGPythonUtil() # for debugging
self.connector = MSGDBConnector()
self.conn = self.connector.connectDB()
self.cursor = self.conn.cursor()
self.dbUtil = MSGDBUtil()
self.notifier = SEKNotifier(
connector=self.connector,
dbUtil=self.dbUtil,
user=self.configer.configOptionValue('Notifications',
'email_username'),
password=self.configer.configOptionValue('Notifications',
'email_password'),
fromaddr=self.configer.configOptionValue('Notifications',
'email_from_address'),
toaddr=self.configer.configOptionValue('Notifications',
'email_recipients'),
testing_toaddr=self.configer.configOptionValue(
'Notifications', 'testing_email_recipients'),
smtp_server_and_port=self.configer.configOptionValue(
'Notifications', 'smtp_server_and_port'))
# Google Drive parameters.
self.clientID = self.configer.configOptionValue(
'Export', 'google_api_client_id')
self.clientSecret = self.configer.configOptionValue(
'Export', 'google_api_client_secret')
self.oauthScope = 'https://www.googleapis.com/auth/drive'
self.oauthConsent = 'urn:ietf:wg:oauth:2.0:oob'
self.googleAPICredentials = ''
self.exportTempWorkPath = self.configer.configOptionValue(
'Export', 'db_export_work_path')
self.credentialPath = self.configer.configOptionValue(
'Export', 'google_api_credentials_path')
self.credentialStorage = Storage('{}/google_api_credentials'.format(
self.credentialPath))
self._driveService = None
self._cloudFiles = None
self.postAgent = 'Maui Smart Grid 1.0.0 DB Exporter'
self.retryDelay = 10
self.availableFilesURL = ''
def __init__(self):
self.configer = MSGConfiger()
self.pool = None
self.fileList = []
self.dateList = []
self.weatherUtil = MSGWeatherDataUtil()
global weatherDataPath
weatherDataPath = self.configer.configOptionValue(
WEATHER_DATA, 'weather_data_path')
global weatherDataURL
weatherDataURL = self.configer.configOptionValue(
WEATHER_DATA, 'weather_data_url')
global weatherDataPattern
weatherDataPattern = self.configer.configOptionValue(
WEATHER_DATA, 'weather_data_pattern')
def __init__(self):
"""
Constructor.
A database connection is not maintained here to keep this class
lightweight.
"""
self.logger = SEKLogger(__name__, DEBUG)
self.configer = MSGConfiger()
self.url = self.configer.configOptionValue('Weather Data',
'weather_data_url')
self.pattern = self.configer.configOptionValue('Weather Data',
'weather_data_pattern')
self.fileList = []
self.dateList = [] # List of dates corresponding weather data files.
self.fillFileListAndDateList()
self.dbUtil = MSGDBUtil()
def setUp(self):
self.i = MECODBInserter()
# Connect to the testing database.
self.connector = MSGDBConnector(testing=True)
self.conn = self.connector.connectDB()
self.lastSeqVal = None
# Does this work having the dictCur be in another class?
self.dictCur = self.connector.dictCur
self.cursor = self.conn.cursor()
self.deleter = MECODBDeleter()
self.tableName = 'MeterData'
self.columnName = 'meter_data_id'
self.configer = MSGConfiger()
self.logger = SEKLogger(__name__, 'debug')
self.dbUtil = MSGDBUtil()
def __init__(self, testing=False, logLevel='silent'):
"""
Constructor.
:param testing: Boolean indicating if Testing Mode is on. When
testing mode is on, a connection to the testing database will be made
instead of the production database. This is useful for unit testing.
:param logLevel
"""
self.logger = SEKLogger(__name__, logLevel)
if testing:
self.logger.log("Testing Mode is ON.")
self.configer = MSGConfiger()
self.dbPassword = self.configer.configOptionValue(
"Database", 'db_password')
self.dbHost = self.configer.configOptionValue("Database", 'db_host')
self.dbPort = self.configer.configOptionValue("Database", 'db_port')
if testing:
self.dbName = self.configer.configOptionValue(
"Database", 'testing_db_name')
else:
self.dbName = self.configer.configOptionValue(
"Database", 'db_name')
self.logger.log("Instantiating DB connector with database {}.".format(
self.dbName))
self.dbUsername = self.configer.configOptionValue(
"Database", 'db_username')
self.conn = self.connectDB()
if not self.conn:
raise Exception('DB connection not available.')
try:
self.dictCur = self.conn.cursor(
cursor_factory=psycopg2.extras.DictCursor)
except AttributeError as error:
self.logger.log('Error while getting DictCursor: {}'.format(error))
def setUp(self):
self.m = MECOMapper()
self.testColumns = self.m.dbColsMeterData.values()
self.testLabels = self.m.dbColsMeterData.keys()
self.testTableName = 'MeterData'
self.config = MSGConfiger()
def setUp(self):
self.weatherUtil = MSGWeatherDataUtil()
self.logger = SEKLogger(__name__, 'DEBUG')
self.dbConnector = MSGDBConnector()
self.cursor = self.dbConnector.conn.cursor()
self.configer = MSGConfiger()
def setUp(self):
self.logger = SEKLogger(__name__)
self.notifier = MSGNotifier()
self.configer = MSGConfiger()
if __name__ == '__main__':
processCommandLineArguments()
filename = COMMAND_LINE_ARGS.filename
success = True
anyFailure = False
connector = MSGDBConnector(testing = COMMAND_LINE_ARGS.testing)
conn = connector.connectDB()
cur = conn.cursor()
dbUtil = MSGDBUtil()
notifier = MSGNotifier()
msgBody = ''
configer = MSGConfiger()
if COMMAND_LINE_ARGS.testing:
dbName = configer.configOptionValue("Database", "testing_db_name")
else:
dbName = configer.configOptionValue("Database", "db_name")
msg = ("Loading Meter Location History data in file %s to database %s.\n" % (
filename, dbName))
sys.stderr.write(msg)
msgBody += msg
f = open(filename, "r")
reader = csv.reader(f)
data = []
def test_init(self):
# @todo Improve this test by using direct introspection instead of
# this roundabout method.
localConfiger = MSGConfiger()
self.assertIsInstance(self.configer, type(localConfiger))
def setUp(self):
self.configer = MSGConfiger()
def setUp(self):
self.connector = MSGDBConnector(True)
self.conn = self.connector.connectDB()
self.configer = MSGConfiger()
def __init__(self, testing = False):
"""
Constructor.
:param testing: (optional) Boolean indicating if Testing Mode is on.
"""
self.logger = SEKLogger(__name__, 'silent')
if (testing):
self.logger.log("Testing Mode is ON.", 'info')
self.debug = False
self.configer = MSGConfiger()
if self.configer.configOptionValue("Debugging", 'debug') == True:
self.debug = True
self.util = MSGDBUtil()
self.mapper = MECOMapper()
self.connector = MSGDBConnector(testing)
self.conn = self.connector.connectDB()
self.filename = None
self.fileObject = None
self.processForInsertElementCount = 0
self.inserter = MECODBInserter()
self.insertDataIntoDatabase = False
# Count number of times sections in source data are encountered.
self.tableNameCount = {'SSNExportDocument': 0, 'MeterData': 0,
'RegisterData': 0, 'RegisterRead': 0, 'Tier': 0,
'Register': 0, 'IntervalReadData': 0,
'Interval': 0, 'Reading': 0, 'IntervalStatus': 0,
'ChannelStatus': 0, 'EventData': 0, 'Event': 0}
# Use this dictionary to track which channels were processed when
# readings are being processed. this is to prevent duplicate channel
# data from being inserted.
self.channelProcessed = {}
self.initChannelProcessed()
# Tables to be inserted to.
self.insertTables = self.configer.insertTables
self.lastSeqVal = None
self.fKeyVal = None
self.lastTable = None
self.fkDeterminer = MECOFKDeterminer()
self.dupeChecker = MECODupeChecker()
self.currentMeterName = None
self.currentIntervalEndTime = None
self.currentRegisterReadReadTime = None
self.dupesExist = False
self.channelDupeExists = False # For Reading dupes.
self.numberDupeExists = False # For Register dupes.
self.eventTimeDupeExists = False # For Event dupes.
self.commitCount = 0
self.readingDupeOnInsertCount = 0 # For Reading dupes.
self.registerDupeOnInsertCount = 0 # For Register dupes.
self.eventDupeOnInsertCount = 0 # For Event dupes.
self.dataProcessCount = 0
self.readingDupeCheckCount = 0 # For Reading dupes.
self.registerDupeCheckCount = 0 # For Register dupes.
self.eventDupeCheckCount = 0 # For Event dupes.
self.insertCount = 0
self.cumulativeInsertCount = 0
self.nonProcessForInsertElementCount = 0
self.readingInsertCount = 0
self.registerInsertCount = 0
self.eventInsertCount = 0
self.totalReadingInsertCount = 0
self.totalRegisterInsertCount = 0
self.totalEventInsertCount = 0
self.totalReadingDupeOnInsertCount = 0
self.totalRegisterDupeOnInsertCount = 0
self.totalEventDupeOnInsertCount = 0
