def setUp(self): self.logger = SEKLogger(__name__, 'DEBUG') self.configer = MSGConfiger() self.exporter = MSGDBExporter() self.testDir = 'db_exporter_test' self.uncompressedTestFilename = 'meco_v3_test_data.sql' self.compressedTestFilename = 'meco_v3_test_data.sql.gz' self.exportTestDataPath = self.configer.configOptionValue( 'Testing', 'export_test_data_path') self.fileUtil = MSGFileUtil() self.fileChunks = [] self.testDataFileID = '' self.pyUtil = MSGPythonUtil() self.timeUtil = MSGTimeUtil() conn = None try: conn = MSGDBConnector().connectDB() except Exception as detail: self.logger.log("Exception occurred: {}".format(detail), 'error') exit(-1) self.logger.log("conn = {}".format(conn), 'debug') self.assertIsNotNone(conn) # Create a temporary working directory. try: os.mkdir(self.testDir) except OSError as detail: self.logger.log( 'Exception during creation of temp directory: %s' % detail, 'ERROR')
def __init__(self): """ Constructor. """ warnings.simplefilter('default') warnings.warn("This module is deprecated in favor of SEKNotifier.", DeprecationWarning) self.config = MSGConfiger() self.logger = SEKLogger(__name__, 'info') self.connector = MSGDBConnector() self.conn = self.connector.connectDB() self.cursor = self.conn.cursor() self.dbUtil = MSGDBUtil() self.noticeTable = 'NotificationHistory' self.notificationHeader = "This is a message from the Hawaii Smart " \ "Energy Project MSG Project notification " \ "system.\n\n" self.noReplyNotice = '\n\nThis email account is not monitored. No ' \ 'replies will originate from this ' \ 'account.\n\nYou are receiving this message ' \ 'because you are on the recipient list for ' \ 'notifications for the Hawaii Smart Energy ' \ 'Project.'
def __init__(self): """ Constructor. """ self.logger = SEKLogger(__name__) self.configer = MSGConfiger() self.fileUtil = MSGFileUtil()
def __init__(self): """ Constructor. """ self.logger = SEKLogger(__name__, 'debug') self.mecoConfig = MSGConfiger() self.currentReadingID = 0 self.dbUtil = MSGDBUtil()
def __init__(self, testing=False): """ Constructor. :param testing: Flag indicating if testing mode is on. """ self.logger = SEKLogger(__name__) self.parser = MECOXMLParser(testing) self.configer = MSGConfiger()
def __init__(self): """ Constructor. """ print __name__ self.logger = SEKLogger(__name__) self.connector = MSGDBConnector() self.dbUtil = MSGDBUtil() self.notifier = MSGNotifier() self.configer = MSGConfiger()
def __init__(self, exitOnError=True, commitOnEveryInsert=False, testing=False): """ Constructor. :param testing: if True, the testing DB will be connected instead of the production DB. """ self.logger = SEKLogger(__name__, 'info') self.configer = MSGConfiger() self.conn = MSGDBConnector().connectDB() self.cursor = self.conn.cursor() self.dbUtil = MSGDBUtil() self.notifier = MSGNotifier() self.mathUtil = MSGMathUtil() self.timeUtil = MSGTimeUtil() self.nextMinuteCrossing = {} self.nextMinuteCrossingWithoutSubkeys = None self.exitOnError = exitOnError self.commitOnEveryInsert = commitOnEveryInsert section = 'Aggregation' tableList = [ 'irradiance', 'agg_irradiance', 'weather', 'agg_weather', 'circuit', 'agg_circuit', 'egauge', 'agg_egauge' ] self.dataParams = { 'weather': ('agg_weather', 'timestamp', ''), 'egauge': ('agg_egauge', 'datetime', 'egauge_id'), 'circuit': ('agg_circuit', 'timestamp', 'circuit'), 'irradiance': ('agg_irradiance', 'timestamp', 'sensor_id') } self.columns = {} # tables[datatype] gives the table name for datatype. self.tables = { t: self.configer.configOptionValue(section, '{}_table'.format(t)) for t in tableList } for t in self.tables.keys(): self.logger.log('t:{}'.format(t), 'DEBUG') try: self.columns[t] = self.dbUtil.columnsString( self.cursor, self.tables[t]) except TypeError as error: self.logger.log( 'Ignoring missing table: Error is {}.'.format(error), 'error')
def __init__(self): """ Constructor. """ self.logger = SEKLogger(__name__, 'DEBUG', useColor=False) self.timeUtil = MSGTimeUtil() self.configer = MSGConfiger() self.fileUtil = MSGFileUtil() self.pythonUtil = MSGPythonUtil() # for debugging self.connector = MSGDBConnector() self.conn = self.connector.connectDB() self.cursor = self.conn.cursor() self.dbUtil = MSGDBUtil() self.notifier = SEKNotifier( connector=self.connector, dbUtil=self.dbUtil, user=self.configer.configOptionValue('Notifications', 'email_username'), password=self.configer.configOptionValue('Notifications', 'email_password'), fromaddr=self.configer.configOptionValue('Notifications', 'email_from_address'), toaddr=self.configer.configOptionValue('Notifications', 'email_recipients'), testing_toaddr=self.configer.configOptionValue( 'Notifications', 'testing_email_recipients'), smtp_server_and_port=self.configer.configOptionValue( 'Notifications', 'smtp_server_and_port')) # Google Drive parameters. self.clientID = self.configer.configOptionValue( 'Export', 'google_api_client_id') self.clientSecret = self.configer.configOptionValue( 'Export', 'google_api_client_secret') self.oauthScope = 'https://www.googleapis.com/auth/drive' self.oauthConsent = 'urn:ietf:wg:oauth:2.0:oob' self.googleAPICredentials = '' self.exportTempWorkPath = self.configer.configOptionValue( 'Export', 'db_export_work_path') self.credentialPath = self.configer.configOptionValue( 'Export', 'google_api_credentials_path') self.credentialStorage = Storage('{}/google_api_credentials'.format( self.credentialPath)) self._driveService = None self._cloudFiles = None self.postAgent = 'Maui Smart Grid 1.0.0 DB Exporter' self.retryDelay = 10 self.availableFilesURL = ''
def __init__(self): self.configer = MSGConfiger() self.pool = None self.fileList = [] self.dateList = [] self.weatherUtil = MSGWeatherDataUtil() global weatherDataPath weatherDataPath = self.configer.configOptionValue( WEATHER_DATA, 'weather_data_path') global weatherDataURL weatherDataURL = self.configer.configOptionValue( WEATHER_DATA, 'weather_data_url') global weatherDataPattern weatherDataPattern = self.configer.configOptionValue( WEATHER_DATA, 'weather_data_pattern')
def __init__(self): """ Constructor. A database connection is not maintained here to keep this class lightweight. """ self.logger = SEKLogger(__name__, DEBUG) self.configer = MSGConfiger() self.url = self.configer.configOptionValue('Weather Data', 'weather_data_url') self.pattern = self.configer.configOptionValue('Weather Data', 'weather_data_pattern') self.fileList = [] self.dateList = [] # List of dates corresponding weather data files. self.fillFileListAndDateList() self.dbUtil = MSGDBUtil()
def setUp(self): self.i = MECODBInserter() # Connect to the testing database. self.connector = MSGDBConnector(testing=True) self.conn = self.connector.connectDB() self.lastSeqVal = None # Does this work having the dictCur be in another class? self.dictCur = self.connector.dictCur self.cursor = self.conn.cursor() self.deleter = MECODBDeleter() self.tableName = 'MeterData' self.columnName = 'meter_data_id' self.configer = MSGConfiger() self.logger = SEKLogger(__name__, 'debug') self.dbUtil = MSGDBUtil()
def __init__(self, testing=False, logLevel='silent'): """ Constructor. :param testing: Boolean indicating if Testing Mode is on. When testing mode is on, a connection to the testing database will be made instead of the production database. This is useful for unit testing. :param logLevel """ self.logger = SEKLogger(__name__, logLevel) if testing: self.logger.log("Testing Mode is ON.") self.configer = MSGConfiger() self.dbPassword = self.configer.configOptionValue( "Database", 'db_password') self.dbHost = self.configer.configOptionValue("Database", 'db_host') self.dbPort = self.configer.configOptionValue("Database", 'db_port') if testing: self.dbName = self.configer.configOptionValue( "Database", 'testing_db_name') else: self.dbName = self.configer.configOptionValue( "Database", 'db_name') self.logger.log("Instantiating DB connector with database {}.".format( self.dbName)) self.dbUsername = self.configer.configOptionValue( "Database", 'db_username') self.conn = self.connectDB() if not self.conn: raise Exception('DB connection not available.') try: self.dictCur = self.conn.cursor( cursor_factory=psycopg2.extras.DictCursor) except AttributeError as error: self.logger.log('Error while getting DictCursor: {}'.format(error))
def setUp(self): self.m = MECOMapper() self.testColumns = self.m.dbColsMeterData.values() self.testLabels = self.m.dbColsMeterData.keys() self.testTableName = 'MeterData' self.config = MSGConfiger()
def setUp(self): self.weatherUtil = MSGWeatherDataUtil() self.logger = SEKLogger(__name__, 'DEBUG') self.dbConnector = MSGDBConnector() self.cursor = self.dbConnector.conn.cursor() self.configer = MSGConfiger()
def setUp(self): self.logger = SEKLogger(__name__) self.notifier = MSGNotifier() self.configer = MSGConfiger()
if __name__ == '__main__': processCommandLineArguments() filename = COMMAND_LINE_ARGS.filename success = True anyFailure = False connector = MSGDBConnector(testing = COMMAND_LINE_ARGS.testing) conn = connector.connectDB() cur = conn.cursor() dbUtil = MSGDBUtil() notifier = MSGNotifier() msgBody = '' configer = MSGConfiger() if COMMAND_LINE_ARGS.testing: dbName = configer.configOptionValue("Database", "testing_db_name") else: dbName = configer.configOptionValue("Database", "db_name") msg = ("Loading Meter Location History data in file %s to database %s.\n" % ( filename, dbName)) sys.stderr.write(msg) msgBody += msg f = open(filename, "r") reader = csv.reader(f) data = []
def test_init(self): # @todo Improve this test by using direct introspection instead of # this roundabout method. localConfiger = MSGConfiger() self.assertIsInstance(self.configer, type(localConfiger))
def setUp(self): self.configer = MSGConfiger()
def setUp(self): self.connector = MSGDBConnector(True) self.conn = self.connector.connectDB() self.configer = MSGConfiger()
def __init__(self, testing = False): """ Constructor. :param testing: (optional) Boolean indicating if Testing Mode is on. """ self.logger = SEKLogger(__name__, 'silent') if (testing): self.logger.log("Testing Mode is ON.", 'info') self.debug = False self.configer = MSGConfiger() if self.configer.configOptionValue("Debugging", 'debug') == True: self.debug = True self.util = MSGDBUtil() self.mapper = MECOMapper() self.connector = MSGDBConnector(testing) self.conn = self.connector.connectDB() self.filename = None self.fileObject = None self.processForInsertElementCount = 0 self.inserter = MECODBInserter() self.insertDataIntoDatabase = False # Count number of times sections in source data are encountered. self.tableNameCount = {'SSNExportDocument': 0, 'MeterData': 0, 'RegisterData': 0, 'RegisterRead': 0, 'Tier': 0, 'Register': 0, 'IntervalReadData': 0, 'Interval': 0, 'Reading': 0, 'IntervalStatus': 0, 'ChannelStatus': 0, 'EventData': 0, 'Event': 0} # Use this dictionary to track which channels were processed when # readings are being processed. this is to prevent duplicate channel # data from being inserted. self.channelProcessed = {} self.initChannelProcessed() # Tables to be inserted to. self.insertTables = self.configer.insertTables self.lastSeqVal = None self.fKeyVal = None self.lastTable = None self.fkDeterminer = MECOFKDeterminer() self.dupeChecker = MECODupeChecker() self.currentMeterName = None self.currentIntervalEndTime = None self.currentRegisterReadReadTime = None self.dupesExist = False self.channelDupeExists = False # For Reading dupes. self.numberDupeExists = False # For Register dupes. self.eventTimeDupeExists = False # For Event dupes. self.commitCount = 0 self.readingDupeOnInsertCount = 0 # For Reading dupes. self.registerDupeOnInsertCount = 0 # For Register dupes. self.eventDupeOnInsertCount = 0 # For Event dupes. self.dataProcessCount = 0 self.readingDupeCheckCount = 0 # For Reading dupes. self.registerDupeCheckCount = 0 # For Register dupes. self.eventDupeCheckCount = 0 # For Event dupes. self.insertCount = 0 self.cumulativeInsertCount = 0 self.nonProcessForInsertElementCount = 0 self.readingInsertCount = 0 self.registerInsertCount = 0 self.eventInsertCount = 0 self.totalReadingInsertCount = 0 self.totalRegisterInsertCount = 0 self.totalEventInsertCount = 0 self.totalReadingDupeOnInsertCount = 0 self.totalRegisterDupeOnInsertCount = 0 self.totalEventDupeOnInsertCount = 0