class DeviceManager(): INSTANCE = None # Singleton pattern def __new__(cls, *args, **kwargs): if not cls.INSTANCE: cls.INSTANCE = super(DeviceManager, cls).__new__( cls, *args, **kwargs) return cls.INSTANCE def __init__(self): # Instantiate Logger self.LOGGER = Debug.getLogger("energyathome.datalogger.serialcomm") self.COMM = serial.Serial() # Create Config manager self.CONFIG = ConfigManager() # Open serial connection def open(self): '''Open connection to USB to serial port''' self.COMM.port = self.CONFIG.getConfig("SerialConnection", "serialport") self.COMM.baudrate = self.CONFIG.getIntConfig("SerialConnection", "baudrate") self.COMM.parity = self.CONFIG.getConfig("SerialConnection", "parity") self.COMM.timeout = None self.COMM.bytesize = self.CONFIG.getIntConfig("SerialConnection", "bytesize") if(self.COMM.isOpen() == False): self.LOGGER.info('Establishing connection to device') self.COMM.open() else: self.LOGGER.info('Connection already opened') self.LOGGER.info('Device connected:' + str(self.COMM.isOpen())) # Close serial connection def close(self): '''Close connection to USB to Serial port''' if(self.COMM.isOpen): self.LOGGER.info('Closing device connection') self.COMM.close() self.LOGGER.info('Device connection closed') # Get data from serial connection def read(self): '''Read data from USB to Serial device. Connection must be established.''' data = self.COMM.readline() if(len(data) > 0): self.LOGGER.debug('serial data: ' + data) return data
def __init__(self): '''Check all global variables have been initialized''' # Configuration Manager self.CONFIG = ConfigManager() # Instantiate logging logging.config.fileConfig(self.CONFIG.getConfigFilePath()) self.LOGGER = logging.getLogger("energyathome.datalogger.database.mysql") # Connection variable self.CONNECTION = None # Number of transaction retries before failing self.RETRIES = self.CONFIG.getIntConfig("Database", "retries") # Number in seconds to wait before re-attempting failed transaction self.WAIT = self.CONFIG.getFloatConfig("Database", "wait") sys.path.append("..") if self.CONFIG is None: print "Unable to get configuration manager. Exit" sys.exit(1) if self.RETRIES is None: print "No DB retry value set. Using default value" self.RETRIES = 3 if self.WAIT is None: print "No DB wait value set. Using default value" self.WAIT = 60
def __init__(self): # Instantiate Logger self.LOGGER = Debug.getLogger("energyathome.datalogger.serialcomm") self.COMM = serial.Serial() # Create Config manager self.CONFIG = ConfigManager()
def __init__(self): # Instantiate Logger self.LOGGER = Debug.getLogger("energyathome.datalogger.offlinehandler") # Configuration Manager self.CONFIG = ConfigManager() # Data validation class self.VALIDATOR = CheckLiveData() # Data trigger class self.TRIGGER = CheckLiveTriggers()
def __init__(self): # Instantiate logger self.LOGGER = Debug.getLogger("energyathome.datalogger.twitter") # Configuration Manager self.CONFIG = ConfigManager() self.LOGGER.debug("Adding to system path: " + os.path.dirname(os.path.abspath(__file__)) + os.sep + "python-twitter") sys.path.append(os.path.dirname(os.path.abspath(__file__)) + os.sep + "python-twitter") currentTime = datetime.now() self.LAST_HOURLY_POST = currentTime.strftime("%H") self.TWITTER = __import__("twitter")
class MySQL(object): INSTANCE = None def __new__(cls, *args, **kwargs): if not cls.INSTANCE: cls.INSTANCE = super(MySQL, cls).__new__( cls, *args, **kwargs) return cls.INSTANCE def __init__(self): '''Check all global variables have been initialized''' # Configuration Manager self.CONFIG = ConfigManager() # Instantiate logging logging.config.fileConfig(self.CONFIG.getConfigFilePath()) self.LOGGER = logging.getLogger("energyathome.datalogger.database.mysql") # Connection variable self.CONNECTION = None # Number of transaction retries before failing self.RETRIES = self.CONFIG.getIntConfig("Database", "retries") # Number in seconds to wait before re-attempting failed transaction self.WAIT = self.CONFIG.getFloatConfig("Database", "wait") sys.path.append("..") if self.CONFIG is None: print "Unable to get configuration manager. Exit" sys.exit(1) if self.RETRIES is None: print "No DB retry value set. Using default value" self.RETRIES = 3 if self.WAIT is None: print "No DB wait value set. Using default value" self.WAIT = 60 # Connect to MySQL using settings from DatabaseConfig.py def connect(self): '''Connect to database''' attempts = 1 databaseSettings = self.CONFIG.getConfigCategory("Database") if self.LOGGER is None: __init__() while attempts <= self.CONFIG.getIntConfig("Database", "retries") and self.CONNECTION is None: try: self.LOGGER.info("Attempting DB connection") self.CONNECTION = MySQLdb.connect( host=databaseSettings["url"], user=databaseSettings["username"], passwd=databaseSettings["password"], db=databaseSettings["database"]) # Turn ping on to automatically connect to DB when idle and disconnects self.CONNECTION.ping(True) self.LOGGER.info("DB connection successful") except(MySQLdb.Error) as e: # Failed to connect self.LOGGER.error("Unable to connect to database:'" + str(databaseSettings["url"]) + "|" + str(e.args[0])) self.LOGGER.error("Attempt " + str(attempts) + " failed. Retrying in " + str(databaseSettings["wait"]) + " secs") # Wait for n seconds before attempting to reconnect time.sleep(self.WAIT) # Continue in loop pass # Increment number of times it tried to connect attempts += 1 # If CONNECTION is None then it was unable to connect if self.CONNECTION is None: raise ConnectionException("Database connection error") # Close DB CONNECTION def disconnect(self): '''Close CONNECTION to database''' try: # Only close CONNECTION if CONNECTION exists if self.CONNECTION is not None: self.LOGGER.info("Attempting to close DB connection") self.CONNECTION.close() self.CONNECTION = None self.LOGGER.info("Close DB connection successful") return True except MySQLdb.Error as e: self.LOGGER.error("Unable to disconnect from DB:") self.LOGGER.error(e.args[0], e.args[1]) return False # Execute query with no return results def executeNonUpdate(self, statement, values): '''Execute SQL with no return results. Accepts an SQL statement with %s parameters which will be replaced with the values parameter. If no %s is used, values should be set to None. Returns last inserted row ID reguardless of SQL type.''' id = None attempts = 1 while attempts <= self.RETRIES: try: cursor = self.CONNECTION.cursor() # Check if values need replacing in SQL statement if values is None: cursor.execute(statement) else: cursor.execute(statement, values) self.CONNECTION.commit() id = cursor.lastrowid cursor.close() # Exit retry loop and return value break except MySQLdb.Error as e: print "Caught error:" print e.args[0], e.args[1] print "Retrying in " + str(self.WAIT) + " secs" # Increment number of times it has tried to execute this function attempts += 1 time.sleep(self.WAIT) try: # Try disconnecting first to avoid holding a CONNECTION open self.disconnect() # Try reconnecting to DB self.connect() except MySQLdb.Error as e: # Ignore connect error and gracefully exit function print "DB reconnect failed:" print e.args[0], e.args[1] raise pass except AttributeError as ae: # Caught when an object such as the connection has not been instantiated. # This may happen if connection is lost whils trying to get a cursor raise ConnectionException("Attribute error in DB") # return id from insert return id # Execute query and return results def executeUpdate(self, statement, values): '''Execute query and return all results.''' attempts = 1 while attempts <= self.RETRIES: try: cursor = self.CONNECTION.cursor() # Check if values need replacing in SQL statement if values is None: cursor.execute(statement) else: cursor.execute(statement, values) results = cursor.fetchall() cursor.close() return results except MySQLdb.Error as e: print "Caught error:" print e.args[0], e.args[1] print "Retrying in " + str(self.WAIT) + " secs" # Increment number of times it has tried to execute this function attempts += 1 time.sleep(self.WAIT) try: # Try disconnecting first to avoid holding a CONNECTION open self.disconnect() # Try reconnecting to DB self.connect() except MySQLdb.Error as e: # Ignore connect error and gracefully exit function print "DB reconnect failed:" print e.args[0], e.args[1] raise except ConnectionException as ce: raise ce pass except AttributeError as ae: # Caught when an object such as the connection has not been instantiated. # This may happen if connection is lost whils trying to get a cursor raise ConnectionException("Attribute error in DB") # Fail after retry return None # Execute query and return one result def executeOneUpdate(self, statement, values): '''Execute SQL and returns one result. Accepts an SQL statement with %s parameters which will be replaced with the values parameter. If no %s is used, values should be set to None. Returns the first result only.''' attempts = 1 while attempts <= self.RETRIES: try: cursor = self.CONNECTION.cursor() # Check if values need replacing in SQL statement if values is None: cursor.execute(statement) else: cursor.execute(statement, values) result = cursor.fetchone() cursor.close() return result except MySQLdb.Error as e: print "Caught error:" print e.args[0], e.args[1] print "Retrying in " + str(self.WAIT) + " secs" # Increment number of times it has tried to execute this function attempts += 1 time.sleep(self.WAIT) try: # Try disconnecting first to avoid holding a CONNECTION open self.disconnect() # Try reconnecting to DB self.connect() except ConnectionException as ce: # Ignore connect error and gracefully exit function print "DB reconnect failed:" print e.args[0], e.args[1] raise ce except AttributeError as ae: # Caught when an object such as the connection has not been instantiated. # This may happen if connection is lost whils trying to get a cursor raise ConnectionException("Attribute error in DB") # Fail after retry return None
class Twitter: def __init__(self): # Instantiate logger self.LOGGER = Debug.getLogger("energyathome.datalogger.twitter") # Configuration Manager self.CONFIG = ConfigManager() self.LOGGER.debug("Adding to system path: " + os.path.dirname(os.path.abspath(__file__)) + os.sep + "python-twitter") sys.path.append(os.path.dirname(os.path.abspath(__file__)) + os.sep + "python-twitter") currentTime = datetime.now() self.LAST_HOURLY_POST = currentTime.strftime("%H") self.TWITTER = __import__("twitter") def postHourlySummary(self): '''Posts an hourly summary to Twitter''' # Get current system time currentTime = datetime.now() # create time difference to be used to work out time period timeDiff = timedelta(hours=1) # Get current hour currentHour = currentTime.strftime("%H") # If current hour does not match last post hour then it's been a new hour since last post if(currentHour != self.LAST_HOURLY_POST): Debug.writeOut("Hourly condtion met (" + currentHour + " != " + self.LAST_HOURLY_POST + "). Posting to Twitter") # Create SQL to get data for tweet sql = "SELECT COALESCE(ROUND(AVG(energy), 2), 0), " +\ "COALESCE(MAX(energy), 0), COALESCE(ROUND(AVG(temperature), 1), 0) " +\ "FROM historical_data WHERE date_time >= ADDDATE(NOW(), INTERVAL -1 HOUR)" self.LOGGER.debug(sql) # Get statistics from DB stats = MySQL.executeOneUpdate(sql, None) # Create tweet message = (currentTime - timeDiff).strftime("%H:%M") + "-" + currentTime.strftime("%H:%M") +\ " Summary: " + str(stats[0]) + "w was used." +\ " Energy usage peaked at " + str(stats[1]) + "w" +\ ". The average temperature was " + str(stats[2]) + "c." # Check if tweet should be a Direct Message or just a tweet if self.CONFIG.getBooleanConfig("Twitter", "directMessagePost"): self.postDirectMessage(self.CONFIG.getBooleanConfig("Twitter", "directMessageUser"), message) else: # Post message to twitter self.tweet(message) # Set last hourly post to current hour self.LAST_HOURLY_POST = currentHour def postDailySummary(self): '''Posts a daily summary to Twitter''' # Get current minutes from system time currentTime = datetime.now() # create time difference to be used to work out time period timeDiff = timedelta(days=1) # Get current minutes currentHour = currentTime.strftime("%H") # Check if the hours of the time is 00 which means midnight and the current day # has changed if(currentHour == "00" and (self.LAST_DAY_POST == "" or currentTime.strftime("%d") != self.LAST_DAY_POST)): Debug.writeOut("Daily condition met (hour of day:" + currentHour + " == 00 && day:" + currentTime.strftime("%d") + " == " + self.LAST_DAY_POST + "). Posting to Twitter") # Create SQL to get data for tweet sql = " SELECT COALESCE(ROUND(AVG(energy), 2), 0), COALESCE(MAX(energy), 0), COALESCE(ROUND(AVG(temperature), 1), 0) FROM historical_data WHERE date_time >= ADDDATE(NOW(), INTERVAL -1 DAY)" self.LOGGER.debug(sql) # Get statistics from DB stats = MySQL.executeOneUpdate(sql, None) # Create tweet message = (currentTime - timeDiff).strftime("%d-%b-%Y") + " Summary: " + str(stats[0]) +\ "w was used. " +\ "Energy usage peaked at " + str(stats[1]) + "w. " +\ "The average temperature was " + str(stats[2]) + "c." # Save new day of tweet self.LAST_DAY_POST = currentTime.strftime("%d") # Check if tweet should be a Direct Message or just a tweet if self.CONFIG.getBooleanConfig("Twitter", "directMessagePost"): self.postDirectMessage(self.CONFIG.getBooleanConfig("Twitter", "directMessageUser"), message) else: # Post message to twitter self.tweet(message) def tweet(self, message): '''Tweet a message''' # Connect to Twitter twit = self.TWITTER.Api(username=self.CONFIG.getBooleanConfig("Twitter", "username"), password=self.CONFIG.getBooleanConfig("Twitter", "password")) return twit.PostUpdate(message) def postDirectMessage(self, user, message): '''Direct Message a user''' twit = self.TWITTER.Api(username=self.CONFIG.getBooleanConfig("Twitter", "username", password=self.CONFIG.getBooleanConfig("Twitter", "password"))) return twit.PostDirectMessage(user, message) def getTweets(self): '''Get tweet @ replies from account''' twit = self.TWITTER.Api(username=self.CONFIG.getBooleanConfig("Twitter", "username", password=self.CONFIG.getBooleanConfig("Twitter", "password"))) return twit.GetReplies() def getDirectMessages(self): '''Get tweet @ replies from account''' twit = self.TWITTER.Api(username=self.CONFIG.getBooleanConfig("Twitter", "username", password=self.CONFIG.getBooleanConfig("Twitter", "password"))) return twit.GetDirectMessages()
def __init__(self): # Instantiate config manager self.CONFIG = ConfigManager() # Get logger instance self.LOGGER = Debug.getLogger("energyathome.datalogger.datatrigger")
class CheckLiveTriggers: def __init__(self): # Instantiate config manager self.CONFIG = ConfigManager() # Get logger instance self.LOGGER = Debug.getLogger("energyathome.datalogger.datatrigger") def checkTriggers(self, historicalData): '''Check if data received meets any one trigger conditions. Returns true if it's met''' trigger = False # Get last recorded data point. Used for trigger information try: previousDataPoint = HistoricalData.getLastHistoricalData(historicalData) except ConnectionException as ce: raise ce # Check if there was data in DB if previousDataPoint is not None: try: # Get all channel data channel = "" for key, value in previousDataPoint.energy.iteritems(): channel += key + "=" + str(value) + "w " self.LOGGER.info("Last data point for " + previousDataPoint.name +\ " app_id=" + str(previousDataPoint.applianceId) + " type=" +\ str(previousDataPoint.sensorType) + " " + "at " +\ str(previousDataPoint.time) + " was " + channel +\ str(previousDataPoint.temperature) + "c") # Check timeout timeout = self.checkTimeTrigger(previousDataPoint) if timeout is True: trigger = True # Check energy variation energy = self.checkEnergyTrigger(historicalData, previousDataPoint) if energy is True: trigger = True # Check temperature variation temp = self.checkTemperatureTrigger(historicalData, previousDataPoint) if temp is True: trigger = True except AttributeError as ae: self.LOGGER.error("Caught Error:" + str(ae)) trigger = False else: # No previous data point found self.LOGGER.info("No data history for device " + historicalData.name +\ " on app_id=" + str(historicalData.applianceId) +\ " type=" + str(historicalData.sensorType)) trigger = True # Historial data existed but no conditions were met. return trigger def checkTimeTrigger(self, previousDataPoint): '''Returns true if time from last datapoint exceeded the maximum''' # Check timeout trigger condition first as it's most common condition to # trigger a save point. Ignoring device time so using system time if (datetime.today() - previousDataPoint.time) >= timedelta(seconds = self.CONFIG.getIntConfig("Trigger", "timeout")): self.LOGGER.info("Timeout trigger condition met with " +\ str(datetime.today() - previousDataPoint.time) + " delta") return True else: return False def checkEnergyTrigger(self, historicalData, previousDataPoint): '''Returns true if the energy difference from the previous value is exceeded''' # Check energy variation. Get absolute value reguardless of positive / negative value # Must loop through each channel for key, value in historicalData.energy.iteritems(): if previousDataPoint.energy.get(key, None) is not None: # Calculate the difference from last data point and the new one energyDiff = math.fabs(value) - previousDataPoint.energy[key] if energyDiff >= self.CONFIG.getFloatConfig("Trigger", "energyvariation"): self.LOGGER.info("Energy trigger condition met with " + str(key) + " " +\ str(energyDiff) + "w delta") return True else: # energy value did not exist in previous reading return True # Deliberate fall through to return false return False def checkTemperatureTrigger(self, historicalData, previousDataPoint): '''Returns true if the temperature difference from the previous value is exceeded''' # Check temperature variation. Get absolute value reguardless of positive / negative value tempDiff = math.fabs(historicalData.temperature - previousDataPoint.temperature) if tempDiff > self.CONFIG.getFloatConfig("Trigger", "temperatureVariation"): self.LOGGER.info("Temperature trigger condition met with " +\ str(tempDiff) + "c delta") return True else: return False
class BackupRestore: def __init__(self): # Instantiate Logger self.LOGGER = Debug.getLogger("energyathome.datalogger.offlinehandler") # Configuration Manager self.CONFIG = ConfigManager() # Data validation class self.VALIDATOR = CheckLiveData() # Data trigger class self.TRIGGER = CheckLiveTriggers() def backup(self, historicalData): '''Writes historical data to file''' #If true exit due to exception exit = False # Get file path from config location = self.CONFIG.getConfig("Application", "offlineFile") #Check if config is empty if len(location) == 0: # Use current directory location = os.path.join(os.getcwd(), "backup.p") self.LOGGER.warning("offlineFile is empty. Using default: '" + location + "'") try: # Append to file path = file(location, "a") # Ensure object has data if historicalData is not None: self.LOGGER.info("Writing data to file:" + str(historicalData.__dict__)) pickle.dump(historicalData, path) except IOError: #Debug.writeOut("Unable to write to backup file: '" + location + "'") # No point running program if it's unable to write to file exit = True finally: # Close file try: path.close() except UnboundLocalError: # File was not opened pass # Check to exit if exit: Core.shutdown() def restore(self): '''Starts the restore process from a file.''' backup = self.restoreFromFile() recordCount = len(backup) if(recordCount > 0): self.LOGGER.info(str(recordCount) + " record(s) found. Saving to DB") for record in backup: # Set data as valid by default hDataValid = True # Get backup record hData = record if self.VALIDATOR is not None and self.CONFIG.getBooleanConfig("Tolerence", "enabled"): try: validatedData = self.VALIDATOR.validateData(hData) except Exception as e: raise e hDataValid = validatedData[0] if hDataValid is True: hData = validatedData[1] if hDataValid and self.CONFIG.getBooleanConfig("Trigger", "enabled"): # Check trigger conditions which return true or false if it's valid try: hDataValid = self.TRIGGER.checkTriggers(hData) except Exception as e: raise e # Insert the first element in list and remove it if hDataValid: self.LOGGER.info("Inserting: " + str(record.__dict__)) #HistoricalData.insertData(record) HistoricalData.insertData(hData) else: self.LOGGER.info("Skipped backup record") # Remove backup file to prevent duplicate data from being restored. self.LOGGER.info("Restore from backup complete.") self.LOGGER.info("Removing backup file.") self.LOGGER.info("File deleted? " + str(self.deleteBackupFile())) def restoreFromFile(self): '''Reads a file and returns an array of objects.''' # List to store restored objects list = [] # Get file path from config location = self.CONFIG.getConfig("Application", "offlineFile") #Check if config is empty if len(location) == 0: # Use current directory location = os.path.join(os.getcwd(), "backup.p") self.LOGGER.info("offlineFile is empty. Using default: '" + location + "'") # Check file exists before deleting if os.path.exists(location): # Create file object path = file(location, "r") # Read file till end of file try: while True: list.append(pickle.load(path)) except EOFError: # Ignore end of file error pass finally: # Close file path.close() self.LOGGER.info("Found " + str(len(list)) + " record(s) in '" + location + "'") return list def deleteBackupFile(self): '''Removed backup file if it exists.''' success = False; # Get file path from config location = self.CONFIG.getConfig("Application", "offlineFile") #Check if config is empty if len(location) == 0: # Use current directory location = os.path.join(os.getcwd(), "backup.p") self.LOGGER.info("offlineFile is empty. Using default: '" + location + "'") # Check file exists before deleting if os.path.exists(location): # Delete file try: os.remove(location) success = True; except OSError: self.LOGGER.error("Unable to remove file: '" + location + "'") return success;
class CheckLiveData: def __init__(self): # Instantiate config manager self.CONFIG = ConfigManager() # Get logger instance self.LOGGER = Debug.getLogger("energyathome.datalogger.datavalidation") # Validate data captured from device def validateData(self, historicalData): '''Checks historical data fall within parameter which is customised in the config file. Returns results in a tuple size of 2. [0] = True or False depending if validation was successful or not [1] = HistoricalData.HistoricalData class of sanitised data''' valid = True # Get maximum appliance Id value maxAppId = self.CONFIG.getIntConfig("Tolerence", "maxApplianceId") try: # If max app id is greater than 0 then checking for app id is enabled if maxAppId is not None and maxAppId >= 0: if self.checkApplianceId(historicalData) is False: valid = False # Check device name matches in the config deviceNames = self.CONFIG.getConfig("Tolerence", "allowedDeviceNames") # If device names are defined then perform a match if deviceNames is not None and deviceNames != "": if self.checkDeviceName(historicalData) is False: valid = False if self.CONFIG.getBooleanConfig("Tolerence", "allowNewAppliances") is False: # If it returns true then it's a new appliance and should be ignored if self.checkNewAppliance(historicalData) is True: valid = False self.LOGGER.info("Appliance ID " + str(historicalData.applianceId) + " was not stored due to allowNewAppliances = False") # Check channel names if self.CONFIG.getBooleanConfig("Tolerence", "allowBlankChannelNames") is False: test = self.checkChannelNames(historicalData) # If test returned None then it failed validation. # Otherwise assign returned Historical Data because it may have changed some attributes if test is None: valid = False else: historicalData = test # Check channels if self.CONFIG.getBooleanConfig("Tolerence", "checkChannels") is True and historicalData.applianceId is not None: test = self.checkChannels(historicalData) # If test returned None then it failed validation. # Otherwise assign returned Historical Data because it may have changed some attributes if test is None: valid = False else: historicalData = test except ConnectionException as ce: raise ce return (valid, historicalData) def checkApplianceId(self, historicalData): '''Checks if appliance ID falls within a set range''' # Get maximum appliance Id value maxAppId = self.CONFIG.getIntConfig("Tolerence", "maxApplianceId") try: # Check if appliance number exceeds maximum if int(historicalData.applianceId) > maxAppId: self.LOGGER.info("Appliance ID " + str(historicalData.applianceId) + " > " + str(maxAppId)) return False else: return True except ValueError, ve: self.LOGGER.info("Check max App ID failed: " + str(historicalData.applianceId) + " > " + str(maxAppId)) return False except AttributeError as ae: self.LOGGER.info("Check max App ID failed: No attribute Error:" + str(ae)) return False