class Persistence(object): def __init__(self, logger=None): if logger == None: self._logger = Logger("Persistence", Logger.DETAIL) else: self._logger = logger self._activity_type = "Environment_Observation" self._couch = CouchUtil(self._logger) #self._sheet = AppendUtil(self._logger) self._test = False self._logger.detail("Initialized Persistence") def save(self, doc, test=False): ''' Args: doc: list of attributes, should be of format: [activity_name, trial, plot, subject, attribute, value, units, participant, status_qualifier, comment] participant may be a device string, or a list: ['person':'hmw'] Returns: None Throws: None ''' self._logger.detail("In saveRecord") # Add a line for each persistence service to be used # be sure to copy the record so formatting from one # does not impact another system self._couch.save(doc.copy())
class AppendUtil(object): def __init__(self, logger=None): """Record optional sensor data Args: lvl: Logging level Returns: None Raises: None """ if logger == None: self._logger = Logger("AppendUtil", Logger.DETAIL) else: self._logger = logger self._sheet_name = '1Mtlr_-yqwdEwXFEcaLoEP01aDN7vvOosDOEerFpDkaI' self._scope = ['https://www.googleapis.com/auth/spreadsheets'] self._l_util = ListUtil(self._logger) self._s_util = SheetUtil(self._sheet_name, self._scope, self._logger) self._logger.debug("Initialized AppendUtil") def append(self, rec, append_range): #print(rec) self._s_util.append(append_range, rec) def save(self, doc): ''' This is the entry function into the object Lookup the range in the dictionary before appending Save list to Google Sheet Args: self doc - formatted list to save (append) Returns: None Throws: None ''' self._logger.detail("In save") name = doc[0] self._logger.debug('%s: %s' % ('Rec Type', name)) rec = self._l_util.build(doc) # dictionary of sheet tabs range = { 'Environment_Observation': 'Environment!A1', 'State_Change': 'State!A1', 'Agronomic_Activity': 'Agronomic!A1', 'Phenotype_Observation': 'Phenotype!A1' } # Lookup range for record type - note this is a lookup on a list of lists append_range = range[name] self._logger.debug('%s: %s' % ('Range', append_range)) self._s_util.append(append_range, rec)
class SCD30: CMD_START_PERIODIC_MEASUREMENT = 0x0010 CMD_STOP_PERIODIC_MEASUREMENT = 0x0104 CMD_READ_MEASUREMENT = 0x0300 CMD_GET_DATA_READY = 0x0202 CMD_SET_MEASUREMENT_INTERVAL = 0x4600 CMD_SET_TEMPERATURE_OFFSET = 0x5403 CMD_SET_ALTITUDE = 0x5102 WORD_LEN = 2 COMMAND_LEN = 2 MAX_BUFFER_WORDS = 24 STATUS_FAIL = 1 STATUS_OK = 0 CO2_DATA_INDEX = 0 TEMP_DATA_INDEX = 1 HUMIDITY_DATA_INDEX = 2 def __init__(self, logger=None): """Create sensor object Args: None Returns: None Raises: None """ self._addr = addr self._path = path self._logger = logger if logger == None: self._logger = Logger("SCD30", Logger.INFO) self._i2c = I2C(path, addr, self._logger) self._logger.debug("initialize SCD30") self.start_periodic_measurement() def start_periodic_measurement(self): """Start sensor to generate data (about every 2 seconds Args: self: test: flag for test logic Returns: None Raises: None """ # command 0x0010, altitude 0x03eb, crc_check 0x87 # altitude set to 1003 mb #msgs = [0x00,0x10, 0x03, 0xeb, 0x87] # Altitude is currently set to 0 self._logger.debug("In Start Periodic Measurment") msgs = [0x00,0x10, 0x00, 0x00, 0x81] self._i2c.msg_write(msgs) # TESTE ADN WORKS! def stop_periodic_measurement(self): """Stop automatic data gathering Args: None Returns: None Raises: None """ self._logger.debug("In Stop Periodic Measurment") msgs = [0x01,0x14] self._i2c.msg_write(msgs) # return self._i2c.msg_write(self.CMD_STOP_PERIODIC_MEASUREMENT pass def read_measurement(self): """Read data Args: self: test: Returns: CO2 Temp Relative Humidity Raises: None """ self._logger.debug("In Read Measurment") self._i2c.msg_write([0x03, 0x00]) msgs = self._i2c.msg_read(18) data3 = self.bytes_to_value(msgs[0].data) return data3[self.CO2_DATA_INDEX], data3[self.TEMP_DATA_INDEX], data3[self.HUMIDITY_DATA_INDEX] def bytes_to_value(self, byte_array): """Convert array of byte values into three float values Args: self byte_array: array of data from I2C sensor test Returns: data: array of float values (CO2, Temp, RH) Raises: None """ self._logger.debug("In Bytes To Value") # Array for value bytes (exclude crc check byte) bytes_buf = [0]*12 # 2 words for each co2, temperature, humidity l = len(bytes_buf) ld = len(byte_array) # array for word conversion - two words per value word_buf = [0]*int(l/2) # final data structure - one place per value data = [0]*int(len(word_buf)/2) # Load bytes_buffer, strip crc bytes y = 0 for x in range(0, ld, 6): # print("x: " + str(x) + " y: " + str(y)) bytes_buf[y] = byte_array[x] bytes_buf[y+1] = byte_array[x+1] bytes_buf[y+2] = byte_array[x+3] bytes_buf[y+3] = byte_array[x+4] y += 4 self._logger.detail("bytes_buf: " + str(bytes_buf)) # Convert sensor data reads to physical value per Sensirion specification # Load buffer with values # Cast 4 bytes to one unsigned 32 bit integer # Cast unsigned 32 bit integer to 32 bit float # Convert bytes to words for i in range(len(word_buf)): word_buf[i] = (bytes_buf[i*2] << 8) | bytes_buf[i*2+1] self._logger.detail(str(hex(bytes_buf[i])) + " " + str(hex(bytes_buf[i+1]))) self._logger.detail(hex(word_buf[i])) #convert words to int32 data[self.CO2_DATA_INDEX] = (word_buf[0] << 16) | word_buf[1] data[self.TEMP_DATA_INDEX] = (word_buf[2] << 16) | word_buf[3] data[self.HUMIDITY_DATA_INDEX] = (word_buf[4] << 16) | word_buf[5] #Convert int32 data to float32 floatData = numpy.array(data, dtype=numpy.int32) data = floatData.view('float32') self._logger.detail("CO2: " + str(data[self.CO2_DATA_INDEX])) self._logger.detail("Temp: " + str(data[self.TEMP_DATA_INDEX])) self._logger.detail("RH: " + str(data[self.HUMIDITY_DATA_INDEX])) return data # interval : (u16 integer) def set_measurement_interval(self, interval_sec): """Set frequency of automatic data collectoin Args: self interval_sec: time in seconds test Returns: None Raises: None """ self._logger.debug("In Set Measurment Interval") if interval_sec < 2 or interval_sec > 1800: return self.STATUS_FAIL # Need to finish this so value is 32 word split to two 16 words # Calculate crc value for last word # [Cmd MSB, Cmd LSB, Interval MSB, Interval LSB, CRC] # msb, lsb = convert_word(interval_sec, test) # crc = calc_crc(msb, lsb, test) # msg = [0x46,0x10, msb, lsb, crc] msgs = [0x46,0x10, 0x00, 0x02, 0xE3] self._i2c.msg_write(msgs) def get_data_ready(self): """Check if have fresh data from periodic update Args: self test Returns: ready flag Raises: None """ self._logger.debug("In Get Data Ready") self._i2c.msg_write([0x02, 0x02]) msgs = self._i2c.msg_read(3) #for msg in msgs: # self._logger.detail("Msg: " + str(msg)) # for d in msg.data: # self._logger.detail("Data: " + str(hex(d))) #self._logger.detail("Data: " + str(msgs[0].data[1])) return msgs[0].data[1] # Strange behaviour def set_temperature_offset(self, temperature_offset): """Temperature compensation offset Args: self temperature offset test Returns: None Raises: None """ self._logger.debug("In Set Temperature Offset") # Need to finish this so value is 32 word split to two 16 words # Calculate crc value for last word # [Cmd MSB, Cmd LSB, Interval MSB, Interval LSB, CRC] # msb, lsb = convert_word(interval_sec, test) # crc = calc_crc(msb, lsb, test) # msg = [0x54,0x03, msb, lsb, crc] msgs = [0x54,0x03, 0x00, 0x02, 0xE3] self._i2c.msg_write(msgs) # TESTE ADN WORKS! def set_altitude(self, altitude): """Altitude compensation Args: self altitude: uint16, height over sea level in [m] above 0 test Returns: None Raises: None """ self._logger.debug("In Set Altitude") # Need to finish this so value is 32 word split to two 16 words # Calculate crc value for last word # [Cmd MSB, Cmd LSB, Interval MSB, Interval LSB, CRC] # msb, lsb = convert_word(interval_sec, test) # crc = calc_crc(msb, lsb, test) # msg = [0x51,0x02, msb, lsb, crc] msgs = [0x51,0x02, 0x00, 0x02, 0xE3] self._i2c.msg_write(msgs) # TESTE ADN WORKS! def get_configured_address(self, test=False): """Altitude compensation Args: self test Returns: self._addr: address of the sensor Raises: None """ self._logger.debug("In Get Configured Address") return self._addr def get_data(self): """High level logic to simply get data Args: self test Returns: co2: co2 value temp: temperature value rh: relative humidity value Raises: NameError: if error in logic (I2C Problem) """ self._logger.debug("In Get Data") for x in range(0, 4): # Only give four reste tries before giving up try: while True: # Test if data is ready if self.get_data_ready(): # fetch data co2, temp, rh = self.read_measurement() self._logger.detail("CO2: " + str(co2)) self._logger.detail("Temp: " + str(temp)) self._logger.detail("RH: " + str(rh)) return co2, temp, rh time.sleep(1) # try reset to see if can recover from errors except Exception as e: self._logger.error("{}, {}".format("Data Ready Err: ", e)) self.__init__() time.sleep(2) # Give up if cannot fix the problems raise NameError("Too Many Failures")
class ListUtil(object): def __init__(self, logger=None): if logger == None: self._logger = Logger("ListUtil", Logger.DETAIL) else: self._logger = logger self._logger.detail("Initialize ListUtil") def build(self, doc): ''' Convert activity list to list of lists Args: doc: list of attributes, should be of format: [activity_name, trial, plot, subject, attribute, value, units, participant, status_qualifier, comment] participant may be a device string, or a list: ['person':'hmw'] Returns: rec: list formatted record ready for the spreadsheet Throws: None ''' # add timestamp and field_id timestamp = datetime.utcnow() ts_str = timestamp.isoformat()[:19] t_str = ts_str.replace('T', ' ') doc.insert(0, env['field']['field_id']) doc.insert(0, t_str) # append date, week of trial start = env['trials'][0]['start_date'] st = datetime.strptime(start, '%Y-%m-%dT%H:%M:%S') dif = timestamp - st # Week of trial wk = int(math.ceil(dif.days / 7)) # date portion doc.append(t_str[:10]) # time portion doc.append(t_str[-8:]) # weeks of trial doc.append(wk) # create binned timestamp - time is 20 minute group ts2 = timestamp.replace(minute=(int(math.floor(timestamp.minute / 20))), second=0) ts2_str = ts2.isoformat()[:19] doc.append(ts2_str) # fix participant if isinstance(doc[PARTICIPANT], list): p_type = doc[PARTICIPANT][0] p_name = doc[PARTICIPANT][1] del doc[PARTICIPANT] doc.insert(PARTICIPANT, p_name) doc.insert(PARTICIPANT, p_type) else: doc.insert(PARTICIPANT, 'device') # Remove 'Field' from Environment & State # 'Plot' is used for Location of sensor self._logger.debug(doc[2]) if doc[2] == 'Environment_Observation': del doc[3] if doc[2] == 'State_Change': del doc[3] # convert to list of lists doc2 = [[el] for el in doc] self._logger.debug(doc2) return doc2
class Reservoir: FULL = 0 EMPTY = 1 OK = 2 vol_per_mm = 0.3785 # estimate 1 gal per 10 mm, this is reservoir specific vol_per_sec = 0.3785 # estimate 100 ml per sec, this is reservoir specific def __init__(self): '''Get distances for determining reservoir levels''' self.res = {'full': full_ec, 'empty': empty_ec, 'timeout': timeout} self._activity_type = 'Agronomic_Activity' self._logger = Logger('LogReservoir', Logger.INFO) self._persist = Persistence(self._logger) self._logger.detail("Reservoir Initialized") # flag for testing self._test = False def getStatus(self): "Logic for calling the reservoir full" self._logger.detail("In getStatus") ec = self.getEC() if ec <= full_ec: self._logger.debug("{}, {}, {:10.1f}".format( "Reservoir Full", EC, ec)) return Reservoir.FULL, ec elif ec >= empty_ec: self._logger.debug("{}, {}, {:10.1f}".format( "Reservoir Empty", EC, ec)) return Reservoir.EMPTY, ec else: self._logger.debug("{}, {}, {:10.1f}".format( "Reservoir not Empty", EC, ec)) return Reservoir.OK, ec def isFull(self): self._logger.detail("In isFull") status, ec = self.getStatus() if status == Reservoir.FULL: return True else: return False def getEC(self): '''Get EC reading''' self._logger.detail("In getEC") snsr = EC(self._logger) return snsr.getEC() def fill(self, test=False): ''' Routine to control re-filling of the reservoir''' self._logger.detail("{}".format("In Fill")) start_ec = self.getEC() start_t = time.time() pump = Pump() pump.on() # Loop till filled or times out while (not self.isFull()) and self.isFilling(start_t): self._logger.detail("{}".format("In Fill Loop")) self._logger.detail("{}".format("Exit Fill Loop, close solenoid")) # Close valve pump.off() # Calculate amount filled stop_t = time.time() stop_ec = self.getEC() dif_t = stop_t - start_t volume = dif_t * self.vol_per_sec self._logger.detail("{}".format("Exit Fill")) return volume def isFilling(self, start_time, test=False): '''Check that actually filling: the distance is actually changing''' start_ec = self.getEC() self._logger.detail("{} {}".format("Filling, Start EC:", start_ec)) time.sleep(fill_time) # Check for level change first, else will never get to this logic till timeout end_ec = self.getEC() change = start_ec - end_ec if end_ec < start_ec: # need to see at least a 5mm change self._logger.detail("{} {} {} {} {} {}".format( "Still Filling, change:", change, "Start", start_ec, "End", end_ec)) return True else: self._logger.detail("{} {} {} {} {} {}".format( "Not Filling, no change:", change, "Start", start_ec, "End", end_ec)) return False # Check for timeout stop_time = time.time() if stop_time - start_time > self.res['timeout']: self._logger.detail("{}".format("Timeout")) return False else: return True def checkReservoir(self): '''Check condition of reservoir and fill if necessary''' self._logger.detail("{}".format("Check Reservoir")) status, ec = self.getStatus() self._logger.debug("{} {} {} {}".format("Status:", status, "EC", ec)) # Is full, log state if status == Reservoir.FULL: self._logger.detail("{}".format("Status: Full")) self._logger.info("{} {} {} {}".format("EC:", ec, "Full level:", self.res['full'], "Empty:", self.res['empty'])) return True else: # Needs filling self._logger.debug("{}".format("Status: Filling")) volume = self.fill() if volume > 0: # Filled, log volume self.logState(volume, 'Success') return True else: # Failure self._logger.debug("{}".format("Status: Failure")) level = 'Empty' if status == '2': level = 'Ok' self._logger.error("{}".format("Failure to fill Reservoir")) self._logger.error("{} {} {} {}".format( "EC:", ec, "Full level:", self.res['full'], "Empty:", self.res['empty'])) self.logState(volume, 'Failure') return False def logState(self, value, status_qualifier): if self._test: status_qualifier = 'Test' txt = { 'Volume': value, 'full_level': self.res['full'], 'empty_level': self.res['empty'], 'status': 'Full' } self._persist.save([ 'State_Change', '', 'Nutrient', 'Reservoir', 'Volume', value, 'Liter', 'Solenoid', status_qualifier, '' ]) self._logger.info(txt)
class CouchUtil(object): def __init__(self, logger=None): if logger == None: self._logger = Logger("LogSensor", Logger.DETAIL) else: self._logger = logger self._activity_type = "Environment_Observation" self._logger.detail("CouchUtil") self._test = False self._server = Server() self._db = self._server[db_name] def processEnv(self, row): ''' Environment specific processing Args: doc: list of attributes, should be of format: [timestamp, field, activity_name, trial, plot, subject, attribute, value, units, participant, status_qualifier, comment] participant may be a device string, or a list: ['person':'hmw'] Returns: rec: json formatted record ready for the database Throws: None ''' rec = self.buildCore(row) rec['activity_type'] = row[ACTIVITY] rec['subject'] = { 'name': row[SUBJECT], 'attribute': { 'name': row[ATTRIBUTE], 'units': row[UNITS], 'value': row[VALUE] }, 'location': row[PLOT] } rec['location'] = {'field': row[FIELD]} return rec def processState(self, row): ''' State specific processing Args: doc: list of attributes, should be of format: [timestamp, field, activity_name, trial, plot, subject, attribute, value, units, participant, status_qualifier, comment] participant may be a device string, or a list: ['person':'hmw'] Returns: rec: json formatted record ready for the database Throws: None ''' self._logger.detail("In State_Change") rec = self.buildCore(row) rec['activity_type'] = row[ACTIVITY] self._logger.detail("Activity: " + str(rec['activity_type'])) rec['subject'] = { 'name': row[SUBJECT], 'attribute': { 'name': row[ATTRIBUTE], 'units': row[UNITS], 'value': row[VALUE] }, 'location': row[PLOT] } self._logger.detail("Subject: " + str(rec['subject'])) rec['participant'] = {'type': 'device', 'name': row[PARTICIPANT]} rec['location'] = {'field': row[FIELD]} self._logger.detail(str(rec)) return rec def processAgro(self, row): ''' Agronomic specific processing Args: doc: list of attributes, should be of format: [timestamp, field, activity_name, trial, plot, subject, attribute, value, units, participant, status_qualifier, comment] participant may be a device string, or a list: ['person':'hmw'] Returns: rec: json formatted record ready for the database Throws: None ''' self._logger.detail("In Process Agro") rec = self.buildCore(row) rec['activity_type'] = row[ACTIVITY] rec['sub-activity'] = row[PLOT] if len(row[SUBJECT]) > 0: rec['subject'] = { 'name': row[SUBJECT], 'attribute': { 'name': row[ATTRIBUTE], 'units': row[UNITS], 'value': row[VALUE] } } rec['location'] = {'field': row[FIELD], 'trial': row[TRIAL]} return rec def processPheno(self, row): ''' Phenotype specific processing Args: doc: list of attributes, should be of format: [timestamp, field, activity_name, trial, plot, subject, attribute, value, units, participant, status_qualifier, comment] participant may be a device string, or a list: ['person':'hmw'] Returns: rec: json formatted record ready for the database Throws: None ''' self._logger.detail("In Process Pheno") rec = self.buildCore(row) rec['activity_type'] = row[ACTIVITY] rec['subject'] = { 'name': row[SUBJECT], 'attribute': { 'name': row[ATTRIBUTE], 'units': row[UNITS], 'value': row[VALUE] } } rec['location'] = { 'field': row[FIELD], 'trial': row[TRIAL], 'plot': row[PLOT] } return rec def buildCore(self, row): ''' Build the core of the json structure, common elements Args: row: list of activity [timestamp, field, activity_name, trial, plot, subject, attribute, value, units, participant, status_qualifier, comment] participant may be a device string, or a list: ['person':'hmw'] Returns: rec: json formatted record ready for the database Throws: None ''' self._logger.detail("In buildCore") rec = {} rec['start_date'] = {'timestamp': row[TS]} if isinstance(row[PARTICIPANT], list): rec['participant'] = { 'type': row[PARTICIPANT][0], 'name': row[PARTICIPANT][1] } else: rec['participant'] = {'type': 'device', 'name': row[PARTICIPANT]} if len(row[COMMENT]) == 0: rec['status'] = { 'status': 'Complete', 'status_qualifier': row[STATUS] } else: rec['status'] = { 'status': 'Complete', 'status_qualifier': row[STATUS], 'comment': row[COMMENT] } if row[STATUS] == "Success": self._logger.detail(rec) elif row[STATUS] == "Failure": self._logger.error("Failure" + str(rec)) elif row[STATUS] == "Test": self._logger.detail(rec) else: self._logger.error("Unknown Status" + str(rec)) return rec def save(self, doc, test=False): ''' Convert activity list to json structure and save to database This is the entry point for all other functions Args: doc: list of attributes, should be of format: [activity_name, trial, plot, subject, attribute, value, units, participant, status_qualifier, comment] participant may be a device string, or a list: ['person':'hmw'] Returns: rec: json formatted record ready for the database Throws: None ''' self._logger.detail("In saveList") # dictionary of activity types and specific processing functions proc = { 'Environment_Observation': self.processEnv, 'State_Change': self.processState, 'Agronomic_Activity': self.processAgro, 'Phenotype_Observation': self.processPheno } # add timestamp and field_id timestamp = datetime.utcnow().isoformat()[:19] doc.insert(0, env['field']['field_id']) doc.insert(0, timestamp) # Use activity type to route processing rec = proc[doc[2]](doc) self.saveRec(rec, test) def saveRec(self, rec, test=False): ''' Persist json structure to a database Args: rec: json structure Returns: id: document id rev: revision number of document Throws: None ''' # print rec id, rev = self._db.save(rec)
class I2C(object): def __init__(self, path, addr, logger=None): self._path = path self._addr = addr self._i2c = pI2C(self._path) self._logger = logger if logger == None: self._logger = Logger("SCD30", Logger.INFO) self._logger.debug("initialize I2C object") def __exit__(self, exc_type, exc_value, traceback): self._i2c.close() def msg_write(self, cmds): """Write to sensor Args: cmds: commands to send Returns: msgs: basically returns the cmds, since nothing altered Raises: None """ self._logger.debug("In Msg Write") for cmd in cmds: self._logger.detail("{}, {}".format("Cmd: ", cmd)) msgs = [self._i2c.Message(cmds)] try: self._logger.detail("Transfer") self._i2c.transfer(self._addr, msgs) msb = msgs[0].data[0] self._logger.detail("{}, {}".format("MSB: ", hex(msb))) return msgs except Exception as e: self._logger.error(str(e)) return None def msg_read(self, size, cmds=None): """Read existing data Args: cmds: addresses to read from - optional for some sensors (SI7021) size: size of byte array for receiving data Returns: msgs: message package, last one should hold data Raises: None """ self._logger.detail("{}, {}, {}, {}".format("Msg Read - size: ", size, " cmds: ", cmds)) sz = self._i2c.Message(bytearray([0x00 for x in range(size)]), read=True) msgs = [sz] # print("C " + str(type(cmds))) if cmds is not None: # print("Cmds " + str(cmds)) rd = self._i2c.Message(cmds) msgs = [rd, sz] try: self._i2c.transfer(self._addr, msgs) return msgs except Exception as e: self._logger.error(str(e)) return None def get_data(self, cmd, sleep, size, read=None): '''Combine sending of command and reading Some sensors default the read to the prior command and don't specify a read address ''' self._logger.debug("{}, {}, {}, {}, {}, {}".format( "In Get Data-cmd: ", cmd, " sleep: ", sleep, " size: ", size)) self.msg_write(cmd) time.sleep(sleep) msgs = self.msg_read(size, read) if msgs == None: return None else: for msg in msgs: self._logger.detail("-") for dt in msg.data: self._logger.detail("Dt " + str(dt)) self._logger.debug("Data " + str(msgs[0].data[0]) + " " + str(msgs[0].data[1])) value = self.bytesToWord(msgs[0].data[0], msgs[0].data[1]) return msgs def bytesToWord(self, high, low): """Convert two byte buffers into a single word value shift the first byte into the work high position then add the low byte Args: high: byte to move to high position of word low: byte to place in low position of word Returns: word: the final value Raises: None """ self._logger.debug("{}, {}, {}, {}".format("In Bytes To Word-high: ", high, " Low: ", low)) word = (high << 8) + low return word