class Harvester( object ):
"""Harvest data from a modbus device"""
def __init__(self, configpath, datapath, context):
self.configpath = configpath
self.datapath = datapath
self.config = ConfigParser.SafeConfigParser()
self.disp = DisplayController()
#paths are absolute
if not isfile(self.configpath):
raise SensorConfigException(self.configpath, 'File Not Found')
if not isdir(self.datapath):
raise SensorConfigException(self.datapath, 'Directory Not Found')
try:
self.config.readfp(open(join(context['config'],'sensor.default.cfg')))
except (ConfigParser.Error, IOError, ValueError, TypeError):
logger.exception("Unable to load default sensor configuration")
try:
self.config.read(configpath)
except (ConfigParser.Error, IOError):
logger.exception("Unable to sensor config from %s" % self.configpath)
if not self.config.has_option('DEFAULT', 'name'):
logger.warn("Config file has no name for sensor, defaulting to filename")
self.config.set('DEFAULT', 'name', splitext(split(self.configpath)[1])[0])
logger.info("Configuration for sensor %s loaded" % self.config.get('DEFAULT', 'name'))
return
def _open_binaryfile(self, sect):
binpath = join(self.datapath, self.config.get('DEFAULT', 'name'))
binpath += '-' + sect.lower() + '.data'
logger.info("Opening binary file %s for append." % binpath)
try:
fh = open(binpath, 'ab')
except (IOError, OSError):
logger.exception("Unable to open bin file.")
fh.close()
return (fh, binpath)
def _append_binaryfile_data(self, sect, bindata):
fh, binpath = self._open_binaryfile(sect)
fh.write(bindata)
fh.close()
return binpath
def _open_datafile(self):
self.filepath = join(self.datapath, self.config.get('DEFAULT', 'name'))
self.filepath += '.csv'
logger.info("Opening data file %s for append." % self.filepath)
if exists(self.filepath):
self.newfile = False
else:
self.newfile = True
try:
tmpfh = open(self.filepath, 'w')
tmpfh.write("# Data file %s\r\n" % self.filepath)
tmpfh.write("# Created on %s by config %s\r\n" % (datetime.now().strftime("%d%b%Y-%H%M%S"), self.configpath))
except (IOError, OSError):
logger.exception("Could not header comments to datafile.")
finally:
tmpfh.close()
try:
fh = open(self.filepath, 'ab')
except (IOError, OSError):
logger.exception("Unable to open datafile for append. (%s)." % self.filepath)
if not fh:
(fh, path) = tempfile.mkstemp()
logger.critical("No datafile, creating temp %s" % path)
self.filepath = path
self.fh = fh
return fh
def _close_datafile(self, filehandle):
filehandle.close()
def _csv_get_columns(self):
fields = ['date']
for section in self.config.sections():
if self.config.has_option(section, 'map_col'):
if self.config.getboolean(section,'map_col'):
if self.config.has_option(section, 'col_name'):
fields.append(self.config.get(section, 'col_name'))
else:
fields.append(str(section))
logger.debug('Mapping config section to column %s' % section)
logger.info('Mapped %d columns.' % len(fields))
return fields
def _get_sampling_data(self, sect):
# sampling interval
if self.config.has_option(sect, 'samp_interval'):
try:
samp_i = self.config.getfloat(sect, 'samp_interval')
except (ConfigParser.Error, TypeError):
samp_i = 2e-5
logger.exception("samp_interval has bad value.")
else:
samp_i = 2e-5
# sampling count
if self.config.has_option(sect, 'samp_count'):
try:
samp_c = self.config.getint(sect, 'samp_count')
except (ConfigParser.Error, TypeError):
samp_c = 20
logger.exception("samp_count has bad value.")
else:
samp_c = 20
return (samp_c, samp_i)
def _process_config(self):
sects = self.config.sections()
sects.sort()
datarow = {'date': datetime.utcnow().strftime("%d-%m-%YT%H-%M-%S")}
files = []
if self.config.has_option('DEFAULT', 'port_num'):
try:
port_num = self.config.getint('DEFAULT', 'port_num')
except ConfigParser.Error:
port_num = 0
else:
port_num = 3
if port_num > 3:
port_num = 0
logger.info("Powering datalogger port %d." % port_num)
self.disp.switch_port_power(port_num, True)
time.sleep(0.2)
for sect in sects:
logger.info("Entering config section %s." % sect)
if self.config.has_option(sect, 'col_name'):
key = self.config.get(sect, 'col_name')
else:
key = str(sect)
output_value = self._process_config_section_mb(sect)
if self.config.has_option(sect, 'operation'):
op = self.config.get(sect, 'operation')
else:
op = 'str'
samp_c, samp_i = self._get_sampling_data(sect)
if output_value:
if op == 'rms':
output_value = str(Processor.get_rms(output_value))
elif op == 'avg':
output_value = str(Processor.get_avg(output_value))
elif op == 'binary':
output_value = str(Processor.encode_binary(datetime.utcnow(), output_value, samp_c))
files.append(self._append_binaryfile_data(sect, output_value))
elif op == 'single':
output_value = Processor.single_value(output_value)
else:
output_value = str(output_value)
else:
output_value = '0'
try:
datarow[key] = output_value
except (KeyError, ValueError, TypeError):
logger.exception("Could not process config section %s." % sect)
if datarow[key] == False:
logger.warn("Error potentially occured in getting data for %s." % sect)
logger.info("Leaving config section %s." % sect)
# sleep if the operation told us to wait
if self.config.has_option(sect, 'wait'):
try:
wait = self.config.getfloat(sect, 'wait')
except ConfigParser.Error:
wait = 5.0
logger.info("Sleeping for %.2f seconds." % wait)
time.sleep(wait)
logger.info("Removing power from datalogger port %d.", port_num)
self.disp.switch_port_power(port_num, False)
logger.info("Making data entry in csv file.")
try:
fh = self._open_datafile()
cols = self._csv_get_columns()
logger.debug("Columns: %s." % str(cols))
logger.debug("Data: %s." % str(datarow))
csvw = csv.DictWriter(fh, cols, extrasaction='ignore')
if self.newfile:
header = {}
for col in cols:
header[col] = col
csvw.writerow(header)
csvw.writerow(datarow)
except:
logger.exception("Could not write data row to csv file.")
finally:
fh.close()
files.append(self.filepath)
return files
def harvest(self):
try:
return self._process_config()
except:
logger.exception("Unable to process %s." % self.configpath)
return False
def _process_config_section_mb(self, section):
# use the config section to open a modbus handle
logger.info("Processing config section %s." % str(section))
try:
portname = self.config.get(section,'serialport')
logger.warn("%s" % portname)
time.sleep(20)
speed = self.config.getint(section, 'serialbaud')
if self.config.has_option(section, 'serialtimeout'):
timeout = self.config.getint(section, 'serialtimeout')
else:
timeout = 5.0
except (IOError, ConfigParser.Error):
logger.warning("Error config file %s, section %s." % (self.configpath, section))
# fallback on defaults
portname = '/dev/ttyS1'
speed = 9600
timeout = 5.0
self.disp.control_led('mbfrm', True)
self.disp.control_led('mberr', False)
try:
ser = serial.Serial(portname, speed, timeout=timeout)
except (OSError, IOError, serial.SerialException):
logger.exception("Unable to open serial port %s@%d." % (portname, speed))
ser.close()
self.disp.control_led('mbfrm', False)
self.disp.control_led('mberr', True)
return False
logger.debug("Took control of serial port %s, speed %d, timeout %d" % (portname, speed, timeout))
# load modbus specific information
try:
# function code : 0x03 READ HOLDING, 0x04 READ INPUT, 0x06 WRIE SINGLE
mb_func = int(self.config.get(section, 'mb_func'), 0)
mb_start = int(self.config.get(section, 'mb_start'), 0)
mb_address = int(self.config.get(section, 'mb_addr'), 0)
# if a read is specified, we add a count
if self.config.has_option(section, 'mb_count'):
mb_count = int(self.config.get(section, 'mb_count'), 0)
else:
mb_count = 0
# if a write is specified, we append output_value
if self.config.has_option(section, 'mb_write'):
mb_write = int(self.config.get(section, 'mb_write'), 0)
else:
mb_write = 0
except ConfigParser.Error:
logger.exception("Unable to get data for modbus transaction.")
ser.close()
self.disp.control_led('mbfrm', False)
self.disp.control_led('mberr', True)
return False
# Create the Modbus Master
master = modbus_rtu.RtuMaster(ser)
#master.set_verbose(True)
master.set_timeout(timeout)
logger.debug("Function to execute: a:0x%02x, f:0x%02x, s:0x%04x, c:0x%04x, w:0x%04x." % (mb_address, mb_func, mb_start, mb_count, mb_write))
logger.info("Running modbus function 0x%02x on device 0x%02x" % (mb_func, mb_address))
try:
if ((mb_func == cst.READ_HOLDING_REGISTERS) or (mb_func == cst.READ_INPUT_REGISTERS)):
data = master.execute(mb_address, mb_func, mb_start, mb_count)
logger.debug('Read Op: return data for 0x%02x fn(0x%02x) is %s.' % (mb_address, mb_func, hexbyte.Int16ToHex(list(data))))
elif mb_func == cst.WRITE_SINGLE_REGISTER:
data = master.execute(mb_address, mb_func, mb_start, output_value=mb_write)
logger.debug('Writing data to 0x%02x: [0x%04x] 0x%02x.' % (mb_address, mb_start, mb_write))
else:
logger.error('Unrecognised function code requested.')
except (serial.SerialException, ModbusFunctionNotSupportedError,
ModbusInvalidResponseError, ModbusNotConnectedError,
ModbusError):
# exception handler
self.disp.control_led('mbfrm', False)
self.disp.control_led('mberr', True)
logger.exception('Func(0x%02x) on 0x%02x failed.' % (mb_func, mb_address))
data = False
time.sleep(0.5)
ser.close()
self.disp.control_led('mbfrm', False)
return data
class DataLogger(object):
datadir = '/data'
logger = logging.getLogger('eko.DataLogger')
def __init__(self, cfg):
config = ConfigParser()
try:
config.read(cfg)
except:
self.logger.exception("Unable to read config file %s." % cfg)
# try to open the databases
self.logger.info("Opening databases.")
try:
DBSetup.check_databases()
except (sqlite3.Error, IOError, OSError):
self.logger.exception("Databases could not be opened.")
self.disp = DisplayController()
try:
## removes all modules, shutsdown hub
#Beagleboard.goto_known_state()
pass
except (OSError, IOError):
self.logger.exception("Unable to reset board to default setting.")
def _try_network(self):
# try ping first
#try:
# self.logger.info("Trying ping for google.com.")
# x = ping.do_one('www.google.com', 10000, 1, 8)
#except socket.error:
# self.logger.exception("Ping failed!")
# x = None
google_req = urllib2.Request('http://www.google.com/index.html')
try:
self.logger.info("Trying url fetch on google.com")
x = urllib2.urlopen(google_req, timeout=60)
except urllib2.URLError:
self.logger.exception("URL Error, unable to reach google.com")
x = None
return x
def ready_internet(self):
self.logger.info("Attempting to connect to the internet.")
## open a pppd instance
OSTools.pppd_launch()
## wait for pppd to settle
time.sleep(5)
retrycount = 5
while retrycount > 0:
## to see if ppp is up
if OSTools.pppd_status():
## ppp is up, try ping
x = self._try_network()
if x is not None:
self.disp.control_led('net', True)
self.disp.control_led('neterr', False)
self.logger.info("Ping success.")
return True
else:
self.disp.control_led('neterr', True)
self.disp.control_led('net', False)
self.logger.info("Sleeping for 10s.")
time.sleep(10)
else:
# ppp is not up
## check if process is running
ppp_pid = OSTools.pppd_pid()
if ppp_pid == 0:
## ppp has quit
raise InternetConnectionError('pppd unexpectedly quit!')
else:
## wait 10 seconds, and retry
time.sleep(10)
retrycount -= 1
self.logger.info("Rechecking network, remaining attempts %d." % retrycount)
OSTools.pppd_terminate(OSTools.pppd_pid())
return False
def stop_internet(self):
self.logger.info("Dropping net connection.")
self.disp.control_led('net', False)
return OSTools.pppd_terminate(OSTools.pppd_pid())
def datalog(self):
# instantiate a harvest dispatcher
dispatch = EkoDispatcher()
dispatch.import_configs()
self.logger.info("Dispatching all sensor polling operations.")
dispatch.dispatch_all()
self.logger.info("All sensors polled.")
return
def upload_kiosk_messages(self):
try:
CMsgs.transmit_clientmessages()
except:
self.logger.exception("Unable to transmit client messages to server.")
return False
return True
def _download_server_messages(self):
messages = SMsgs.get_messages()
return messages
def service_server_messages(self, ignore_sa=False):
msgs = self._download_server_messages()
for msg in msgs:
self.logger.debug("Message %s." % str(msg))
if 'msg_type' not in msg.keys():
self.logger.warn("No message type specified.")
continue
if 'msg' not in msg.keys():
self.logger.warn("No message specified.")
continue
if msg['msg_type'] == 'STAYALIVE':
if not ignore_sa:
self.stay_alive_routine(msg['msg'])
elif msg['msg_type'] == 'CMD':
self._exec_process(msg['msg'])
elif msg['msg_type'] == 'GIVELOGS':
res = self.upload_logs()
CMsgs.add_clientmessage("Logs delivered.", res, "Command parser", datetime.utcnow())
else:
self.logger.warn("Unrecogised command.")
return
def _exec_process(self,message):
self.logger.info("Executing message from server.")
self.logger.debug("Command is %s." % message)
try:
proc = subprocess.Popen("message", shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
except:
self.logger.exception("Subprocess could not be opened.")
return False
now = datetime.utcnow()
start = datetime.utcnow() + timedelta(seconds=240)
while now < start:
now = datetime.now()
if proc.poll() is not None:
timeout = 0
break;
timeout = 1
if timeout:
self.logger.error("Process timed out.")
return False
ret_tuple = proc.communicate()
self.logger.debug("Command returns: %s." % str(ret_tuple))
CMsgs.add_clientmessage("Command Executed:\n%s" % str(ret_tuple), '', "Command Executer", datetime.utcnow())
return
def stay_alive_routine(self,message_text):
self.logger.info("Stay Alive for 30 minutes.")
now = datetime.utcnow()
stop = now + timedelta(seconds=1800)
CMsgs.add_clientmessage("Staying Alive.", '', "Command parser", datetime.utcnow())
while now < stop:
now = datetime.utcnow()
time.sleep(60)
try:
self.upload_kiosk_messages()
except:
self.logger.exception("An error occured when uploading messages.")
# ignore another call to stay alive
try:
self.service_server_messages(ignore_sa=True)
except:
self.logger.exception("An error occured when attempting to fetch new orders.")
CMsgs.add_clientmessage("Going to die.", '', "Command parser", datetime.utcnow())
try:
self.upload_kiosk_messages()
except:
self.logger.exception("An error occured when uploading messages.")
def upload_logs(self):
upd = Uploader.DataUploader()
ret = upd.zip_logfiles()
if not ret:
self.logger.info("Upload task exited. Error or nothing to sync.")
return False
(zipfile, manifest) = ret
res = upd.upload_file(zipfile, manifest, upload_type="logs")
if res:
upd.create_sync_record(zipfile)
else:
self.disp.control_led('neterr', True)
def upload_data_messages(self):
upd = Uploader.DataUploader()
upd.get_filelist()
ret = upd.build_zip_file()
if not ret:
self.logger.info("Upload task failed!")
return False
(zipfile, manifest) = ret
res = upd.upload_file(zipfile, manifest)
if res:
upd.create_sync_record(zipfile)
upd.update_filelist()
else:
self.disp.control_led('neterr', True)
def netsync(self):
# open a internet connection
## power the modem
self.disp.control_led('all', False)
#try:
# x = Beagleboard.turn_on_usbhub()
#except (OSError, IOError):
# self.logger.exception("Error encountered when attempting to turn on USB Hub.")
# x = False
### raise error led if hub power failed
#if not x:
# self.disp.control_led('neterr', True)
## wait for system to settle
time.sleep(10)
retrycount = 3
while retrycount > 0:
try:
#res = self.ready_internet()
res = 'Foo'
except InternetConnectionError:
self.logger.exception('Could not dial modem.')
res = None
if res is not None:
break
self.logger.info("Waiting 30 seconds till next attempt.")
time.sleep(30)
retrycount -= 1
self.logger.info("%d attempts left." % retrycount)
# Assume we have net connectivity by this point.
## sync time if need be
os.popen('ntpdate -t 90 0.pool.ntp.org 1.pool.ntp.org 2.pool.ntp.org')
self.disp.control_led('sync', True)
try:
self.upload_kiosk_messages()
except:
self.logger.exception("Unable to upload kiosk messages")
self.disp.control_led('neterr', True)
try:
self.upload_data_messages()
except:
self.logger.exception('Network Synchronisation Failed!')
self.disp.control_led('neterr', True)
try:
self.service_server_messages()
except:
self.logger.exception("Unable to get commands from server.")
self.disp.control_led('sync', False)
# terminate network
#self.stop_internet()
time.sleep(5)
## power off the modem
#try:
# Beagleboard.turn_off_usbhub()
#except:
# self.logger.exception("Error encountered when attempting to power off USB hub.")
def run(self):
# mark
starttime = datetime.utcnow()
nextsync = starttime
while True:
try:
self.datalog()
except KeyboardInterrupt:
exit(0)
except:
self.logger.exception("Unhandled exception!")
self.logger.info("Data logging operation complete.")
# next poll is scheduled for time nextpoll. If nextpoll is ahead
# tell beagle to sleep for 10 mins
self.logger.info("Sleeping for 600 seconds.")
try:
os.popen('echo 120 > /debug/pm_debug/wakeup_timer_seconds')
os.popen('echo mem > /sys/power/state')
except (IOError, OSError):
self.logger.exception("Unable to put system to sleep")
# wait 60 seconds
time.sleep(20)
# check if its time for a netsync
if datetime.utcnow() > nextsync:
self.netsync()
# next sync is in 6 hours
nextsync = datetime.utcnow() + timedelta(minutes=15)
td = nextsync - datetime.utcnow()
self.logger.info("Next internet sync is in %.2f minutes." % (td.seconds/60.0))