def processArduinoData(self, sensorid, meta, data):
"""Convert raw ADC counts into SI units as per datasheets"""
currenttime = datetime.utcnow()
outdate = datetime.strftime(currenttime, "%Y-%m-%d")
actualtime = datetime.strftime(currenttime, "%Y-%m-%dT%H:%M:%S.%f")
outtime = datetime.strftime(currenttime, "%H:%M:%S")
timestamp = datetime.strftime(currenttime, "%Y-%m-%d %H:%M:%S.%f")
filename = outdate
datearray = acs.timeToArray(timestamp)
packcode = '6hL'
#sensorid = self.sensordict.get(idnum)
#events = self.eventdict.get(idnum).replace('evt','').split(',')[3:-1]
values = []
multiplier = []
for dat in data:
try:
values.append(float(dat))
datearray.append(int(float(dat) * 10000))
packcode = packcode + 'l'
multiplier.append(10000)
except:
log.msg(
'{} protocol: Error while appending data to file (non-float?): {}'
.format(self.sensordict.get('protocol'), dat))
try:
data_bin = struct.pack('<' + packcode, *datearray) #little endian
except:
log.msg('{} protocol: Error while packing binary data'.format(
self.sensordict.get('protocol')))
pass
key = '[' + str(meta.get('SensorKeys')).replace("'", "").strip() + ']'
ele = '[' + str(meta.get('SensorElements')).replace("'",
"").strip() + ']'
unit = '[' + str(meta.get('SensorUnits')).replace("'",
"").strip() + ']'
multplier = str(multiplier).replace(" ", "")
header = "# MagPyBin %s %s %s %s %s %s %d" % (
sensorid, key, ele, unit, multplier, packcode,
struct.calcsize('<' + packcode))
if not self.confdict.get('bufferdirectory', '') == '':
acs.dataToFile(self.confdict.get('bufferdirectory'), sensorid,
filename, data_bin, header)
return ','.join(list(map(str, datearray))), header
def processData(self, data):
"""Convert raw ADC counts into SI units as per datasheets"""
currenttime = datetime.utcnow()
# Correction for ms time to work with databank:
currenttime_ms = currenttime.microsecond / 1000000.
ms_rounded = round(float(currenttime_ms), 3)
if not ms_rounded >= 1.0:
currenttime = currenttime.replace(microsecond=int(ms_rounded *
1000000.))
else:
currenttime = currenttime.replace(microsecond=0) + timedelta(
seconds=1.0)
filename = datetime.strftime(currenttime, "%Y-%m-%d")
actualtime = datetime.strftime(currenttime, "%Y-%m-%dT%H:%M:%S.%f")
lastActualtime = currenttime
outtime = datetime.strftime(currenttime, "%H:%M:%S")
timestamp = datetime.strftime(currenttime, "%Y-%m-%d %H:%M:%S.%f")
sensorid = self.sensor
packcode = '6hLL'
header = "# MagPyBin %s %s %s %s %s %s %d" % (
self.sensor, '[f]', '[f]', '[nT]', '[1000]', packcode,
struct.calcsize('<' + packcode))
try:
intval = data[1].split(',')
value = float(intval[0].strip())
if 10000 < value < 100000:
intensity = value
else:
intensity = 88888.0
except ValueError:
log.err("CS - Protocol: Not a number. Instead found:", data[0])
intensity = 88888.0
try:
datearray = acs.timeToArray(timestamp)
datearray.append(int(intensity * 1000))
data_bin = struct.pack('<' + packcode, *datearray)
except:
log.msg('Error while packing binary data')
if not self.confdict.get('bufferdirectory', '') == '':
acs.dataToFile(self.confdict.get('bufferdirectory'), sensorid,
filename, data_bin, header)
return ','.join(list(map(str, datearray))), header
def processData(self, data):
"""Process Environment data """
currenttime = datetime.utcnow()
outdate = datetime.strftime(currenttime, "%Y-%m-%d")
filename = outdate
actualtime = datetime.strftime(currenttime, "%Y-%m-%dT%H:%M:%S.%f")
outtime = datetime.strftime(currenttime, "%H:%M:%S")
timestamp = datetime.strftime(currenttime, "%Y-%m-%d %H:%M:%S.%f")
packcode = '6hLllL'
sensorid = self.sensor
header = "# MagPyBin %s %s %s %s %s %s %d" % (
sensorid, '[t1,t2,var1]', '[T,DewPoint,RH]', '[degC,degC,per]',
'[1000,1000,1000]', packcode, struct.calcsize('<' + packcode))
valrh = re.findall(r'\d+', data[0])
if len(valrh) > 1:
temp = float(valrh[0] + '.' + valrh[1])
else:
temp = float(valrh[0])
valrh = re.findall(r'\d+', data[1])
if len(valrh) > 1:
rh = float(valrh[0] + '.' + valrh[1])
else:
rh = float(valrh[0])
valrh = re.findall(r'\d+', data[2])
if len(valrh) > 1:
dew = float(valrh[0] + '.' + valrh[1])
else:
dew = float(valrh[0])
try:
datearray = acs.timeToArray(timestamp)
datearray.append(int(temp * 1000))
datearray.append(int(dew * 1000))
datearray.append(int(rh * 1000))
data_bin = struct.pack('<' + packcode,
*datearray) #use little endian byte order
except:
log.msg('Error while packing binary data')
pass
if not self.confdict.get('bufferdirectory', '') == '':
acs.dataToFile(self.confdict.get('bufferdirectory'), sensorid,
filename, data_bin, header)
return ','.join(list(map(str, datearray))), header
def processOwData(self, sensorid, datadict):
"""Process OW data """
currenttime = datetime.utcnow()
outdate = datetime.strftime(currenttime, "%Y-%m-%d")
filename = outdate
actualtime = datetime.strftime(currenttime, "%Y-%m-%dT%H:%M:%S.%f")
outtime = datetime.strftime(currenttime, "%H:%M:%S")
timestamp = datetime.strftime(currenttime, "%Y-%m-%d %H:%M:%S.%f")
packcode = '6hL' + 'l' * len(datadict)
multplier = str([1000] * len(datadict)).replace(' ', '')
if sensorid.startswith('DS18'):
key = '[t1]'
ele = '[T]'
unit = '[degC]'
elif sensorid.startswith('DS2438'):
#'temperature','VAD','VDD','humidity','vis'
key = '[t1,var1,var2,var3,var4]'
ele = '[T,RH,VDD,VAD,VIS]'
unit = '[degC,per,V,V,V,V]'
header = "# MagPyBin %s %s %s %s %s %s %d" % (
sensorid, key, ele, unit, multplier, packcode,
struct.calcsize('<' + packcode))
data_bin = None
datearray = ''
try:
datearray = acs.timeToArray(timestamp)
paralst = typedef.get(sensorid.split('_')[0])
for para in paralst:
if para in datadict:
datearray.append(int(float(datadict[para]) * 1000))
data_bin = struct.pack('<' + packcode,
*datearray) # little endian
except:
log.msg('Error while packing binary data')
if not self.confdict.get('bufferdirectory', '') == '' and data_bin:
acs.dataToFile(self.confdict.get('bufferdirectory'), sensorid,
filename, data_bin, header)
#print ("Sending", ','.join(list(map(str,datearray))), header)
return ','.join(list(map(str, datearray))), header
def processData(self, data):
currenttime = datetime.utcnow()
date = datetime.strftime(currenttime, "%Y-%m-%d")
actualtime = datetime.strftime(currenttime, "%Y-%m-%dT%H:%M:%S.%f")
outtime = datetime.strftime(currenttime, "%H:%M:%S")
filename = date
timestamp = datetime.strftime(currenttime, "%Y-%m-%d %H:%M:%S.%f")
intensity = 88888.8
typ = "none"
dontsavedata = False
packcode = '6hLLl'
header = "# MagPyBin %s %s %s %s %s %s %d" % (
self.sensor, '[f,var1]', '[f,err]', '[nT,none]', '[1000,1000]',
packcode, struct.calcsize('<' + packcode))
try:
# Extract data
data_array = data
if len(data_array) == 2:
typ = "oldbase"
elif len(data_array) == 3:
typ = "valid"
# add other types here
except:
log.err(
'GSM19 - Protocol: Output format not supported - use either base, ... or mobile'
)
# Extracting the data from the station
# Extrat time info and use as primary if GPS is on (in this case PC time is secondary)
# PC is primary when a GPS is not connected
if typ == "valid" or typ == "oldbase": # Comprises Mobile and Base Station mode with single sensor and no GPS
intensity = float(data_array[1])
try:
systemtime = datetime.strptime(date + "-" + data_array[0],
"%Y-%m-%d-%H%M%S.%f")
except:
# This exception happens for old GSM19 because time is
# provided e.g. as 410356 instead of 170356 for 17:03:56 (Thursday)
# e.g 570301.0 instead of 09:03:01 (Friday)
try:
hournum = int(data_array[0][:-6])
rest = data_array[0][-6:]
factor = np.floor(hournum /
24.) # factor = days since starting
hour = int(hournum - factor * 24.)
systemtime = datetime.strptime(
date + "-" + str(hour) + rest, "%Y-%m-%d-%H%M%S.%f")
#print ("Got oldbase systemtime")
except:
systemtime = currenttime
self.timesource = 'NTP'
if len(data_array) == 2:
typ = "base"
errorcode = 99
elif len(data_array[2]) == 3:
typ = "base"
errorcode = int(data_array[2])
else:
typ = "gradient"
gradient = float(data_array[2])
elif typ == "none":
dontsavedata = True
pass
gpstime = datetime.strftime(systemtime, "%Y-%m-%d %H:%M:%S.%f")
try:
# Analyze time difference between GSM internal time and utc from PC
timelist = sorted([systemtime, currenttime])
timediff = timelist[1] - timelist[0]
#secdiff = timediff.seconds + timediff.microseconds/1E6
delta = timediff.total_seconds()
if not delta in [0.0, np.nan, None]:
self.delaylist.append(timediff.total_seconds())
self.delaylist = self.delaylist[-1000:]
if len(self.delaylist) > 100:
try:
self.timedelay = np.median(np.asarray(self.delaylist))
except:
self.timedelay = 0.0
if delta > self.timethreshold:
self.errorcnt['time'] += 1
if self.errorcnt.get('time') < 2:
log.msg(
"{} protocol: large time difference observed for {}: {} sec"
.format(self.sensordict.get('protocol'), sensorid,
secdiff))
else:
self.errorcnt['time'] = 0
except:
pass
if self.sensordict.get('ptime', '') in ['NTP', 'ntp']:
secondtime = gpstime
maintime = timestamp
else:
maintime = gpstime
secondtime = timestamp
try:
if not typ == "none":
# extract time data
datearray = acs.timeToArray(maintime)
try:
datearray.append(int(intensity * 1000.))
if typ == 'base':
datearray.append(int(errorcode * 1000.))
else:
datearray.append(int(gradient * 1000.))
data_bin = struct.pack('<' + packcode, *datearray)
except:
log.msg(
'GSM19 - Protocol: Error while packing binary data')
pass
except:
log.msg('GSM19 - Protocol: Error with binary save routine')
pass
if not self.confdict.get('bufferdirectory', '') == '':
acs.dataToFile(self.confdict.get('bufferdirectory'), self.sensor,
filename, data_bin, header)
return ','.join(list(map(str, datearray))), header
def processLemiData(self, data):
"""Convert raw ADC counts into SI units as per datasheets"""
if len(data) != 153:
log.err('LEMI - Protocol: Unable to parse data of length %i' %
len(data))
#print ("Processing data ...")
""" TIMESHIFT between serial output (and thus NTP time) and GPS timestamp """
currenttime = datetime.utcnow()
date = datetime.strftime(currenttime, "%Y-%m-%d")
timestamp = datetime.strftime(currenttime, "%Y-%m-%d %H:%M:%S.%f")
outtime = datetime.strftime(currenttime, "%H:%M:%S")
datearray = acs.timeToArray(timestamp)
date_bin = struct.pack('6hL', datearray[0] - 2000, datearray[1],
datearray[2], datearray[3], datearray[4],
datearray[5], datearray[6])
# define pathname for local file storage
# (default dir plus hostname plus sensor plus year) and create if not existing
path = os.path.join(self.confdict.get('bufferdirectory'), self.sensor)
if not os.path.exists(path):
os.makedirs(path)
packcode = "<4cb6B8hb30f3BcBcc5hL"
header = "LemiBin %s %s %s %s %s %s %d\n" % (
self.sensor, '[x,y,z,t1,t2]', '[X,Y,Z,T_sensor,T_elec]',
'[nT,nT,nT,deg_C,deg_C]', '[0.001,0.001,0.001,100,100]', packcode,
struct.calcsize(packcode))
headforsend = "# MagPyBin {} {} {} {} {} {} {}\n".format(
self.sensor, '[x,y,z,t1,t2,var1]', '[X,Y,Z,T_sensor,T_elec,VDD]',
'[nT,nT,nT,deg_C,deg_C,V]', '[0.001,0.001,0.001,100,100,10]',
packcode, struct.calcsize(packcode))
# save binary raw data to buffer file ### please note that this file always contains GPS readings
lemipath = os.path.join(path, self.sensor + '_' + date + ".bin")
if not os.path.exists(lemipath):
with open(lemipath, "ab") as myfile:
myfile.write(header)
try:
with open(lemipath, "ab") as myfile:
myfile.write(data + date_bin)
pass
except:
log.err('LEMI - Protocol: Could not write data to file.')
# unpack data and extract time and first field values
# This data is streamed via mqtt
try:
data_array = struct.unpack("<4cB6B8hb30f3BcB", data)
except:
log.err("LEMI - Protocol: Bit error while reading.")
try:
newtime = []
biasx = float(data_array[16]) / 400.
biasy = float(data_array[17]) / 400.
biasz = float(data_array[18]) / 400.
x = (data_array[20]) * 1000.
xarray = [elem * 1000. for elem in data_array[20:50:3]]
y = (data_array[21]) * 1000.
yarray = [elem * 1000. for elem in data_array[21:50:3]]
z = (data_array[22]) * 1000.
zarray = [elem * 1000. for elem in data_array[22:50:3]]
temp_sensor = data_array[11] / 100.
temp_el = data_array[12] / 100.
vdd = float(data_array[52]) / 10.
gpsstat = data_array[53]
gpstime = datetime(
2000 + self.h2d(data_array[5]), self.h2d(data_array[6]),
self.h2d(data_array[7]), self.h2d(data_array[8]),
self.h2d(data_array[9]), self.h2d(
data_array[10])) - timedelta(microseconds=300000)
#gps_time = datetime.strftime(gps_array, "%Y-%m-%d %H:%M:%S")
except:
log.err("LEMI - Protocol: Number conversion error.")
#print ("HERE2", packcode, struct.calcsize(packcode))
if not gpsstat in ['P', 'A']:
print(" ERROR in BINDATA:", struct.unpack("<4cB6B8hb30f3BcB",
data))
# get the most frequent gpsstate of the last 10 secs
# this avoids error messages for singular one sec state changes
self.gpsstatelst.append(gpsstat)
self.gpsstatelst = self.gpsstatelst[-10:]
self.gpsstate1 = max(set(self.gpsstatelst), key=self.gpsstatelst.count)
if not self.gpsstate1 == self.gpsstate2:
log.msg('LEMI - Protocol: GPSSTATE changed to %s .' % gpsstat)
self.gpsstate2 = self.gpsstate1
try:
# Analyze time difference between GPS and NTP
timelist = sorted([gpstime, currenttime])
timediff = timelist[1] - timelist[0]
delta = timediff.total_seconds()
if not delta in [0.0, np.nan, None]:
self.delaylist.append(timediff.total_seconds())
self.delaylist = self.delaylist[-1000:]
if len(self.delaylist) > 100:
try:
self.timedelay = np.median(np.asarray(self.delaylist))
except:
self.timedelay = 0.0
if delta - self.ntp_gps_offset > self.timethreshold:
self.errorcnt['time'] += 1
if self.errorcnt.get('time') < 2:
log.msg(
" -> {} protocol: large time difference observed for {}: {} sec"
.format(self.sensordict.get('protocol'), sensorid,
secdiff))
else:
self.errorcnt['time'] = 0
except:
pass
### NNOOOO, always send GPS time - but provide median time delays with the dictionary
### check LEMI Records whether secondary time (NTP) is readable and extractable
# Create a dataarray
linelst = []
for idx, el in enumerate(xarray):
datalst = []
tincr = idx / 10.
timear = gpstime + timedelta(seconds=tincr)
gps_time = datetime.strftime(timear, "%Y-%m-%d %H:%M:%S.%f")
datalst = acs.timeToArray(gps_time)
datalst.append(xarray[idx] / 1000.)
datalst.append(yarray[idx] / 1000.)
datalst.append(zarray[idx] / 1000.)
datalst.append(temp_sensor * 100)
datalst.append(temp_el * 100)
datalst.append(vdd * 10)
linestr = ','.join(list(map(str, datalst)))
linelst.append(linestr)
dataarray = ';'.join(linelst)
#print ("Processing successful")
return dataarray, headforsend
def processPos1Data(self, data):
"""Convert raw ADC counts into SI units as per datasheets"""
if len(data) != 44:
log.err('POS1 - Protocol: Unable to parse data of length %i' % len(data))
currenttime = datetime.utcnow()
outdate = datetime.strftime(currenttime, "%Y-%m-%d")
actualtime = datetime.strftime(currenttime, "%Y-%m-%dT%H:%M:%S.%f")
outtime = datetime.strftime(currenttime, "%H:%M:%S")
timestamp = datetime.strftime(currenttime, "%Y-%m-%d %H:%M:%S.%f")
filename = outdate
sensorid = self.sensor
packcode = '6hLLLh6hL'
header = "# MagPyBin %s %s %s %s %s %s %d" % (self.sensor, '[f,df,var1,sectime]', '[f,df,var1,GPStime]', '[nT,nT,none,none]', '[1000,1000,1,1]', packcode, struct.calcsize('<'+packcode))
try:
# Extract data
data_array = data.split()
intensity = float(data_array[0])/1000.
sigma_int = float(data_array[2])/1000.
err_code = int(data_array[3].strip('[').strip(']'))
dataelements = datetime.strptime(data_array[4],"%m-%d-%y")
newdate = datetime.strftime(dataelements,"%Y-%m-%d")
gps_time = newdate + ' ' + str(data_array[5])[:11]
except:
log.err('POS1 - Protocol: Data formatting error.')
intensity = 0.0
sigma_int = 0.0
err_code = 0.0
try:
# Analyze time difference between POS1 internal time and utc from PC
# Please note that the time difference between POS1-GPS (data recorded)
# and NTP (data received at PC) can be very large
# for our POS1 it is 6.2 seconds
gpstime = datetime.strptime(gps_time, "%Y-%m-%d %H:%M:%S.%f")
timelist = sorted([gpstime,currenttime])
timediff = timelist[1]-timelist[0]
delta = timediff.total_seconds()
if not delta in [0.0, np.nan, None]:
self.delaylist.append(delta)
self.delaylist = self.delaylist[-1000:]
if len(self.delaylist) > 100:
try:
self.timedelay = np.median(np.asarray(self.delaylist))
except:
self.timedelay = 0.0
if delta-self.ntp_gps_offset > self.timethreshold:
self.errorcnt['time'] +=1
if self.errorcnt.get('time') < 2:
log.msg("{} protocol: large time difference observed for {}: {} sec".format(self.sensordict.get('protocol'), sensorid, secdiff))
else:
self.errorcnt['time'] = 0
except:
pass
if self.sensordict.get('ptime','') in ['NTP','ntp']:
secondtime = gps_time
maintime = timestamp
else:
maintime = gps_time
secondtime = timestamp
try:
# extract time data
datearray = acs.timeToArray(maintime)
sectarray = acs.timeToArray(secondtime)
try:
datearray.append(int(intensity*1000))
datearray.append(int(sigma_int*1000))
datearray.append(err_code)
datearray.extend(sectarray)
data_bin = struct.pack('<'+packcode,datearray[0],datearray[1],datearray[2],datearray[3],datearray[4],datearray[5],datearray[6],datearray[7],datearray[8],datearray[9],datearray[10],datearray[11],datearray[12],datearray[13],datearray[14],datearray[15],datearray[16])
except:
log.msg('POS1 - Protocol: Error while packing binary data')
pass
if not self.confdict.get('bufferdirectory','') == '':
acs.dataToFile(self.confdict.get('bufferdirectory'), sensorid, filename, data_bin, header)
except:
log.msg('POS1 - Protocol: Error with binary save routine')
pass
return ','.join(list(map(str,datearray))), header
def processData(self, data):
""" GSM90 data """
currenttime = datetime.utcnow()
outdate = datetime.strftime(currenttime, "%Y-%m-%d")
actualtime = datetime.strftime(currenttime, "%Y-%m-%dT%H:%M:%S.%f")
outtime = datetime.strftime(currenttime, "%H:%M:%S")
timestamp = datetime.strftime(currenttime, "%Y-%m-%d %H:%M:%S.%f")
filename = outdate
sensorid = self.sensor
packcode = '6hLLL6hL'
header = "# MagPyBin %s %s %s %s %s %s %d" % (
self.sensor, '[f,var1,sectime]', '[f,errorcode,internaltime]',
'[nT,none,none]', '[1000,1,1]', packcode,
struct.calcsize('<' + packcode))
try:
# Extract data
# old data looks like 04-22-2015 142244 48464.53 99
data_array = data
if len(data) == 4:
intensity = float(data[2])
err_code = int(data[3])
try:
try:
internal_t = datetime.strptime(data[0] + 'T' + data[1],
"%m-%d-%YT%H%M%S.%f")
except:
internal_t = datetime.strptime(data[0] + 'T' + data[1],
"%m-%d-%YT%H%M%S")
internal_time = datetime.strftime(internal_t,
"%Y-%m-%d %H:%M:%S.%f")
except:
internal_time = timestamp #datetime.strftime(datetime.utcnow(), "%Y-%m-%d %H:%M:%S.%f")
#print internal_time
elif len(data) == 3: # GSM v7.0
intensity = float(data[1])
err_code = int(data[2])
try:
internal_t = datetime.strptime(outdate + 'T' + data[0],
"%Y-%m-%dT%H%M%S.%f")
internal_time = datetime.strftime(internal_t,
"%Y-%m-%d %H:%M:%S.%f")
except:
internal_time = timestamp #datetime.strftime(datetime.utcnow(), "%Y-%m-%d %H:%M:%S.%f")
else:
err_code = 0
intensity = float(data[0])
internal_time = timestamp #datetime.strftime(datetime.utcnow(), "%Y-%m-%d %H:%M:%S")
except:
log.err('{} protocol: Data formatting error. Data looks like: {}'.
format(self.sensordict.get('protocol'), data))
try:
# Analyze time difference between GSM internal time and utc from PC
timelist = sorted([internal_t, currenttime])
timediff = timelist[1] - timelist[0]
#secdiff = timediff.seconds + timediff.microseconds/1E6
#timethreshold = 3
delta = timediff.total_seconds()
if not delta in [0.0, np.nan, None]:
self.delaylist.append(timediff.total_seconds())
self.delaylist = self.delaylist[-1000:]
if len(self.delaylist) > 100:
try:
self.timedelay = np.median(np.asarray(self.delaylist))
except:
self.timedelay = 0.0
#if secdiff > timethreshold:
if delta > self.timethreshold:
self.errorcnt['time'] += 1
if self.errorcnt.get('time') < 2:
log.msg(
"{} protocol: large time difference observed for {}: {} sec"
.format(self.sensordict.get('protocol'), sensorid,
secdiff))
else:
self.errorcnt['time'] = 0
except:
pass
if self.sensordict.get('ptime', '') in ['NTP', 'ntp']:
secondtime = internal_time
maintime = timestamp
else:
maintime = internal_time
secondtime = timestamp
try:
## GSM90 does not provide any info on whether the GPS reading is OK or not
# extract time data
datearray = acs.timeToArray(maintime)
try:
datearray.append(int(intensity * 1000.))
datearray.append(err_code)
#print timestamp, internal_time
internalarray = acs.timeToArray(secondtime)
datearray.extend(internalarray)
data_bin = struct.pack('<' + packcode, *datearray)
except:
log.msg('{} protocol: Error while packing binary data'.format(
self.sensordict.get('protocol')))
if not self.confdict.get('bufferdirectory', '') == '':
acs.dataToFile(self.confdict.get('bufferdirectory'), sensorid,
filename, data_bin, header)
except:
log.msg('{} protocol: Error with binary save routine'.format(
self.sensordict.get('protocol')))
return ','.join(list(map(str, datearray))), header