def processData(self, data):
"""Process Environment data """
currenttime = datetime.utcnow()
outdate = datetime.strftime(currenttime, "%Y-%m-%d")
filename = outdate
actualtime = datetime.strftime(currenttime, "%Y-%m-%dT%H:%M:%S.%f")
outtime = datetime.strftime(currenttime, "%H:%M:%S")
timestamp = datetime.strftime(currenttime, "%Y-%m-%d %H:%M:%S.%f")
packcode = '6hLl'
sensorid = self.sensor
header = "# MagPyBin %s %s %s %s %s %s %d" % (
sensorid, '[x]', '[RN]', '[random]', '[1000]', packcode,
struct.calcsize('<' + packcode))
try:
datearray = acs.timeToArray(timestamp)
datearray.append(int(data * 1000))
data_bin = struct.pack('<' + packcode,
*datearray) #use little endian byte order
except:
log.msg('Error while packing binary data')
pass
if not self.confdict.get('bufferdirectory', '') == '':
acs.dataToFile(self.confdict.get('bufferdirectory'), sensorid,
filename, data_bin, header)
return ','.join(list(map(str, datearray))), header
def processArduinoData(self, sensorid, meta, data):
"""Convert raw ADC counts into SI units as per datasheets"""
currenttime = datetime.utcnow()
outdate = datetime.strftime(currenttime, "%Y-%m-%d")
actualtime = datetime.strftime(currenttime, "%Y-%m-%dT%H:%M:%S.%f")
outtime = datetime.strftime(currenttime, "%H:%M:%S")
timestamp = datetime.strftime(currenttime, "%Y-%m-%d %H:%M:%S.%f")
filename = outdate
datearray = acs.timeToArray(timestamp)
packcode = '6hL'
#sensorid = self.sensordict.get(idnum)
#events = self.eventdict.get(idnum).replace('evt','').split(',')[3:-1]
values = []
multiplier = []
for dat in data:
try:
values.append(float(dat))
datearray.append(int(float(dat) * 10000))
packcode = packcode + 'l'
multiplier.append(10000)
except:
log.msg(
'{} protocol: Error while appending data to file (non-float?): {}'
.format(self.sensordict.get('protocol'), dat))
try:
data_bin = struct.pack('<' + packcode, *datearray) #little endian
except:
log.msg('{} protocol: Error while packing binary data'.format(
self.sensordict.get('protocol')))
pass
key = '[' + str(meta.get('SensorKeys')).replace("'", "").strip() + ']'
ele = '[' + str(meta.get('SensorElements')).replace("'",
"").strip() + ']'
unit = '[' + str(meta.get('SensorUnits')).replace("'",
"").strip() + ']'
multplier = str(multiplier).replace(" ", "")
# Correct some common old problem
unit = unit.replace('deg C', 'degC')
header = "# MagPyBin %s %s %s %s %s %s %d" % (
sensorid, key, ele, unit, multplier, packcode,
struct.calcsize('<' + packcode))
if not self.confdict.get('bufferdirectory', '') == '':
acs.dataToFile(self.confdict.get('bufferdirectory'), sensorid,
filename, data_bin, header)
return ','.join(list(map(str, datearray))), header
def processData(self, data):
"""Convert raw ADC counts into SI units as per datasheets"""
currenttime = datetime.utcnow()
# Correction for ms time to work with databank:
currenttime_ms = currenttime.microsecond / 1000000.
ms_rounded = round(float(currenttime_ms), 3)
if not ms_rounded >= 1.0:
currenttime = currenttime.replace(microsecond=int(ms_rounded *
1000000.))
else:
currenttime = currenttime.replace(microsecond=0) + timedelta(
seconds=1.0)
filename = datetime.strftime(currenttime, "%Y-%m-%d")
actualtime = datetime.strftime(currenttime, "%Y-%m-%dT%H:%M:%S.%f")
lastActualtime = currenttime
outtime = datetime.strftime(currenttime, "%H:%M:%S")
timestamp = datetime.strftime(currenttime, "%Y-%m-%d %H:%M:%S.%f")
sensorid = self.sensor
packcode = '6hLL'
header = "# MagPyBin %s %s %s %s %s %s %d" % (
self.sensor, '[f]', '[f]', '[nT]', '[1000]', packcode,
struct.calcsize('<' + packcode))
try:
intval = data[1].split(',')
value = float(intval[0].strip())
if 10000 < value < 100000:
intensity = value
else:
intensity = 88888.0
except ValueError:
log.err("CS - Protocol: Not a number. Instead found:", data[0])
intensity = 88888.0
try:
datearray = acs.timeToArray(timestamp)
datearray.append(int(intensity * 1000))
data_bin = struct.pack('<' + packcode, *datearray)
except:
log.msg('Error while packing binary data')
if not self.confdict.get('bufferdirectory', '') == '':
acs.dataToFile(self.confdict.get('bufferdirectory'), sensorid,
filename, data_bin, header)
return ','.join(list(map(str, datearray))), header
def processOwData(self, sensorid, datadict):
"""Process OW data """
currenttime = datetime.utcnow()
outdate = datetime.strftime(currenttime, "%Y-%m-%d")
filename = outdate
actualtime = datetime.strftime(currenttime, "%Y-%m-%dT%H:%M:%S.%f")
outtime = datetime.strftime(currenttime, "%H:%M:%S")
timestamp = datetime.strftime(currenttime, "%Y-%m-%d %H:%M:%S.%f")
packcode = '6hL' + 'l' * len(datadict)
multplier = str([1000] * len(datadict)).replace(' ', '')
if sensorid.startswith('DS18'):
key = '[t1]'
ele = '[T]'
unit = '[degC]'
elif sensorid.startswith('DS2438'):
#'temperature','VAD','VDD','humidity','vis'
key = '[t1,var1,var2,var3,var4]'
ele = '[T,RH,VDD,VAD,VIS]'
unit = '[degC,per,V,V,V,V]'
header = "# MagPyBin %s %s %s %s %s %s %d" % (
sensorid, key, ele, unit, multplier, packcode,
struct.calcsize('<' + packcode))
data_bin = None
datearray = ''
try:
datearray = acs.timeToArray(timestamp)
paralst = typedef.get(sensorid.split('_')[0])
for para in paralst:
if para in datadict:
datearray.append(int(float(datadict[para]) * 1000))
data_bin = struct.pack('<' + packcode,
*datearray) # little endian
except:
log.msg('Error while packing binary data')
if not self.confdict.get('bufferdirectory', '') == '' and data_bin:
acs.dataToFile(self.confdict.get('bufferdirectory'), sensorid,
filename, data_bin, header)
#print ("Sending", ','.join(list(map(str,datearray))), header)
return ','.join(list(map(str, datearray))), header
def processData(self, data):
"""Process Environment data """
currenttime = datetime.utcnow()
outdate = datetime.strftime(currenttime, "%Y-%m-%d")
filename = outdate
actualtime = datetime.strftime(currenttime, "%Y-%m-%dT%H:%M:%S.%f")
outtime = datetime.strftime(currenttime, "%H:%M:%S")
timestamp = datetime.strftime(currenttime, "%Y-%m-%d %H:%M:%S.%f")
packcode = '6hLllL'
sensorid = self.sensor
header = "# MagPyBin %s %s %s %s %s %s %d" % (sensorid, '[t1,t2,var1]', '[T,DewPoint,RH]', '[degC,degC,per]', '[1000,1000,1000]', packcode, struct.calcsize('<'+packcode))
valrh = re.findall(r'\d+',data[0])
if len(valrh) > 1:
temp = float(valrh[0] + '.' + valrh[1])
else:
temp = float(valrh[0])
valrh = re.findall(r'\d+',data[1])
if len(valrh) > 1:
rh = float(valrh[0] + '.' + valrh[1])
else:
rh = float(valrh[0])
valrh = re.findall(r'\d+',data[2])
if len(valrh) > 1:
dew = float(valrh[0] + '.' + valrh[1])
else:
dew = float(valrh[0])
try:
datearray = acs.timeToArray(timestamp)
datearray.append(int(temp*1000))
datearray.append(int(dew*1000))
datearray.append(int(rh*1000))
data_bin = struct.pack('<'+packcode,*datearray) #use little endian byte order
except:
log.msg('Error while packing binary data')
pass
if not self.confdict.get('bufferdirectory','') == '':
acs.dataToFile(self.confdict.get('bufferdirectory'), sensorid, filename, data_bin, header)
return ','.join(list(map(str,datearray))), header
def processLemiData(self, data):
"""Convert raw ADC counts into SI units as per datasheets"""
if len(data) != 153:
log.err('LEMI - Protocol: Unable to parse data of length %i' %
len(data))
#print ("Processing data ...")
""" TIMESHIFT between serial output (and thus NTP time) and GPS timestamp """
currenttime = datetime.utcnow()
date = datetime.strftime(currenttime, "%Y-%m-%d")
timestamp = datetime.strftime(currenttime, "%Y-%m-%d %H:%M:%S.%f")
outtime = datetime.strftime(currenttime, "%H:%M:%S")
datearray = acs.timeToArray(timestamp)
date_bin = struct.pack(
'<6hL', datearray[0] - 2000, datearray[1], datearray[2],
datearray[3], datearray[4], datearray[5], datearray[6]
) ## Added "<" to pack code to get correct length in new machines
# define pathname for local file storage
# (default dir plus hostname plus sensor plus year) and create if not existing
path = os.path.join(self.confdict.get('bufferdirectory'), self.sensor)
if not os.path.exists(path):
os.makedirs(path)
packcode = "<4cb6B8hb30f3BcBcc5hL"
header = "LemiBin %s %s %s %s %s %s %d\n" % (
self.sensor, '[x,y,z,t1,t2]', '[X,Y,Z,T_sensor,T_elec]',
'[nT,nT,nT,deg_C,deg_C]', '[0.001,0.001,0.001,100,100]', packcode,
struct.calcsize(packcode))
sendpackcode = '6hLffflll'
#headforsend = "# MagPyBin {} {} {} {} {} {} {}".format(self.sensor, '[x,y,z,t1,t2,var2,str1]', '[X,Y,Z,T_sensor,T_elec,VDD,GPS]', '[nT,nT,nT,deg_C,deg_C,V,Status]', '[0.001,0.001,0.001,100,100,10]', sendpackcode, struct.calcsize('<'+sendpackcode))
headforsend = "# MagPyBin {} {} {} {} {} {} {}".format(
self.sensor, '[x,y,z,t1,t2,var2]', '[X,Y,Z,T_sensor,T_elec,VDD]',
'[nT,nT,nT,deg_C,deg_C,V]', '[0.001,0.001,0.001,100,100,10]',
sendpackcode, struct.calcsize('<' + sendpackcode))
# save binary raw data to buffer file ### please note that this file always contains GPS readings
lemipath = os.path.join(path, self.sensor + '_' + date + ".bin")
if not os.path.exists(lemipath):
with open(lemipath, "ab") as myfile:
myfile.write(header)
try:
with open(lemipath, "ab") as myfile:
myfile.write(data + date_bin)
pass
except:
log.err('LEMI - Protocol: Could not write data to file.')
# unpack data and extract time and first field values
# This data is streamed via mqtt
try:
data_array = struct.unpack("<4cB6B8hb30f3BcB", data)
except:
log.err("LEMI - Protocol: Bit error while reading.")
try:
newtime = []
biasx = float(data_array[16]) / 400.
biasy = float(data_array[17]) / 400.
biasz = float(data_array[18]) / 400.
x = (data_array[20]) * 1000.
xarray = [elem * 1000. for elem in data_array[20:50:3]]
y = (data_array[21]) * 1000.
yarray = [elem * 1000. for elem in data_array[21:50:3]]
z = (data_array[22]) * 1000.
zarray = [elem * 1000. for elem in data_array[22:50:3]]
temp_sensor = data_array[11] / 100.
temp_el = data_array[12] / 100.
vdd = float(data_array[52]) / 10.
gpsstat = data_array[53]
gpstime = datetime(
2000 + self.h2d(data_array[5]), self.h2d(data_array[6]),
self.h2d(data_array[7]), self.h2d(data_array[8]),
self.h2d(data_array[9]), self.h2d(
data_array[10])) - timedelta(microseconds=300000)
#gps_time = datetime.strftime(gps_array, "%Y-%m-%d %H:%M:%S")
self.compensation[0] = biasx
self.compensation[1] = biasy
self.compensation[2] = biasz
except:
log.err("LEMI - Protocol: Number conversion error.")
#print ("HERE2", packcode, struct.calcsize(packcode))
processerror = False
if not gpsstat in ['P', 'A']:
print(" ERROR in BINDATA:", struct.unpack("<4cB6B8hb30f3BcB",
data))
print(" Rawdata looks like:", data)
self.buffererrorcnt += 1
processerror = True
if self.buffererrorcnt == 10:
self.initiateRestart()
# get the most frequent gpsstate of the last 10 min
# this avoids error messages for singular one sec state changes
self.gpsstatelst.append(gpsstat)
self.gpsstatelst = self.gpsstatelst[-600:]
self.gpsstate1 = max(set(self.gpsstatelst), key=self.gpsstatelst.count)
if not self.gpsstate1 == self.gpsstate2:
log.msg('LEMI - Protocol: GPSSTATE changed to %s .' % gpsstat)
self.gpsstate2 = self.gpsstate1
try:
# Analyze time difference between GPS and NTP
timelist = sorted([gpstime, currenttime])
timediff = timelist[1] - timelist[0]
delta = timediff.total_seconds()
if not delta in [0.0, np.nan, None]:
self.delaylist.append(timediff.total_seconds())
self.delaylist = self.delaylist[-1000:]
if len(self.delaylist) > 100:
try:
self.timedelay = np.median(np.asarray(self.delaylist))
except:
self.timedelay = 0.0
if delta - self.ntp_gps_offset > self.timethreshold:
self.errorcnt['time'] += 1
if self.errorcnt.get('time') < 2:
log.msg(
" -> {} protocol: large time difference observed for {}: {} sec"
.format(self.sensordict.get('protocol'), sensorid,
secdiff))
else:
self.errorcnt['time'] = 0
except:
pass
### NNOOOO, always send GPS time - but provide median time delays with the dictionary
### check LEMI Records whether secondary time (NTP) is readable and extractable
# Create a dataarray
linelst = []
for idx, el in enumerate(xarray):
datalst = []
tincr = idx / 10.
timear = gpstime + timedelta(seconds=tincr)
gps_time = datetime.strftime(timear.replace(tzinfo=None),
"%Y-%m-%d %H:%M:%S.%f")
datalst = acs.timeToArray(gps_time)
datalst.append(xarray[idx] / 1000.)
datalst.append(yarray[idx] / 1000.)
datalst.append(zarray[idx] / 1000.)
datalst.append(int(temp_sensor * 100))
datalst.append(int(temp_el * 100))
datalst.append(int(vdd * 10))
### TODO Add GPS and secondary time to this list
#datalst.append(gpsstat)
#current_time = datetime.strftime(currenttime.replace(tzinfo=None), "%Y-%m-%d %H:%M:%S.%f")
#datalst.extend(current_time)
linestr = ','.join(list(map(str, datalst)))
linelst.append(linestr)
dataarray = ';'.join(linelst)
if processerror:
print("Processing unsuccessful")
dataarray = ''
return dataarray, headforsend
def processData(self, data):
""" GP20S3 data """
"""
Data looks like--- (with GPS lines every minute):
-- vertical sensor - Old software
3,3,12.00 111 field1 field2 field3
3,3,12.00 111 field1 field2 field3
GPS 16.00 111 field1 field2 field3
-- horizontal sensor - New software
time 111 field1 field2 field3 (every sec or faster)
$$$ (every hour, preceeds status line)
10071506 A 13 250 492 496 329 150 1023 39 39 39 30 29 30 YYYyyyEEENNN 148 149 117 (every hour)
time 111 field1 field2 field3 (every sec or faster)
time 111 field1 field2 field3 (every sec or faster)
"""
currenttime = datetime.utcnow()
outdate = datetime.strftime(currenttime, "%Y-%m-%d")
actualtime = datetime.strftime(currenttime, "%Y-%m-%dT%H:%M:%S.%f")
outtime = datetime.strftime(currenttime, "%H:%M:%S")
timestamp = datetime.strftime(currenttime, "%Y-%m-%d %H:%M:%S.%f")
filename = outdate
sensorid = self.sensor
headerlinecoming = False
datearray = []
headarray = []
statusname = "Status_123_0001"
sensororientation = self.sensor.split('_')[0].replace(self.sensordict.get('protocol'),'')
if len(sensororientation) > 1:
sens1 = sensororientation
sens2 = sensororientation[0]
sens3 = sensororientation[1]
else:
sens1 = 'TA'
sens2 = 'B'
sens3 = 'TB'
celem = '[{},{},{},{}{},{}{},{}{},None]'.format(sens1,sens2,sens3, sens3,sens1, sens3,sens2, sens2,sens1)
packcode = '6hLQQQqqq6hL'
header = "# MagPyBin %s %s %s %s %s %s %d" % (self.sensor, '[x,y,z,dx,dy,dz,sectime]', celem, '[pT,pT,pT,pT,pT,pT,None]', '[1000,1000,1000,1000,1000,1000,1]', packcode, struct.calcsize('<'+packcode))
try:
# Extract data
data_array = data
if len(data_array) == 5:
intensity1 = float(data_array[2])
intensity2 = float(data_array[3])
intensity3 = float(data_array[4])
grad1 = intensity3-intensity1
grad2 = intensity3-intensity2
grad3 = intensity2-intensity1
try:
gpstime = float(data[0]) # will fail for old dataformat -> NTP
if gpstime > 235900.0: # use date of last day if gpstime > 235900 to prevent next day date for 235959 gps when pctime already is on next day
cdate = dateprev
else:
cdate = outdate
dateprev = outdate
try:
internal_t = datetime.strptime(cdate+'T'+data[0], "%Y-%m-%dT%H%M%S.%f")
except:
internal_t = datetime.strptime(cdate+'T'+data[0], "%Y-%m-%dT%H%M%S")
internal_time = datetime.strftime(internal_t, "%Y-%m-%d %H:%M:%S.%f")
except:
internal_time = timestamp #datetime.strftime(datetime.utcnow(), "%Y-%m-%d %H:%M:%S.%f")
elif len(data_array) == 19:
"""
10071506 A 13 250 492 496 329 150 1023 39 39 39 30 29 30 YYYyyyEEENNN 148 149 117
<GPS> day/month/year/hour A - locked, V unlocked
<13> Console outside air temperature (13C)
<250> Battery voltage (25.0V)
<492> +5V supply voltage (4.92V)
<496> -5V supply voltage (-4.96)
<3.3V> +3.3V supply voltage (3.3V)
<15.0> silver box power supply (15.0V)
<1023> OCXO internal trimpot adjustment level, automatically adjusted via GPS
<39> Sensor 1 temperature in C
<39> Sensor 2 temperature in C
<39> Sensor 3 temperature in C
<30> Light current sensor 1 (3.0uA)
<29> Light current sensor 2 (2.9uA)
<30> Light current sensor 3 (3.0uA)
<YYY> Sensor 1, sensor 2 sensor 3 lock status Y- locked, N - unlocked
<yyy> Sensor 1 heater status, sensor 2 heater status, sensor 3 heater status y-on, n-off
<EEE> Sensor 1 heater, sensor 2 heater, sensor 3 heater E-enabled, D-disabled (used for over heat protection)
<NNN> RF sensor 1, RF sensor 2, RF sensor 3, N -on, F-off
<148> Sensor 1 RF dc voltage (14.8V)
<149> Sensor 2 RF dc voltage (14.9V)
<117> Sensor 3 RF dc voltage (11.7V)
"""
headerlinecoming = True
try:
gpstime = str(data_array[0])
internal_t = datetime.strptime(gpstime, "%d%m%y%H")
gpstimestamp = datetime.strftime(internal_t, "%Y-%m-%d %H:%M:%S.%f")
except:
gpstimestamp = timestamp
internal_time = gpstimestamp
gpstatus = data_array[1] # str1
telec = int(data_array[2]) # t2
Vbat = float(data_array[3])/10. # f
Vsup1 = float(data_array[4])/100. # var4
Vsup2 = float(data_array[5])/100. # var5
Vlow = float(data_array[6])/100. # t1
PowerSup = float(data_array[7])/10. # df
level = data_array[8] # str3
tsens1 = int(data_array[9]) # x
tsens2 = int(data_array[10]) # y
tsens3 = int(data_array[11]) # z
lightcurrent1 = float(data_array[12])/10. # dx
lightcurrent2 = float(data_array[13])/10. # dy
lightcurrent3 = float(data_array[14])/10. # dz
statusstring = data_array[15] # str2
Vsens1 = float(data_array[16])/10. # var1
Vsens2 = float(data_array[17])/10. # var2
Vsens3 = float(data_array[18])/10. # var3
elif len(data_array) == 1 and data_array[0] == '$$$':
return "","",""
else:
log.msg('{} protocol: data line could not be interpreted: ({}) of length {}'.format(self.sensordict.get('protocol'),data, len(data_array)))
except:
log.err('{} protocol: Data formatting error. Data looks like: {}'.format(self.sensordict.get('protocol'),data))
try:
# Analyze time difference between GSM internal time and utc from PC
timelist = sorted([internal_t,currenttime])
timediff = timelist[1]-timelist[0]
delta = timediff.total_seconds()
if not delta in [0.0, float('NAN'), None]:
self.delaylist.append(timediff.total_seconds())
if len(self.delaylist) > 600:
self.delaylist = self.delaylist[-600:]
if len(self.delaylist) > 100:
try:
self.timedelay = np.abs(np.median(np.asarray(self.delaylist)))
except:
self.timedelay = 0.0
if self.timedelay > self.timethreshold:
self.errorcnt['time'] +=1
if self.errorcnt.get('time') < 2:
log.msg("{} protocol: large time difference observed for {}: {} sec".format(self.sensordict.get('protocol'), sensorid, self.timedelay))
if self.errorcnt.get('time') > 1000:
self.errorcnt['time'] = 1000
else:
self.errorcnt['time'] = 0
except:
pass
if self.sensordict.get('ptime','') in ['NTP','ntp']:
secondtime = internal_time
maintime = timestamp
else:
maintime = internal_time
secondtime = timestamp
if not headerlinecoming:
try:
## GP20S3 provides info on whether the GPS reading is OK - use it
# extract time data
datearray = acs.timeToArray(maintime)
try:
datearray.append(int(intensity1*1000.))
datearray.append(int(intensity2*1000.))
datearray.append(int(intensity3*1000.))
datearray.append(int(grad1*1000.))
datearray.append(int(grad2*1000.))
datearray.append(int(grad3*1000.))
internalarray = acs.timeToArray(secondtime)
datearray.extend(internalarray)
data_bin = struct.pack('<'+packcode,*datearray)
except:
log.msg('{} protocol: Error while packing binary data'.format(self.sensordict.get('protocol')))
if not self.confdict.get('bufferdirectory','') == '':
acs.dataToFile(self.confdict.get('bufferdirectory'), sensorid, filename, data_bin, header)
except:
log.msg('{} protocol: Error with binary save routine'.format(self.sensordict.get('protocol')))
if headerlinecoming:
if self.debug:
print (" now writing header info")
headpackcode = '6hL15ls12s4s' #'6hLlllllllllllllllsss'
statusname = "Status_123_0001"
try:
# extract time data
headarray = acs.timeToArray(maintime)
try:
headarray.append(int(tsens1)) # x
headarray.append(int(tsens2)) # y
headarray.append(int(tsens3)) # z
headarray.append(int(Vbat*10.)) # f
headarray.append(int(Vlow*100.)) # t1
headarray.append(int(telec)) # t2
headarray.append(int(lightcurrent1*10.)) # dx
headarray.append(int(lightcurrent2*10.)) # dy
headarray.append(int(lightcurrent3*10.)) # dz
headarray.append(int(PowerSup*10.)) # df
headarray.append(int(Vsens1*10.)) # var1
headarray.append(int(Vsens2*10.)) # var2
headarray.append(int(Vsens3*10.)) # var3
headarray.append(int(Vsup1*100.)) # var4
headarray.append(int(Vsup2*100.)) # var5
headarray.append(gpstatus) # str1
headarray.append(statusstring) # str2
headarray.append(level) # str3
data_bin = struct.pack('<'+headpackcode,*headarray)
statuslst = self.sensor.split('_')
if self.debug:
print ("Headerdata has been packed")
if len(statuslst) == 3:
statusname = '_'.join([statuslst[0]+'status',statuslst[1],statuslst[2]])
headheader = "# MagPyBin %s %s %s %s %s %s %d" % (statusname, '[x,y,z,f,t1,t2,dx,dy,dz,df,var1,var2,var3,var4,var5,str1,str2,str3]', '[Ts1,Ts2,Ts3,Vbat,V3,Tel,L1,L2,L3,Vps,V1,V2,V3,V5p,V5n,GPSstat,Status,OCXO]', '[degC,degC,degC,V,V,degC,A,A,A,V,V,V,V,V,V,None,None,None]', '[1,1,1,10,100,1,10,10,10,10,10,10,10,100,100,1,1,1]', headpackcode, struct.calcsize('<'+headpackcode))
if self.debug:
print ("Header looks like: {} ".format(headheader))
print ("Writing to file: {}, {}, {}".format(statusname,filename,headheader))
if not self.confdict.get('bufferdirectory','') == '':
acs.dataToFile(self.confdict.get('bufferdirectory'), statusname, filename, data_bin, headheader)
except:
log.msg('GSMP20 - Protocol: Error while packing binary data')
except:
pass
if len(datearray) > 0:
topic = self.confdict.get('station') + '/' + self.sensordict.get('sensorid')
return ','.join(list(map(str,datearray))), header, topic
elif len(headarray) > 0:
topic = self.confdict.get('station') + '/' + statusname
return ','.join(list(map(str,headarray))), headheader, topic
def processData(self, data):
""" GSM90 data """
currenttime = datetime.utcnow()
outdate = datetime.strftime(currenttime, "%Y-%m-%d")
actualtime = datetime.strftime(currenttime, "%Y-%m-%dT%H:%M:%S.%f")
outtime = datetime.strftime(currenttime, "%H:%M:%S")
timestamp = datetime.strftime(currenttime, "%Y-%m-%d %H:%M:%S.%f")
filename = outdate
sensorid = self.sensor
packcode = '6hLLL6hL'
header = "# MagPyBin %s %s %s %s %s %s %d" % (self.sensor, '[f,var1,sectime]', '[f,errorcode,internaltime]', '[nT,none,none]', '[1000,1,1]', packcode, struct.calcsize('<'+packcode))
try:
# Extract data
# old data looks like 04-22-2015 142244 48464.53 99
data_array = data
if len(data) == 4:
intensity = float(data[2])
err_code = int(data[3])
try:
try:
internal_t = datetime.strptime(data[0]+'T'+data[1], "%m-%d-%YT%H%M%S.%f")
except:
internal_t = datetime.strptime(data[0]+'T'+data[1], "%m-%d-%YT%H%M%S")
internal_time = datetime.strftime(internal_t, "%Y-%m-%d %H:%M:%S.%f")
except:
internal_time = timestamp #datetime.strftime(datetime.utcnow(), "%Y-%m-%d %H:%M:%S.%f")
#print internal_time
elif len(data) == 3: # GSM v7.0
intensity = float(data[1])
err_code = int(data[2])
try:
internal_t = datetime.strptime(outdate+'T'+data[0], "%Y-%m-%dT%H%M%S.%f")
internal_time = datetime.strftime(internal_t, "%Y-%m-%d %H:%M:%S.%f")
except:
internal_time = timestamp #datetime.strftime(datetime.utcnow(), "%Y-%m-%d %H:%M:%S.%f")
else:
err_code = 0
intensity = float(data[0])
internal_time = timestamp #datetime.strftime(datetime.utcnow(), "%Y-%m-%d %H:%M:%S")
except:
log.err('{} protocol: Data formatting error. Data looks like: {}'.format(self.sensordict.get('protocol'),data))
try:
# Analyze time difference between GSM internal time and utc from PC
timelist = sorted([internal_t,currenttime])
timediff = timelist[1]-timelist[0]
#secdiff = timediff.seconds + timediff.microseconds/1E6
#timethreshold = 3
delta = timediff.total_seconds()
if not delta in [0.0, np.nan, None]:
self.delaylist.append(timediff.total_seconds())
self.delaylist = self.delaylist[-1000:]
if len(self.delaylist) > 100:
try:
self.timedelay = np.median(np.asarray(self.delaylist))
except:
self.timedelay = 0.0
#if secdiff > timethreshold:
if delta > self.timethreshold:
self.errorcnt['time'] +=1
if self.errorcnt.get('time') < 2:
log.msg("{} protocol: large time difference observed for {}: {} sec".format(self.sensordict.get('protocol'), sensorid, secdiff))
else:
self.errorcnt['time'] = 0
except:
pass
if self.sensordict.get('ptime','') in ['NTP','ntp']:
secondtime = internal_time
maintime = timestamp
else:
maintime = internal_time
secondtime = timestamp
try:
## GSM90 does not provide any info on whether the GPS reading is OK or not
# extract time data
datearray = acs.timeToArray(maintime)
try:
datearray.append(int(intensity*1000.))
datearray.append(err_code)
#print timestamp, internal_time
internalarray = acs.timeToArray(secondtime)
datearray.extend(internalarray)
data_bin = struct.pack('<'+packcode,*datearray)
except:
log.msg('{} protocol: Error while packing binary data'.format(self.sensordict.get('protocol')))
if not self.confdict.get('bufferdirectory','') == '':
acs.dataToFile(self.confdict.get('bufferdirectory'), sensorid, filename, data_bin, header)
except:
log.msg('{} protocol: Error with binary save routine'.format(self.sensordict.get('protocol')))
return ','.join(list(map(str,datearray))), header
def processData(self, data):
currenttime = datetime.utcnow()
date = datetime.strftime(currenttime, "%Y-%m-%d")
actualtime = datetime.strftime(currenttime, "%Y-%m-%dT%H:%M:%S.%f")
outtime = datetime.strftime(currenttime, "%H:%M:%S")
filename = date
timestamp = datetime.strftime(currenttime, "%Y-%m-%d %H:%M:%S.%f")
intensity = 88888.8
typ = "none"
dontsavedata = False
packcode = '6hLLl'
header = "# MagPyBin %s %s %s %s %s %s %d" % (
self.sensor, '[f,var1]', '[f,err]', '[nT,none]', '[1000,1000]',
packcode, struct.calcsize('<' + packcode))
try:
# Extract data
data_array = data
if len(data_array) == 2:
typ = "oldbase"
elif len(data_array) == 3:
typ = "valid"
# add other types here
except:
log.err(
'GSM19 - Protocol: Output format not supported - use either base, ... or mobile'
)
# Extracting the data from the station
# Extrat time info and use as primary if GPS is on (in this case PC time is secondary)
# PC is primary when a GPS is not connected
if typ == "valid" or typ == "oldbase": # Comprises Mobile and Base Station mode with single sensor and no GPS
intensity = float(data_array[1])
try:
systemtime = datetime.strptime(date + "-" + data_array[0],
"%Y-%m-%d-%H%M%S.%f")
except:
# This exception happens for old GSM19 because time is
# provided e.g. as 410356 instead of 170356 for 17:03:56 (Thursday)
# e.g 570301.0 instead of 09:03:01 (Friday)
try:
hournum = int(data_array[0][:-6])
rest = data_array[0][-6:]
factor = np.floor(hournum /
24.) # factor = days since starting
hour = int(hournum - factor * 24.)
systemtime = datetime.strptime(
date + "-" + str(hour) + rest, "%Y-%m-%d-%H%M%S.%f")
#print ("Got oldbase systemtime")
except:
systemtime = currenttime
self.timesource = 'NTP'
if len(data_array) == 2:
typ = "base"
errorcode = 99
elif len(data_array[2]) == 3:
typ = "base"
errorcode = int(data_array[2])
else:
typ = "gradient"
gradient = float(data_array[2])
elif typ == "none":
dontsavedata = True
pass
gpstime = datetime.strftime(systemtime, "%Y-%m-%d %H:%M:%S.%f")
try:
# Analyze time difference between GSM internal time and utc from PC
timelist = sorted([systemtime, currenttime])
timediff = timelist[1] - timelist[0]
#secdiff = timediff.seconds + timediff.microseconds/1E6
delta = timediff.total_seconds()
if not delta in [0.0, np.nan, None]:
self.delaylist.append(timediff.total_seconds())
self.delaylist = self.delaylist[-1000:]
if len(self.delaylist) > 100:
try:
self.timedelay = np.median(np.asarray(self.delaylist))
except:
self.timedelay = 0.0
if delta > self.timethreshold:
self.errorcnt['time'] += 1
if self.errorcnt.get('time') < 2:
log.msg(
"{} protocol: large time difference observed for {}: {} sec"
.format(self.sensordict.get('protocol'), sensorid,
secdiff))
else:
self.errorcnt['time'] = 0
except:
pass
if self.sensordict.get('ptime', '') in ['NTP', 'ntp']:
secondtime = gpstime
maintime = timestamp
else:
maintime = gpstime
secondtime = timestamp
try:
if not typ == "none":
# extract time data
datearray = acs.timeToArray(maintime)
try:
datearray.append(int(intensity * 1000.))
if typ == 'base':
datearray.append(int(errorcode * 1000.))
else:
datearray.append(int(gradient * 1000.))
data_bin = struct.pack('<' + packcode, *datearray)
except:
log.msg(
'GSM19 - Protocol: Error while packing binary data')
pass
except:
log.msg('GSM19 - Protocol: Error with binary save routine')
pass
if not self.confdict.get('bufferdirectory', '') == '':
acs.dataToFile(self.confdict.get('bufferdirectory'), self.sensor,
filename, data_bin, header)
return ','.join(list(map(str, datearray))), header
def sendRequest(self):
"""
source:mysql:
Method to obtain data from table
"""
t1 = datetime.utcnow()
outdate = datetime.strftime(t1, "%Y-%m-%d")
filename = outdate
if self.debug:
log.msg(" -> DEBUG - Sending periodic request ...")
def getList(sql):
cursor = self.db.cursor()
try:
cursor.execute(sql)
except mysql.IntegrityError as message:
return message
except mysql.Error as message:
return message
except:
return 'dbgetlines: unkown error'
head = cursor.fetchall()
keys = list(np.transpose(np.asarray(head))[0])
return keys
# get self.sensorlist
# get last timestamps
# read all data for each sensor since last timestamp
# send that and store last timestamp
for index,sensdict in enumerate(self.sensorlist):
sensorid = sensdict.get('sensorid')
if self.debug:
log.msg(" -> DEBUG - dealing with sensor {}".format(sensorid))
# 1. Getting header
# -----------------
# load keys, elements and units
#header = "# MagPyBin %s %s %s %s %s %s %d" % (sensorid, key, ele, unit, multplier, packcode, struct.calcsize('<'+packcode))
dataid = sensorid+'_'+self.revision
keyssql = 'SHOW COLUMNS FROM %s' % (dataid)
keystab = getList(keyssql)
if 'time' in keystab:
keystab.remove('time')
if 'flag' in keystab:
keystab.remove('flag')
if 'typ' in keystab:
keystab.remove('typ')
if 'comment' in keystab:
keystab.remove('comment')
keys = ','.join(keystab)
if self.debug:
log.msg(" -> DEBUG - requesting header {}".format(sensorid))
sql1 = 'SELECT SensorElements FROM SENSORS WHERE SensorID LIKE "{}"'.format(sensorid)
sql2 = 'SELECT Sensorkeys FROM SENSORS WHERE SensorID LIKE "{}"'.format(sensorid)
sql3 = 'SELECT ColumnUnits FROM DATAINFO WHERE SensorID LIKE "{}"'.format(sensorid)
sql4 = 'SELECT ColumnContents FROM DATAINFO WHERE SensorID LIKE "{}"'.format(sensorid)
try:
elem = getList(sql1)[0].split(',')
except:
elem =[]
try:
keyssens = getList(sql2)[0].split(',')
except:
keyssens =[]
try:
unit = getList(sql3)[0].split(',')
except:
unit =[]
try:
cont = getList(sql4)[0].split(',')
except:
cont =[]
units, elems = [], []
for key in keystab:
try:
pos1 = keyssens.index(key)
ele = elem[pos1]
except:
ele = key
elems.append(ele)
try:
pos2 = cont.index(ele)
units.append(unit[pos2])
except:
units.append('None')
if self.debug:
log.msg(" -> DEBUG - creating head line {}".format(sensorid))
multplier = '['+','.join(map(str, [10000]*len(keystab)))+']'
packcode = '6HL'+''.join(['q']*len(keystab))
header = ("# MagPyBin {} {} {} {} {} {} {}".format(sensorid, '['+','.join(keystab)+']', '['+','.join(elems)+']', '['+','.join(units)+']', multplier, packcode, struct.calcsize('<'+packcode)))
# 2. Getting dict
sql = 'SELECT DataSamplingRate FROM DATAINFO WHERE SensorID LIKE "{}"'.format(sensorid)
sr = float(getList(sql)[0])
coverage = int(self.requestrate/sr)+120
# 3. Getting data
# get data and create typical message topic
# based on sampling rate and collection rate -> define coverage
li = sorted(mdb.dbselect(self.db, 'time,'+keys, dataid, expert='ORDER BY time DESC LIMIT {}'.format(int(coverage))))
if not self.lastt[index]:
self.lastt[index]=li[0][0]
# drop
newdat = False
newli = []
for elem in li:
if elem[0] == self.lastt[index]:
newdat = True
if newdat:
newli.append(elem)
if not len(newli) > 0:
# if last time not included in li then newli will be empty
# in this case just add the list
for elem in li:
newli.append(elem)
for dataline in newli:
timestamp = dataline[0]
data_bin = None
datearray = ''
try:
datearray = acs.timeToArray(timestamp)
for i,para in enumerate(keystab):
try:
val=int(float(dataline[i+1])*10000)
except:
val=999990000
datearray.append(val)
data_bin = struct.pack('<'+packcode,*datearray) # little endian
except:
log.msg('Error while packing binary data')
if not self.confdict.get('bufferdirectory','') == '' and data_bin:
acs.dataToFile(self.confdict.get('bufferdirectory'), sensorid, filename, data_bin, header)
if self.debug:
log.msg(" -> DEBUG - sending ... {}".format(','.join(list(map(str,datearray))), header))
self.sendData(sensorid,','.join(list(map(str,datearray))),header,len(newli)-1)
self.lastt[index]=li[-1][0]
t2 = datetime.utcnow()
if self.debug:
log.msg(" -> DEBUG - Needed {}".format(t2-t1))
def processPos1Data(self, data):
"""Convert raw ADC counts into SI units as per datasheets"""
if len(data) != 44:
log.err('POS1 - Protocol: Unable to parse data of length %i' % len(data))
currenttime = datetime.utcnow()
outdate = datetime.strftime(currenttime, "%Y-%m-%d")
actualtime = datetime.strftime(currenttime, "%Y-%m-%dT%H:%M:%S.%f")
outtime = datetime.strftime(currenttime, "%H:%M:%S")
timestamp = datetime.strftime(currenttime, "%Y-%m-%d %H:%M:%S.%f")
filename = outdate
sensorid = self.sensor
packcode = '6hLLLh6hL'
header = "# MagPyBin %s %s %s %s %s %s %d" % (self.sensor, '[f,df,var1,sectime]', '[f,df,var1,GPStime]', '[nT,nT,none,none]', '[1000,1000,1,1]', packcode, struct.calcsize('<'+packcode))
try:
# Extract data
data_array = data.split()
intensity = float(data_array[0])/1000.
sigma_int = float(data_array[2])/1000.
err_code = int(data_array[3].strip('[').strip(']'))
dataelements = datetime.strptime(data_array[4],"%m-%d-%y")
newdate = datetime.strftime(dataelements,"%Y-%m-%d")
gps_time = newdate + ' ' + str(data_array[5])[:11]
except:
log.err('POS1 - Protocol: Data formatting error.')
intensity = 0.0
sigma_int = 0.0
err_code = 0.0
try:
# Analyze time difference between POS1 internal time and utc from PC
# Please note that the time difference between POS1-GPS (data recorded)
# and NTP (data received at PC) can be very large
# for our POS1 it is 6.2 seconds
gpstime = datetime.strptime(gps_time, "%Y-%m-%d %H:%M:%S.%f")
timelist = sorted([gpstime,currenttime])
timediff = timelist[1]-timelist[0]
delta = timediff.total_seconds()
if not delta in [0.0, np.nan, None]:
self.delaylist.append(delta)
self.delaylist = self.delaylist[-1000:]
if len(self.delaylist) > 100:
try:
self.timedelay = np.median(np.asarray(self.delaylist))
except:
self.timedelay = 0.0
if delta-self.ntp_gps_offset > self.timethreshold:
self.errorcnt['time'] +=1
if self.errorcnt.get('time') < 2:
log.msg("{} protocol: large time difference observed for {}: {} sec".format(self.sensordict.get('protocol'), sensorid, secdiff))
else:
self.errorcnt['time'] = 0
except:
pass
if self.sensordict.get('ptime','') in ['NTP','ntp']:
secondtime = gps_time
maintime = timestamp
else:
maintime = gps_time
secondtime = timestamp
try:
# extract time data
datearray = acs.timeToArray(maintime)
sectarray = acs.timeToArray(secondtime)
try:
datearray.append(int(intensity*1000))
datearray.append(int(sigma_int*1000))
datearray.append(err_code)
datearray.extend(sectarray)
data_bin = struct.pack('<'+packcode,datearray[0],datearray[1],datearray[2],datearray[3],datearray[4],datearray[5],datearray[6],datearray[7],datearray[8],datearray[9],datearray[10],datearray[11],datearray[12],datearray[13],datearray[14],datearray[15],datearray[16])
except:
log.msg('POS1 - Protocol: Error while packing binary data')
pass
if not self.confdict.get('bufferdirectory','') == '':
acs.dataToFile(self.confdict.get('bufferdirectory'), sensorid, filename, data_bin, header)
except:
log.msg('POS1 - Protocol: Error with binary save routine')
pass
return ','.join(list(map(str,datearray))), header