def processData(self, data):
"""Process Environment data """
currenttime = datetime.utcnow()
outdate = datetime.strftime(currenttime, "%Y-%m-%d")
filename = outdate
actualtime = datetime.strftime(currenttime, "%Y-%m-%dT%H:%M:%S.%f")
outtime = datetime.strftime(currenttime, "%H:%M:%S")
timestamp = datetime.strftime(currenttime, "%Y-%m-%d %H:%M:%S.%f")
packcode = '6hLl'
sensorid = self.sensor
header = "# MagPyBin %s %s %s %s %s %s %d" % (
sensorid, '[x]', '[RN]', '[random]', '[1000]', packcode,
struct.calcsize('<' + packcode))
try:
datearray = acs.timeToArray(timestamp)
datearray.append(int(data * 1000))
data_bin = struct.pack('<' + packcode,
*datearray) #use little endian byte order
except:
log.msg('Error while packing binary data')
pass
if not self.confdict.get('bufferdirectory', '') == '':
acs.dataToFile(self.confdict.get('bufferdirectory'), sensorid,
filename, data_bin, header)
return ','.join(list(map(str, datearray))), header
def processData(self, sensorid, line, ntptime):
"""processing and storing data - requires sensorid and meta info
data looks like (TR00002):
01.6 290 +14.8 0E*4F
windspeed, winddirection, virtualtemperature, status*pruefsumme
"""
# currenttime = datetime.utcnow()
outdate = datetime.strftime(ntptime, "%Y-%m-%d")
filename = outdate
header = ''
datearray = datetime2array(ntptime)
packcode = '6hLlll'
multiplier = [10, 10, 1]
#print ("Processing line for {}: {}".format(sensorid, line))
vals = line.split()
if len(vals) > 3:
try:
datearray.append(int(float(vals[2]) * 10))
datearray.append(int(float(vals[0]) * 10))
datearray.append(int(float(vals[1]) * 1))
except:
log.msg(
'{} protocol: Error while appending data to file'.format(
self.sensordict.get('protocol')))
try:
data_bin = struct.pack('<' + packcode,
*datearray) #little endian
except:
log.msg('{} protocol: Error while packing binary data'.format(
self.sensordict.get('protocol')))
#asseble header from available global information - write only if information is complete
key = '[t2,var1,var2]'
ele = '[Tv,V,Dir]'
unit = '[degC,m_s,deg]'
multplier = str(multiplier).replace(" ", "")
# Correct some common old problem
unit = unit.replace('deg C', 'degC')
#print ("ID process", sensorid)
header = "# MagPyBin {} {} {} {} {} {} {}".format(
sensorid, key, ele, unit, multplier, packcode,
struct.calcsize('<' + packcode))
data = ','.join(list(map(str, datearray)))
if not self.confdict.get('bufferdirectory', '') == '':
acs.dataToFile(self.confdict.get('bufferdirectory'), sensorid,
filename, data_bin, header)
else:
data = ''
return data, header
def processBlock(self, sensorid, meta, data):
"""Convert raw ADC counts into SI units as per datasheets"""
currenttime = datetime.utcnow()
outdate = datetime.strftime(currenttime, "%Y-%m-%d")
actualtime = datetime.strftime(currenttime, "%Y-%m-%dT%H:%M:%S.%f")
outtime = datetime.strftime(currenttime, "%H:%M:%S")
filename = outdate
datearray = datetime2array(currenttime)
packcode = '6hL'
#sensorid = self.sensordict.get(idnum)
#events = self.eventdict.get(idnum).replace('evt','').split(',')[3:-1]
values = []
multiplier = []
for dat in data:
try:
values.append(float(dat))
datearray.append(int(float(dat) * 10000))
packcode = packcode + 'l'
multiplier.append(10000)
except:
log.msg(
'{} protocol: Error while appending data to file (non-float?): {}'
.format(self.sensordict.get('protocol'), dat))
try:
data_bin = struct.pack('<' + packcode, *datearray) #little endian
except:
log.msg('{} protocol: Error while packing binary data'.format(
self.sensordict.get('protocol')))
pass
#asseble header from available global information - write only if information is complete
key = '[' + str(meta.get('SensorKeys')).replace("'", "").strip() + ']'
ele = '[' + str(meta.get('SensorElements')).replace("'",
"").strip() + ']'
unit = '[' + str(meta.get('SensorUnits')).replace("'",
"").strip() + ']'
multplier = str(multiplier).replace(" ", "")
# Correct some common old problem
unit = unit.replace('deg C', 'degC')
header = "# MagPyBin %s %s %s %s %s %s %d" % (
sensorid, key, ele, unit, multplier, packcode,
struct.calcsize('<' + packcode))
if not self.confdict.get('bufferdirectory',
'') == '' and headercomplete:
acs.dataToFile(self.confdict.get('bufferdirectory'), sensorid,
filename, data_bin, header)
return ','.join(list(map(str, datearray))), header
def processData(self, data):
currenttime = datetime.utcnow()
outdate = datetime.strftime(currenttime, "%Y-%m-%d")
filename = outdate
sensorid = self.sensor
datearray = []
pressure_raw = 88888.8
pressure = 88888.8
typ = "none"
dontsavedata = False
packcode = '6hLL'
header = "# MagPyBin %s %s %s %s %s %s %d" % (
self.sensor, '[var3]', '[p1]', '[mBar]', '[1000]', packcode,
struct.calcsize(packcode))
try:
if len(data) == 2:
typ = "valid"
# add other types here
except:
# TODO??? base x mobile?
log.err(
'BM35 - Protocol: Output format not supported - use either base, ... or mobile'
)
if typ == "valid":
pressure_raw = float(data[0].strip())
pressure = float(data[1].strip())
elif typ == "none":
dontsavedata = True
pass
if not typ == "none":
# extract time data
datearray = datetime2array(currenttime)
try:
datearray.append(int(pressure * 1000.))
data_bin = struct.pack('<' + packcode, *datearray)
except:
log.msg('{} protocol: Error while packing binary data'.format(
self.sensordict.get('protocol')))
if not self.confdict.get('bufferdirectory', '') == '':
acs.dataToFile(self.confdict.get('bufferdirectory'), sensorid,
filename, data_bin, header)
returndata = ','.join(list(map(str, datearray)))
else:
returndata = ''
return returndata, header
def processData(self, data):
"""Convert raw ADC counts into SI units as per datasheets"""
currenttime = datetime.utcnow()
# Correction for ms time to work with databank:
currenttime_ms = currenttime.microsecond / 1000000.
ms_rounded = round(float(currenttime_ms), 3)
if not ms_rounded >= 1.0:
currenttime = currenttime.replace(microsecond=int(ms_rounded *
1000000.))
else:
currenttime = currenttime.replace(microsecond=0) + timedelta(
seconds=1.0)
filename = datetime.strftime(currenttime, "%Y-%m-%d")
actualtime = datetime.strftime(currenttime, "%Y-%m-%dT%H:%M:%S.%f")
lastActualtime = currenttime
outtime = datetime.strftime(currenttime, "%H:%M:%S")
timestamp = datetime.strftime(currenttime, "%Y-%m-%d %H:%M:%S.%f")
sensorid = self.sensor
packcode = '6hLL'
header = "# MagPyBin %s %s %s %s %s %s %d" % (
self.sensor, '[f]', '[f]', '[nT]', '[1000]', packcode,
struct.calcsize('<' + packcode))
try:
intval = data[1].split(',')
value = float(intval[0].strip())
if 10000 < value < 100000:
intensity = value
else:
intensity = 88888.0
except ValueError:
log.err("CS - Protocol: Not a number. Instead found:", data[0])
intensity = 88888.0
try:
datearray = acs.timeToArray(timestamp)
datearray.append(int(intensity * 1000))
data_bin = struct.pack('<' + packcode, *datearray)
except:
log.msg('Error while packing binary data')
if not self.confdict.get('bufferdirectory', '') == '':
acs.dataToFile(self.confdict.get('bufferdirectory'), sensorid,
filename, data_bin, header)
return ','.join(list(map(str, datearray))), header
def processOwData(self, sensorid, datadict):
"""Process OW data """
currenttime = datetime.utcnow()
outdate = datetime.strftime(currenttime, "%Y-%m-%d")
filename = outdate
actualtime = datetime.strftime(currenttime, "%Y-%m-%dT%H:%M:%S.%f")
outtime = datetime.strftime(currenttime, "%H:%M:%S")
timestamp = datetime.strftime(currenttime, "%Y-%m-%d %H:%M:%S.%f")
packcode = '6hL' + 'l' * len(datadict)
multplier = str([1000] * len(datadict)).replace(' ', '')
if sensorid.startswith('DS18'):
key = '[t1]'
ele = '[T]'
unit = '[degC]'
elif sensorid.startswith('DS2438'):
#'temperature','VAD','VDD','humidity','vis'
key = '[t1,var1,var2,var3,var4]'
ele = '[T,RH,VDD,VAD,VIS]'
unit = '[degC,per,V,V,V,V]'
header = "# MagPyBin %s %s %s %s %s %s %d" % (
sensorid, key, ele, unit, multplier, packcode,
struct.calcsize('<' + packcode))
data_bin = None
datearray = ''
try:
datearray = acs.timeToArray(timestamp)
paralst = typedef.get(sensorid.split('_')[0])
for para in paralst:
if para in datadict:
datearray.append(int(float(datadict[para]) * 1000))
data_bin = struct.pack('<' + packcode,
*datearray) # little endian
except:
log.msg('Error while packing binary data')
if not self.confdict.get('bufferdirectory', '') == '' and data_bin:
acs.dataToFile(self.confdict.get('bufferdirectory'), sensorid,
filename, data_bin, header)
#print ("Sending", ','.join(list(map(str,datearray))), header)
return ','.join(list(map(str, datearray))), header
def processData(self, data):
"""Process Environment data """
currenttime = datetime.utcnow()
outdate = datetime.strftime(currenttime, "%Y-%m-%d")
filename = outdate
actualtime = datetime.strftime(currenttime, "%Y-%m-%dT%H:%M:%S.%f")
outtime = datetime.strftime(currenttime, "%H:%M:%S")
timestamp = datetime.strftime(currenttime, "%Y-%m-%d %H:%M:%S.%f")
packcode = '6hLllL'
sensorid = self.sensor
header = "# MagPyBin %s %s %s %s %s %s %d" % (sensorid, '[t1,t2,var1]', '[T,DewPoint,RH]', '[degC,degC,per]', '[1000,1000,1000]', packcode, struct.calcsize('<'+packcode))
valrh = re.findall(r'\d+',data[0])
if len(valrh) > 1:
temp = float(valrh[0] + '.' + valrh[1])
else:
temp = float(valrh[0])
valrh = re.findall(r'\d+',data[1])
if len(valrh) > 1:
rh = float(valrh[0] + '.' + valrh[1])
else:
rh = float(valrh[0])
valrh = re.findall(r'\d+',data[2])
if len(valrh) > 1:
dew = float(valrh[0] + '.' + valrh[1])
else:
dew = float(valrh[0])
try:
datearray = acs.timeToArray(timestamp)
datearray.append(int(temp*1000))
datearray.append(int(dew*1000))
datearray.append(int(rh*1000))
data_bin = struct.pack('<'+packcode,*datearray) #use little endian byte order
except:
log.msg('Error while packing binary data')
pass
if not self.confdict.get('bufferdirectory','') == '':
acs.dataToFile(self.confdict.get('bufferdirectory'), sensorid, filename, data_bin, header)
return ','.join(list(map(str,datearray))), header
def on_message(client, userdata, msg):
if pyversion.startswith('3'):
msg.payload = msg.payload.decode('ascii')
global qos
global verifiedlocation
arrayinterpreted = False
if stationid in ['all', 'All', 'ALL']:
stid = msg.topic.split('/')[0]
else:
stid = stationid
try:
sensorind = msg.topic.split('/')[1]
sensorid = sensorind.replace('meta',
'').replace('data',
'').replace('dict', '')
except:
# Above will fail if msg.topic does not contain /
# TODO (previous version was without 1, first occurrence -> the following line should work as well although the code above is more general)
sensorid = msg.topic.replace(stid, "", 1).replace('/', '').replace(
'meta', '').replace('data', '').replace('dict', '')
# define a new data stream for each non-existing sensor
if not instrument == '':
if not sensorid.find(instrument) > -1:
return
if sensorid in blacklist:
if debug:
print("Sensor {} in blacklist - not collecting".format(sensorid))
return
## ################################################################################
## #### Eventually check for additional format libraries #########
## ################################################################################
identdic = {}
if addlib and len(addlib) > 0:
# Currently only one additioal library is supported
lib = addlib[0]
#for lib in addlib:
elemlist = []
for elem in topic_identifiers[lib]:
strelem = "msg.topic.{}('{}')".format(elem,
topic_identifiers[lib][elem])
elemlist.append(strelem)
if len(elemlist) > 1:
teststring = " and ".join(elemlist)
else:
teststring = "".join(elemlist)
if eval(teststring):
classref = class_reference.get(lib)
#print ("1", msg.payload)
try:
msg.payload, sensorid, headerline, headerdictionary, identdic = classref.GetPayload(
msg.payload, msg.topic)
except:
print("Interpretation error for {}".format(msg.topic))
return
#print (payload, sensorid, headerline)
headdict[sensorid] = headerline
headstream[sensorid] = create_head_dict(headerline, sensorid)
headstream[sensorid] = merge_two_dicts(headstream[sensorid],
headerdictionary)
msg.topic = msg.topic + '/data'
for el in identdic:
po.identifier[el] = identdic[el]
metacheck = po.identifier.get(sensorid + ':packingcode', '')
## ################################################################################
if msg.topic.endswith('meta') and metacheck == '':
log.msg("Found basic header:{}".format(str(msg.payload)))
log.msg("Quality of Service (QOS):{}".format(str(msg.qos)))
analyse_meta(str(msg.payload), sensorid)
if not sensorid in headdict:
headdict[sensorid] = msg.payload
# create stream.header dictionary and it here
headstream[sensorid] = create_head_dict(str(msg.payload), sensorid)
if debug:
log.msg("New headdict: {}".format(headdict))
elif msg.topic.endswith('dict') and sensorid in headdict:
#log.msg("Found Dictionary:{}".format(str(msg.payload)))
head_dict = headstream[sensorid]
for elem in str(msg.payload).split(','):
keyvaluespair = elem.split(':')
try:
if not keyvaluespair[1] in ['-', '-\n', '-\r\n']:
head_dict[keyvaluespair[0]] = keyvaluespair[1].strip()
except:
pass
if debug:
log.msg("Dictionary now looks like {}".format(
headstream[sensorid]))
elif msg.topic.endswith('data'): # or readable json
#if readable json -> create stream.ndarray and set arrayinterpreted :
# log.msg("Found data:", str(msg.payload), metacheck)
if not metacheck == '':
if 'file' in destination:
# Import module for writing data from acquistion
# -------------------
#if debug:
# log.msg(sensorid, metacheck, msg.payload) # payload can be split
# Check whether header is already identified
# -------------------
if sensorid in headdict:
header = headdict.get(sensorid)
if sys.version_info >= (3, 0):
metacheck = metacheck.decode()
if metacheck.endswith('B'):
packcode = metacheck.strip(
'<')[:-1] # drop leading < and final B
else:
packcode = metacheck.strip('<') # drop leading <
# temporary code - too be deleted when lemi protocol has been updated
if packcode.find('4cb6B8hb30f3Bc') >= 0:
header = header.replace(
'<4cb6B8hb30f3BcBcc5hL 169\n',
'6hLffflll {}'.format(
struct.calcsize('<6hLffflll')))
packcode = '6hLffflll'
arrayelem = msg.payload.split(';')
for ar in arrayelem:
datearray = ar.split(',')
# identify string values in packcode
# -------------------
# convert packcode numbers
cpack = []
for c in packcode:
if c.isdigit():
digit = int(c)
else:
cpack.extend([c] * digit)
digit = 1
cpackcode = "".join(cpack)
for i in range(len(cpackcode)):
if cpackcode[-i] == 's':
datearray[-i] = datearray[-i]
elif cpackcode[-i] == 'f':
datearray[-i] = float(datearray[-i])
else:
datearray[-i] = int(float(datearray[-i]))
# pack data using little endian byte order
data_bin = struct.pack('<' + packcode, *datearray)
# Check whether destination path has been verified already
# -------------------
if not verifiedlocation:
if not location in [None, ''
] and os.path.exists(location):
verifiedlocation = True
else:
log.msg(
"File: destination location {} is not accessible"
.format(location))
log.msg(
" -> please use option l (e.g. -l '/my/path') to define"
)
if verifiedlocation:
filename = "{}-{:02d}-{:02d}".format(
datearray[0], datearray[1], datearray[2])
acs.dataToFile(location, sensorid, filename,
data_bin, header)
if 'websocket' in destination:
if not arrayinterpreted:
stream.ndarray = interprete_data(msg.payload, stream,
sensorid)
#streamdict[sensorid] = stream.ndarray # to store data from different sensors
arrayinterpreted = True
for idx, el in enumerate(stream.ndarray[0]):
time = num2date(el).replace(tzinfo=None)
msecSince1970 = int(
(time - datetime(1970, 1, 1)).total_seconds() * 1000)
datastring = ','.join([
str(val[idx]) for i, val in enumerate(stream.ndarray)
if len(val) > 0 and not i == 0
])
if debug:
print("Sending {}: {},{} to webserver".format(
sensorid, msecSince1970, datastring))
wsserver.send_message_to_all("{}: {},{}".format(
sensorid, msecSince1970, datastring))
if 'diff' in destination:
global counter
counter += 1
global number
amount = int(number)
cover = 5
if not arrayinterpreted:
ar = interprete_data(msg.payload, stream, sensorid)
if not sensorid in senslst:
senslst.append(sensorid)
st.append(DataStream([], {}, ar))
idx = senslst.index(sensorid)
st[idx].extend(stream.container, {'SensorID': sensorid},
ar)
arrayinterpreted = True
st[idx].ndarray = np.asarray(
[np.asarray(el[-cover:]) for el in st[idx].ndarray])
if len(st) < 2:
print("Not enough streams for subtraction yet")
try:
if counter > amount:
counter = 0
sub = subtractStreams(st[0], st[1])
try:
part1 = (
st[0].header.get('SensorID').split('_')[1])
except:
part1 = 'unkown'
try:
part2 = (
st[1].header.get('SensorID').split('_')[1])
except:
part2 = 'unkown'
name = "Diff_{}-{}_0001".format(part1, part2)
# get head line for pub
#name = "diff_xxx_0001"
keys = sub._get_key_headers(numerical=True)
ilst = [KEYLIST.index(key) for key in keys]
keystr = "[{}]".format(",".join(keys))
#takeunits = ### take from st[0]
packcode = "6hL{}".format("".join(['l'] * len(keys)))
multi = "[{}]".format(",".join(['1000'] * len(keys)))
unit = "[{}]".format(",".join(['arb'] * len(keys)))
head = "# MagPyBin {} {} {} {} {} {} {}".format(
name, keystr, keystr, unit, multi, packcode,
struct.calcsize('<' + packcode))
#print (head)
# get data line for pub
time = sub.ndarray[0][-1]
timestr = (datetime.strftime(
num2date(float(time)).replace(tzinfo=None),
"%Y,%m,%d,%H,%M,%S,%f"))
val = [sub.ndarray[i][-1] for i in ilst]
if len(val) > 1:
valstr = ",".join(int(val * 1000))
else:
valstr = int(val[0] * 1000)
data = "{},{}".format(timestr, valstr)
#print (data)
topic = "wic/{}".format(name)
client.publish(topic + "/data", data, qos=qos)
client.publish(topic + "/meta", head, qos=qos)
except:
print("Found error in subtraction")
if 'stdout' in destination:
if not arrayinterpreted:
stream.ndarray = interprete_data(msg.payload, stream,
sensorid)
#streamdict[sensorid] = stream.ndarray # to store data from different sensors
arrayinterpreted = True
for idx, el in enumerate(stream.ndarray[0]):
time = num2date(el).replace(tzinfo=None)
datastring = ','.join([
str(val[idx]) for i, val in enumerate(stream.ndarray)
if len(val) > 0 and not i == 0
])
log.msg("{}: {},{}".format(sensorid, time, datastring))
elif 'db' in destination:
if not arrayinterpreted:
stream.ndarray = interprete_data(msg.payload, stream,
sensorid)
#streamdict[sensorid] = stream.ndarray # to store data from different sensors
arrayinterpreted = True
# create a stream.header
#if debug:
# log.msg(stream.ndarray)
stream.header = headstream[sensorid]
if debug:
log.msg("writing header: {}".format(headstream[sensorid]))
if revision != 'free':
writeDB(db,
stream,
tablename="{}_{}".format(sensorid, '0001'))
else:
writeDB(db, stream)
elif 'stringio' in destination:
if not arrayinterpreted:
stream.ndarray = interprete_data(msg.payload, stream,
sensorid)
#streamdict[sensorid] = stream.ndarray # to store data from different sensors
arrayinterpreted = True
for idx, el in enumerate(stream.ndarray[0]):
time = num2date(el).replace(tzinfo=None)
date = datetime.strftime(time, "%Y-%m-%d %H:%M:%S.%f")
linelist = list(map(str, [el, date]))
linelist.extend([
str(val[idx]) for i, val in enumerate(stream.ndarray)
if len(val) > 0 and not i == 0
])
line = ','.join(linelist)
eol = '\r\n'
output.write(line + eol)
elif 'serial' in destination:
if not arrayinterpreted:
stream.ndarray = interprete_data(msg.payload, stream,
sensorid)
#streamdict[sensorid] = stream.ndarray # to store data from different sensors
arrayinterpreted = True
"""
# send json like structures
collcount = 10
if sercount <= collcount:
for idx,col in enumerate(stream.ndarray):
if not len(col) == 0:
keyname = KEYLIST[idx]
if idx == 0:
time = num2date(col).replace(tzinfo=None)
col = int((time - datetime(1970,1,1)).total_seconds()*1000)
excol = datacol.get(keyname,[])
datacol[keyname] = excol.append(col)
sersount += 1
if sercount == collcount:
sercount = 0
jsonstr={}
jsonstr['sensorid'] = sensorid
jsonstr['nr'] = i
jsonstr['key'] = po.identifier[sensorid+':keylist'][i]
jsonstr['elem'] = po.identifier[sensorid+':elemlist'][i]
jsonstr['unit'] = po.identifier[sensorid+':unitlist'][i]
payload = json.dumps(jsonstr)
# write input to a another serial port (e.g. for radio transmisson)
# requires serdef = e.g. [115200,8,1,N]
# eventually create minimum 30 sec json blocks
#sendline='{"SensorID":"{}","Units":["Sec1970","degC"],"Keys":{"time":"time","x":"Temperature"}, "time":[{}],"x":[{}]}'.format(sensorid,time,x)
#{"SensorID":"ID","units":["Sec1970","degC"],"keys":{"time":"time","x":"Temperature"}, "data":{"time":[12,13,45],"x":[12,13,45]}}
#ser.write("{}: {},{}".format(sensorid,msecSince1970,datastring))
"""
else:
pass
else:
log.msg("Non-interpreted format: {} {}".format(
msg.topic, str(msg.payload)))
elif msg.topic.find('statuslog') > 0:
# json style statusinfo is coming
hostname = msg.topic.split('/')[-1]
#log.msg("---------------------------------------------------------------")
#log.msg("Receiving updated status information from {}".format(hostname))
#log.msg("---------------------------------------------------------------")
print("FOUND STATUS CHANGE", telegramconf)
statusdict = json.loads(msg.payload)
for elem in statusdict:
logmsg = "{}: {} - {}".format(hostname, elem, statusdict[elem])
# For Nagios - add in marcos.log
log.msg(logmsg)
# For Telegram
try:
# try to import telegram and telegram.cfg
##### Add the configuration to input and marcos.cfg
## Please note: requires anaconda2/bin/python on my test PC
## !!! Requires network connection !!!
if not telegramconf == '/telegram.conf':
martaslog = ml(receiver='telegram')
martaslog.receiveroptions('telegram',
options={'conf': telegramconf})
statusdict['Hostname'] = hostname
martaslog.notify(statusdict)
except:
pass
#telegram.send(msg)
if msg.topic.endswith('meta') and 'websocket' in destination:
# send header info for each element (# sensorid nr key elem unit)
analyse_meta(str(msg.payload), sensorid)
for (i, void) in enumerate(po.identifier[sensorid + ':keylist']):
jsonstr = {}
jsonstr['sensorid'] = sensorid
jsonstr['nr'] = i
jsonstr['key'] = po.identifier[sensorid + ':keylist'][i]
jsonstr['elem'] = po.identifier[sensorid + ':elemlist'][i]
jsonstr['unit'] = po.identifier[sensorid + ':unitlist'][i]
payload = json.dumps(jsonstr)
wsserver.send_message_to_all('# ' + payload)
def interruptRead(s):
"""
interrupt routine of class AD7714Protocol
triggered by AD7714 /DRDY signal
"""
# at first get the time...
currenttime = datetime.utcnow()
# read from data register
arrvalue = rxreg(5, CHANNEL)
if len(arrvalue) == 2:
# 16 -> 24bit
arrvalue.append(0)
intvalue = (arrvalue[0] << 16) | (arrvalue[1] << 8) | arrvalue[2]
voltvalue = float(intvalue) / 2**24 * 5 - 2.5
# mV better for display
voltvalue = voltvalue * 1000
# TIME TO COMMUNICATE!
global int_comm
if int_comm == "mySettings":
mySettings()
int_comm = "ok"
if int_comm == "myCalibration":
myCalibration()
int_comm = "ok"
if int_comm == "info":
info()
int_comm = "ok"
# watchdog
global watchdog
global Objekt
if watchdog['oldvalue'] == 999999:
print('watchdog active')
if watchdog['oldvalue'] == intvalue:
watchdog['count_repetitions'] = watchdog['count_repetitions'] + 1
else:
if watchdog['count_repetitions'] > 5:
# avoid a lot of log entries - filter double and triple values in a row
print('watchdog resetted, count_repetitions:',
watchdog['count_repetitions'], 'oldvalue:',
watchdog['oldvalue'], 'intvalue:', intvalue)
watchdog['count_repetitions'] = 0
watchdog['max_repetitions'] = watchdog['init_max_rep']
if watchdog['count_repetitions'] == watchdog['max_repetitions']:
# probably hung up, too many same values
print('watchdog ad7714protocol:')
print(' ', watchdog['max_repetitions'], 'same values (intvalue:',
intvalue, ') in one row - hung up?')
print(' trying to reset AD7714...')
# sending LOW to /RESET pin
reset()
time.sleep(0.01)
# loading settings
mySettings()
# zero calibration
myCalibration()
watchdog['max_repetitions'] = watchdog['max_repetitions'] * 2
watchdog['count_repetitions'] = 0
watchdog['oldvalue'] = intvalue
packcode = "6hLl"
sensorid = Objekt.sensordict['sensorid']
header = "# MagPyBin %s %s %s %s %s %s %d" % (sensorid, '[var1]', '[U]',
'[mV]', '[1000]', packcode,
struct.calcsize(packcode))
#timestamp=datetime.strftime(currenttime, "%Y-%m-%d %H:%M:%S.%f")
timestamp = datetime2array(currenttime)
darray = timestamp
darray.append(int(round(voltvalue * 1000)))
# TO FILE
data_bin = struct.pack(packcode, *darray)
filedate = datetime.strftime(datetime(darray[0], darray[1], darray[2]),
"%Y-%m-%d")
if not Objekt.confdict.get('bufferdirectory', '') == '':
acs.dataToFile(Objekt.confdict.get('bufferdirectory'), sensorid,
filedate, data_bin, header)
# VIA MQTT
# instead of external program file TODO: better!
#def sendData(self, sensorid, data, head, stack=None):
#sendData.sendData(Objekt,sensorid, ','.join(list(map(str,darray))), header)
data = ','.join(list(map(str, darray)))
head = header
# TODO: implement stack correctly!
stack = 1
topic = Objekt.confdict.get('station') + '/' + sensorid
senddata = False
if not stack:
stack = int(Objekt.sensordict.get('stack'))
coll = stack
if coll > 1:
Objekt.metacnt = 1 # send meta data with every block
if Objekt.datacnt < coll:
Objekt.datalst.append(data)
Objekt.datacnt += 1
else:
senddata = True
data = ';'.join(Objekt.datalst)
Objekt.datalst = []
Objekt.datacnt = 0
else:
senddata = True
if senddata:
if Objekt.count == 0:
# get all values initially from the database
#add = "SensoriD:{},StationID:{},DataPier:{},SensorModule:{},SensorGroup:{},SensorDecription:{},DataTimeProtocol:{}".format( sensorid, self.confdict.get('station',''),self.sensordict.get('pierid',''), self.sensordict.get('protocol',''),self.sensordict.get('sensorgroup',''),self.sensordict.get('sensordesc',''), self.sensordict.get('ptime','') )
#self.client.publish(topic+"/dict", add, qos=self.qos)
Objekt.client.publish(topic + "/meta", head, qos=Objekt.qos)
if Objekt.debug:
log.msg(" -> DEBUG - Publishing meta --", topic, head)
Objekt.client.publish(topic + "/data", data, qos=Objekt.qos)
if Objekt.debug:
log.msg(" -> DEBUG - Publishing data")
Objekt.count += 1
if Objekt.count >= Objekt.metacnt:
Objekt.count = 0
def processData(self, data):
""" GP20S3 data """
"""
Data looks like--- (with GPS lines every minute):
-- vertical sensor - Old software
3,3,12.00 111 field1 field2 field3
3,3,12.00 111 field1 field2 field3
GPS 16.00 111 field1 field2 field3
-- horizontal sensor - New software
time 111 field1 field2 field3 (every sec or faster)
$$$ (every hour, preceeds status line)
10071506 A 13 250 492 496 329 150 1023 39 39 39 30 29 30 YYYyyyEEENNN 148 149 117 (every hour)
time 111 field1 field2 field3 (every sec or faster)
time 111 field1 field2 field3 (every sec or faster)
"""
currenttime = datetime.utcnow()
outdate = datetime.strftime(currenttime, "%Y-%m-%d")
actualtime = datetime.strftime(currenttime, "%Y-%m-%dT%H:%M:%S.%f")
outtime = datetime.strftime(currenttime, "%H:%M:%S")
timestamp = datetime.strftime(currenttime, "%Y-%m-%d %H:%M:%S.%f")
filename = outdate
sensorid = self.sensor
headerlinecoming = False
datearray = []
headarray = []
statusname = "Status_123_0001"
sensororientation = self.sensor.split('_')[0].replace(self.sensordict.get('protocol'),'')
if len(sensororientation) > 1:
sens1 = sensororientation
sens2 = sensororientation[0]
sens3 = sensororientation[1]
else:
sens1 = 'TA'
sens2 = 'B'
sens3 = 'TB'
celem = '[{},{},{},{}{},{}{},{}{},None]'.format(sens1,sens2,sens3, sens3,sens1, sens3,sens2, sens2,sens1)
packcode = '6hLQQQqqq6hL'
header = "# MagPyBin %s %s %s %s %s %s %d" % (self.sensor, '[x,y,z,dx,dy,dz,sectime]', celem, '[pT,pT,pT,pT,pT,pT,None]', '[1000,1000,1000,1000,1000,1000,1]', packcode, struct.calcsize('<'+packcode))
try:
# Extract data
data_array = data
if len(data_array) == 5:
intensity1 = float(data_array[2])
intensity2 = float(data_array[3])
intensity3 = float(data_array[4])
grad1 = intensity3-intensity1
grad2 = intensity3-intensity2
grad3 = intensity2-intensity1
try:
gpstime = float(data[0]) # will fail for old dataformat -> NTP
if gpstime > 235900.0: # use date of last day if gpstime > 235900 to prevent next day date for 235959 gps when pctime already is on next day
cdate = dateprev
else:
cdate = outdate
dateprev = outdate
try:
internal_t = datetime.strptime(cdate+'T'+data[0], "%Y-%m-%dT%H%M%S.%f")
except:
internal_t = datetime.strptime(cdate+'T'+data[0], "%Y-%m-%dT%H%M%S")
internal_time = datetime.strftime(internal_t, "%Y-%m-%d %H:%M:%S.%f")
except:
internal_time = timestamp #datetime.strftime(datetime.utcnow(), "%Y-%m-%d %H:%M:%S.%f")
elif len(data_array) == 19:
"""
10071506 A 13 250 492 496 329 150 1023 39 39 39 30 29 30 YYYyyyEEENNN 148 149 117
<GPS> day/month/year/hour A - locked, V unlocked
<13> Console outside air temperature (13C)
<250> Battery voltage (25.0V)
<492> +5V supply voltage (4.92V)
<496> -5V supply voltage (-4.96)
<3.3V> +3.3V supply voltage (3.3V)
<15.0> silver box power supply (15.0V)
<1023> OCXO internal trimpot adjustment level, automatically adjusted via GPS
<39> Sensor 1 temperature in C
<39> Sensor 2 temperature in C
<39> Sensor 3 temperature in C
<30> Light current sensor 1 (3.0uA)
<29> Light current sensor 2 (2.9uA)
<30> Light current sensor 3 (3.0uA)
<YYY> Sensor 1, sensor 2 sensor 3 lock status Y- locked, N - unlocked
<yyy> Sensor 1 heater status, sensor 2 heater status, sensor 3 heater status y-on, n-off
<EEE> Sensor 1 heater, sensor 2 heater, sensor 3 heater E-enabled, D-disabled (used for over heat protection)
<NNN> RF sensor 1, RF sensor 2, RF sensor 3, N -on, F-off
<148> Sensor 1 RF dc voltage (14.8V)
<149> Sensor 2 RF dc voltage (14.9V)
<117> Sensor 3 RF dc voltage (11.7V)
"""
headerlinecoming = True
try:
gpstime = str(data_array[0])
internal_t = datetime.strptime(gpstime, "%d%m%y%H")
gpstimestamp = datetime.strftime(internal_t, "%Y-%m-%d %H:%M:%S.%f")
except:
gpstimestamp = timestamp
internal_time = gpstimestamp
gpstatus = data_array[1] # str1
telec = int(data_array[2]) # t2
Vbat = float(data_array[3])/10. # f
Vsup1 = float(data_array[4])/100. # var4
Vsup2 = float(data_array[5])/100. # var5
Vlow = float(data_array[6])/100. # t1
PowerSup = float(data_array[7])/10. # df
level = data_array[8] # str3
tsens1 = int(data_array[9]) # x
tsens2 = int(data_array[10]) # y
tsens3 = int(data_array[11]) # z
lightcurrent1 = float(data_array[12])/10. # dx
lightcurrent2 = float(data_array[13])/10. # dy
lightcurrent3 = float(data_array[14])/10. # dz
statusstring = data_array[15] # str2
Vsens1 = float(data_array[16])/10. # var1
Vsens2 = float(data_array[17])/10. # var2
Vsens3 = float(data_array[18])/10. # var3
elif len(data_array) == 1 and data_array[0] == '$$$':
return "","",""
else:
log.msg('{} protocol: data line could not be interpreted: ({}) of length {}'.format(self.sensordict.get('protocol'),data, len(data_array)))
except:
log.err('{} protocol: Data formatting error. Data looks like: {}'.format(self.sensordict.get('protocol'),data))
try:
# Analyze time difference between GSM internal time and utc from PC
timelist = sorted([internal_t,currenttime])
timediff = timelist[1]-timelist[0]
delta = timediff.total_seconds()
if not delta in [0.0, float('NAN'), None]:
self.delaylist.append(timediff.total_seconds())
if len(self.delaylist) > 600:
self.delaylist = self.delaylist[-600:]
if len(self.delaylist) > 100:
try:
self.timedelay = np.abs(np.median(np.asarray(self.delaylist)))
except:
self.timedelay = 0.0
if self.timedelay > self.timethreshold:
self.errorcnt['time'] +=1
if self.errorcnt.get('time') < 2:
log.msg("{} protocol: large time difference observed for {}: {} sec".format(self.sensordict.get('protocol'), sensorid, self.timedelay))
if self.errorcnt.get('time') > 1000:
self.errorcnt['time'] = 1000
else:
self.errorcnt['time'] = 0
except:
pass
if self.sensordict.get('ptime','') in ['NTP','ntp']:
secondtime = internal_time
maintime = timestamp
else:
maintime = internal_time
secondtime = timestamp
if not headerlinecoming:
try:
## GP20S3 provides info on whether the GPS reading is OK - use it
# extract time data
datearray = acs.timeToArray(maintime)
try:
datearray.append(int(intensity1*1000.))
datearray.append(int(intensity2*1000.))
datearray.append(int(intensity3*1000.))
datearray.append(int(grad1*1000.))
datearray.append(int(grad2*1000.))
datearray.append(int(grad3*1000.))
internalarray = acs.timeToArray(secondtime)
datearray.extend(internalarray)
data_bin = struct.pack('<'+packcode,*datearray)
except:
log.msg('{} protocol: Error while packing binary data'.format(self.sensordict.get('protocol')))
if not self.confdict.get('bufferdirectory','') == '':
acs.dataToFile(self.confdict.get('bufferdirectory'), sensorid, filename, data_bin, header)
except:
log.msg('{} protocol: Error with binary save routine'.format(self.sensordict.get('protocol')))
if headerlinecoming:
if self.debug:
print (" now writing header info")
headpackcode = '6hL15ls12s4s' #'6hLlllllllllllllllsss'
statusname = "Status_123_0001"
try:
# extract time data
headarray = acs.timeToArray(maintime)
try:
headarray.append(int(tsens1)) # x
headarray.append(int(tsens2)) # y
headarray.append(int(tsens3)) # z
headarray.append(int(Vbat*10.)) # f
headarray.append(int(Vlow*100.)) # t1
headarray.append(int(telec)) # t2
headarray.append(int(lightcurrent1*10.)) # dx
headarray.append(int(lightcurrent2*10.)) # dy
headarray.append(int(lightcurrent3*10.)) # dz
headarray.append(int(PowerSup*10.)) # df
headarray.append(int(Vsens1*10.)) # var1
headarray.append(int(Vsens2*10.)) # var2
headarray.append(int(Vsens3*10.)) # var3
headarray.append(int(Vsup1*100.)) # var4
headarray.append(int(Vsup2*100.)) # var5
headarray.append(gpstatus) # str1
headarray.append(statusstring) # str2
headarray.append(level) # str3
data_bin = struct.pack('<'+headpackcode,*headarray)
statuslst = self.sensor.split('_')
if self.debug:
print ("Headerdata has been packed")
if len(statuslst) == 3:
statusname = '_'.join([statuslst[0]+'status',statuslst[1],statuslst[2]])
headheader = "# MagPyBin %s %s %s %s %s %s %d" % (statusname, '[x,y,z,f,t1,t2,dx,dy,dz,df,var1,var2,var3,var4,var5,str1,str2,str3]', '[Ts1,Ts2,Ts3,Vbat,V3,Tel,L1,L2,L3,Vps,V1,V2,V3,V5p,V5n,GPSstat,Status,OCXO]', '[degC,degC,degC,V,V,degC,A,A,A,V,V,V,V,V,V,None,None,None]', '[1,1,1,10,100,1,10,10,10,10,10,10,10,100,100,1,1,1]', headpackcode, struct.calcsize('<'+headpackcode))
if self.debug:
print ("Header looks like: {} ".format(headheader))
print ("Writing to file: {}, {}, {}".format(statusname,filename,headheader))
if not self.confdict.get('bufferdirectory','') == '':
acs.dataToFile(self.confdict.get('bufferdirectory'), statusname, filename, data_bin, headheader)
except:
log.msg('GSMP20 - Protocol: Error while packing binary data')
except:
pass
if len(datearray) > 0:
topic = self.confdict.get('station') + '/' + self.sensordict.get('sensorid')
return ','.join(list(map(str,datearray))), header, topic
elif len(headarray) > 0:
topic = self.confdict.get('station') + '/' + statusname
return ','.join(list(map(str,headarray))), headheader, topic
def sendRequest(self):
# TODO wohin mit debug?
debug = False
if self.reconnect.is_set():
log.msg('exiting, mutex locked!')
return
t = datetime.utcnow()
past = t-timedelta(seconds=3)
vals = self.device.get_data('SamplesEvery2s',past,t)
# vals[0] because we grap no older data, there is only one value in 2 seconds
# timestamp directly from datetime into array
# TODO Roman fragen, ob oder wie Vergleich mit Computerzeit
try:
darray = datetime2array(vals[0]['Datetime'])
# TODO "again" ist Provisorium
again = False
except:
again = True
try:
if again:
t = datetime.utcnow()
past = t-timedelta(seconds=3)
darray = datetime2array(vals[0]['Datetime'])
log.msg("IT TOOK A SECOND TIME TO GET DATA PROPERLY!")
except:
# there will be no log messages when the logger is turned off
return
# TODO reconnect Loesung, nur wenn sie sauber funktioniert!
log.msg('NO DATA FROM CR1000 !!! - vals:')
log.msg(vals)
port = self.confdict['serialport']+self.sensordict.get('port')
baudrate = self.sensordict.get('baudrate')
self.reconnect.set()
connected=False
while not connected:
self.device.bye()
log.msg('reconnecting to device...')
time.sleep(5)
try:
self.device = CR1000.from_url('serial:{}:{}'.format(port,baudrate))
tables = self.device.list_tables()
if tables == ['Status', 'SamplesEvery2s', 'ValuesEveryMinute', 'Public']:
connected=True
log.msg('schaut ok aus...')
time.sleep(2)
except:
log.msg('reconnect wohl missglueckt!')
try:
past = t-timedelta(seconds=3)
vals = self.device.get_data('SamplesEvery2s',past,t,debug=True)
log.msg(SENSOR_HEIGHT*1000.-vals[0]['DiffVolt']*250.)
except:
log.msg('...wohl doch nicht!')
connected=False
self.reconnect.clear()
log.msg('mutex released...')
return
if debug:
log.msg('getting data...')
# snowheight (1000mV is 250cm) - values from CR1000 in mV, factor 1000 for packing
snowheight = SENSOR_HEIGHT*1000.-vals[0]['DiffVolt']*250.
darray.append(int(round(snowheight)))
# TODO weg
if debug:
log.msg(darray)
# preparations for file save
# date 6 short microsecond unsigned long snowheight signed long
# TODO alter packcode!
packcode = "6hLl"
#packcode = "<6hLl"
# header
sensorid = self.sensordict['sensorid']
header = "# MagPyBin %s %s %s %s %s %s %d" % (sensorid,'[f]','[JC]','[cm]','[1000]',packcode,struct.calcsize(packcode))
data_bin = struct.pack(packcode,*darray)
# date of dataloggers timestamp
filedate = datetime.strftime(datetime(darray[0],darray[1],darray[2]), "%Y-%m-%d")
if not self.confdict.get('bufferdirectory','') == '':
acs.dataToFile(self.confdict.get('bufferdirectory'), sensorid, filedate, data_bin, header)
if debug:
log.msg('Daten gesichert...')
# sending via MQTT
data = ','.join(list(map(str,darray)))
head = header
topic = self.confdict.get('station') + '/' + self.sensordict.get('sensorid')
coll = int(self.sensordict.get('stack'))
if coll > 1:
self.metacnt = 1 # send meta data with every block
if self.datacnt < coll:
self.datalst.append(data)
self.datacnt += 1
else:
senddata = True
data = ';'.join(self.datalst)
self.datalst = []
self.datacnt = 0
else:
senddata = True
if senddata:
self.client.publish(topic+"/data", data)
if self.count == 0:
self.client.publish(topic+"/meta", head)
self.count += 1
if self.count >= self.metacnt:
self.count = 0
# right now auxiliary data only in the log file
if t.second<2:
# every minute aux data (battery voltage and logger temperature) will be available
# going 61s into the past to make sure there are already data
past=t-timedelta(seconds=61)
aux=self.device.get_data('ValuesEveryMinute',past,t)
log.msg('----- aux every minute:')
# timestamp directly from datetime into array
try:
darray = datetime2array(aux[0]['Datetime'])
except:
# following should never happen...
log.msg('AUXILIARY DATA NOT GOT PROPERLY! - aux:')
log.msg(aux)
log.msg('trying again...')
past=t-timedelta(seconds=62)
aux=self.device.get_data('ValuesEveryMinute',past,t)
try:
darray = datetime2array(aux[0]['Datetime'])
except:
log.msg('giving up...')
return
# battery voltage - factor 1000 for packing
BattV_Min=int(round(aux[0]['BattV_Min']*1000))
PTemp_C_Avg=int(round(aux[0]['PTemp_C_Avg']*1000))
darray.extend([BattV_Min,PTemp_C_Avg])
# alernative reading:
#aux = (aux.filter(('Datetime', 'BattV_Min','PTemp_C_Avg')).to_csv(header=False))
log.msg(darray)
packcode="<6hLLl"
def processData(self, data):
""" GSM90 data """
currenttime = datetime.utcnow()
outdate = datetime.strftime(currenttime, "%Y-%m-%d")
actualtime = datetime.strftime(currenttime, "%Y-%m-%dT%H:%M:%S.%f")
outtime = datetime.strftime(currenttime, "%H:%M:%S")
timestamp = datetime.strftime(currenttime, "%Y-%m-%d %H:%M:%S.%f")
filename = outdate
sensorid = self.sensor
packcode = '6hLLL6hL'
header = "# MagPyBin %s %s %s %s %s %s %d" % (self.sensor, '[f,var1,sectime]', '[f,errorcode,internaltime]', '[nT,none,none]', '[1000,1,1]', packcode, struct.calcsize('<'+packcode))
try:
# Extract data
# old data looks like 04-22-2015 142244 48464.53 99
data_array = data
if len(data) == 4:
intensity = float(data[2])
err_code = int(data[3])
try:
try:
internal_t = datetime.strptime(data[0]+'T'+data[1], "%m-%d-%YT%H%M%S.%f")
except:
internal_t = datetime.strptime(data[0]+'T'+data[1], "%m-%d-%YT%H%M%S")
internal_time = datetime.strftime(internal_t, "%Y-%m-%d %H:%M:%S.%f")
except:
internal_time = timestamp #datetime.strftime(datetime.utcnow(), "%Y-%m-%d %H:%M:%S.%f")
#print internal_time
elif len(data) == 3: # GSM v7.0
intensity = float(data[1])
err_code = int(data[2])
try:
internal_t = datetime.strptime(outdate+'T'+data[0], "%Y-%m-%dT%H%M%S.%f")
internal_time = datetime.strftime(internal_t, "%Y-%m-%d %H:%M:%S.%f")
except:
internal_time = timestamp #datetime.strftime(datetime.utcnow(), "%Y-%m-%d %H:%M:%S.%f")
else:
err_code = 0
intensity = float(data[0])
internal_time = timestamp #datetime.strftime(datetime.utcnow(), "%Y-%m-%d %H:%M:%S")
except:
log.err('{} protocol: Data formatting error. Data looks like: {}'.format(self.sensordict.get('protocol'),data))
try:
# Analyze time difference between GSM internal time and utc from PC
timelist = sorted([internal_t,currenttime])
timediff = timelist[1]-timelist[0]
#secdiff = timediff.seconds + timediff.microseconds/1E6
#timethreshold = 3
delta = timediff.total_seconds()
if not delta in [0.0, np.nan, None]:
self.delaylist.append(timediff.total_seconds())
self.delaylist = self.delaylist[-1000:]
if len(self.delaylist) > 100:
try:
self.timedelay = np.median(np.asarray(self.delaylist))
except:
self.timedelay = 0.0
#if secdiff > timethreshold:
if delta > self.timethreshold:
self.errorcnt['time'] +=1
if self.errorcnt.get('time') < 2:
log.msg("{} protocol: large time difference observed for {}: {} sec".format(self.sensordict.get('protocol'), sensorid, secdiff))
else:
self.errorcnt['time'] = 0
except:
pass
if self.sensordict.get('ptime','') in ['NTP','ntp']:
secondtime = internal_time
maintime = timestamp
else:
maintime = internal_time
secondtime = timestamp
try:
## GSM90 does not provide any info on whether the GPS reading is OK or not
# extract time data
datearray = acs.timeToArray(maintime)
try:
datearray.append(int(intensity*1000.))
datearray.append(err_code)
#print timestamp, internal_time
internalarray = acs.timeToArray(secondtime)
datearray.extend(internalarray)
data_bin = struct.pack('<'+packcode,*datearray)
except:
log.msg('{} protocol: Error while packing binary data'.format(self.sensordict.get('protocol')))
if not self.confdict.get('bufferdirectory','') == '':
acs.dataToFile(self.confdict.get('bufferdirectory'), sensorid, filename, data_bin, header)
except:
log.msg('{} protocol: Error with binary save routine'.format(self.sensordict.get('protocol')))
return ','.join(list(map(str,datearray))), header
def processData(self, data):
currenttime = datetime.utcnow()
date = datetime.strftime(currenttime, "%Y-%m-%d")
actualtime = datetime.strftime(currenttime, "%Y-%m-%dT%H:%M:%S.%f")
outtime = datetime.strftime(currenttime, "%H:%M:%S")
filename = date
timestamp = datetime.strftime(currenttime, "%Y-%m-%d %H:%M:%S.%f")
intensity = 88888.8
typ = "none"
dontsavedata = False
packcode = '6hLLl'
header = "# MagPyBin %s %s %s %s %s %s %d" % (
self.sensor, '[f,var1]', '[f,err]', '[nT,none]', '[1000,1000]',
packcode, struct.calcsize('<' + packcode))
try:
# Extract data
data_array = data
if len(data_array) == 2:
typ = "oldbase"
elif len(data_array) == 3:
typ = "valid"
# add other types here
except:
log.err(
'GSM19 - Protocol: Output format not supported - use either base, ... or mobile'
)
# Extracting the data from the station
# Extrat time info and use as primary if GPS is on (in this case PC time is secondary)
# PC is primary when a GPS is not connected
if typ == "valid" or typ == "oldbase": # Comprises Mobile and Base Station mode with single sensor and no GPS
intensity = float(data_array[1])
try:
systemtime = datetime.strptime(date + "-" + data_array[0],
"%Y-%m-%d-%H%M%S.%f")
except:
# This exception happens for old GSM19 because time is
# provided e.g. as 410356 instead of 170356 for 17:03:56 (Thursday)
# e.g 570301.0 instead of 09:03:01 (Friday)
try:
hournum = int(data_array[0][:-6])
rest = data_array[0][-6:]
factor = np.floor(hournum /
24.) # factor = days since starting
hour = int(hournum - factor * 24.)
systemtime = datetime.strptime(
date + "-" + str(hour) + rest, "%Y-%m-%d-%H%M%S.%f")
#print ("Got oldbase systemtime")
except:
systemtime = currenttime
self.timesource = 'NTP'
if len(data_array) == 2:
typ = "base"
errorcode = 99
elif len(data_array[2]) == 3:
typ = "base"
errorcode = int(data_array[2])
else:
typ = "gradient"
gradient = float(data_array[2])
elif typ == "none":
dontsavedata = True
pass
gpstime = datetime.strftime(systemtime, "%Y-%m-%d %H:%M:%S.%f")
try:
# Analyze time difference between GSM internal time and utc from PC
timelist = sorted([systemtime, currenttime])
timediff = timelist[1] - timelist[0]
#secdiff = timediff.seconds + timediff.microseconds/1E6
delta = timediff.total_seconds()
if not delta in [0.0, np.nan, None]:
self.delaylist.append(timediff.total_seconds())
self.delaylist = self.delaylist[-1000:]
if len(self.delaylist) > 100:
try:
self.timedelay = np.median(np.asarray(self.delaylist))
except:
self.timedelay = 0.0
if delta > self.timethreshold:
self.errorcnt['time'] += 1
if self.errorcnt.get('time') < 2:
log.msg(
"{} protocol: large time difference observed for {}: {} sec"
.format(self.sensordict.get('protocol'), sensorid,
secdiff))
else:
self.errorcnt['time'] = 0
except:
pass
if self.sensordict.get('ptime', '') in ['NTP', 'ntp']:
secondtime = gpstime
maintime = timestamp
else:
maintime = gpstime
secondtime = timestamp
try:
if not typ == "none":
# extract time data
datearray = acs.timeToArray(maintime)
try:
datearray.append(int(intensity * 1000.))
if typ == 'base':
datearray.append(int(errorcode * 1000.))
else:
datearray.append(int(gradient * 1000.))
data_bin = struct.pack('<' + packcode, *datearray)
except:
log.msg(
'GSM19 - Protocol: Error while packing binary data')
pass
except:
log.msg('GSM19 - Protocol: Error with binary save routine')
pass
if not self.confdict.get('bufferdirectory', '') == '':
acs.dataToFile(self.confdict.get('bufferdirectory'), self.sensor,
filename, data_bin, header)
return ','.join(list(map(str, datearray))), header
def sendRequest(self):
"""
source:mysql:
Method to obtain data from table
"""
t1 = datetime.utcnow()
outdate = datetime.strftime(t1, "%Y-%m-%d")
filename = outdate
if self.debug:
log.msg(" -> DEBUG - Sending periodic request ...")
def getList(sql):
cursor = self.db.cursor()
try:
cursor.execute(sql)
except mysql.IntegrityError as message:
return message
except mysql.Error as message:
return message
except:
return 'dbgetlines: unkown error'
head = cursor.fetchall()
keys = list(np.transpose(np.asarray(head))[0])
return keys
# get self.sensorlist
# get last timestamps
# read all data for each sensor since last timestamp
# send that and store last timestamp
for index,sensdict in enumerate(self.sensorlist):
sensorid = sensdict.get('sensorid')
if self.debug:
log.msg(" -> DEBUG - dealing with sensor {}".format(sensorid))
# 1. Getting header
# -----------------
# load keys, elements and units
#header = "# MagPyBin %s %s %s %s %s %s %d" % (sensorid, key, ele, unit, multplier, packcode, struct.calcsize('<'+packcode))
dataid = sensorid+'_'+self.revision
keyssql = 'SHOW COLUMNS FROM %s' % (dataid)
keystab = getList(keyssql)
if 'time' in keystab:
keystab.remove('time')
if 'flag' in keystab:
keystab.remove('flag')
if 'typ' in keystab:
keystab.remove('typ')
if 'comment' in keystab:
keystab.remove('comment')
keys = ','.join(keystab)
if self.debug:
log.msg(" -> DEBUG - requesting header {}".format(sensorid))
sql1 = 'SELECT SensorElements FROM SENSORS WHERE SensorID LIKE "{}"'.format(sensorid)
sql2 = 'SELECT Sensorkeys FROM SENSORS WHERE SensorID LIKE "{}"'.format(sensorid)
sql3 = 'SELECT ColumnUnits FROM DATAINFO WHERE SensorID LIKE "{}"'.format(sensorid)
sql4 = 'SELECT ColumnContents FROM DATAINFO WHERE SensorID LIKE "{}"'.format(sensorid)
try:
elem = getList(sql1)[0].split(',')
except:
elem =[]
try:
keyssens = getList(sql2)[0].split(',')
except:
keyssens =[]
try:
unit = getList(sql3)[0].split(',')
except:
unit =[]
try:
cont = getList(sql4)[0].split(',')
except:
cont =[]
units, elems = [], []
for key in keystab:
try:
pos1 = keyssens.index(key)
ele = elem[pos1]
except:
ele = key
elems.append(ele)
try:
pos2 = cont.index(ele)
units.append(unit[pos2])
except:
units.append('None')
if self.debug:
log.msg(" -> DEBUG - creating head line {}".format(sensorid))
multplier = '['+','.join(map(str, [10000]*len(keystab)))+']'
packcode = '6HL'+''.join(['q']*len(keystab))
header = ("# MagPyBin {} {} {} {} {} {} {}".format(sensorid, '['+','.join(keystab)+']', '['+','.join(elems)+']', '['+','.join(units)+']', multplier, packcode, struct.calcsize('<'+packcode)))
# 2. Getting dict
sql = 'SELECT DataSamplingRate FROM DATAINFO WHERE SensorID LIKE "{}"'.format(sensorid)
sr = float(getList(sql)[0])
coverage = int(self.requestrate/sr)+120
# 3. Getting data
# get data and create typical message topic
# based on sampling rate and collection rate -> define coverage
li = sorted(mdb.dbselect(self.db, 'time,'+keys, dataid, expert='ORDER BY time DESC LIMIT {}'.format(int(coverage))))
if not self.lastt[index]:
self.lastt[index]=li[0][0]
# drop
newdat = False
newli = []
for elem in li:
if elem[0] == self.lastt[index]:
newdat = True
if newdat:
newli.append(elem)
if not len(newli) > 0:
# if last time not included in li then newli will be empty
# in this case just add the list
for elem in li:
newli.append(elem)
for dataline in newli:
timestamp = dataline[0]
data_bin = None
datearray = ''
try:
datearray = acs.timeToArray(timestamp)
for i,para in enumerate(keystab):
try:
val=int(float(dataline[i+1])*10000)
except:
val=999990000
datearray.append(val)
data_bin = struct.pack('<'+packcode,*datearray) # little endian
except:
log.msg('Error while packing binary data')
if not self.confdict.get('bufferdirectory','') == '' and data_bin:
acs.dataToFile(self.confdict.get('bufferdirectory'), sensorid, filename, data_bin, header)
if self.debug:
log.msg(" -> DEBUG - sending ... {}".format(','.join(list(map(str,datearray))), header))
self.sendData(sensorid,','.join(list(map(str,datearray))),header,len(newli)-1)
self.lastt[index]=li[-1][0]
t2 = datetime.utcnow()
if self.debug:
log.msg(" -> DEBUG - Needed {}".format(t2-t1))
def processPos1Data(self, data):
"""Convert raw ADC counts into SI units as per datasheets"""
if len(data) != 44:
log.err('POS1 - Protocol: Unable to parse data of length %i' % len(data))
currenttime = datetime.utcnow()
outdate = datetime.strftime(currenttime, "%Y-%m-%d")
actualtime = datetime.strftime(currenttime, "%Y-%m-%dT%H:%M:%S.%f")
outtime = datetime.strftime(currenttime, "%H:%M:%S")
timestamp = datetime.strftime(currenttime, "%Y-%m-%d %H:%M:%S.%f")
filename = outdate
sensorid = self.sensor
packcode = '6hLLLh6hL'
header = "# MagPyBin %s %s %s %s %s %s %d" % (self.sensor, '[f,df,var1,sectime]', '[f,df,var1,GPStime]', '[nT,nT,none,none]', '[1000,1000,1,1]', packcode, struct.calcsize('<'+packcode))
try:
# Extract data
data_array = data.split()
intensity = float(data_array[0])/1000.
sigma_int = float(data_array[2])/1000.
err_code = int(data_array[3].strip('[').strip(']'))
dataelements = datetime.strptime(data_array[4],"%m-%d-%y")
newdate = datetime.strftime(dataelements,"%Y-%m-%d")
gps_time = newdate + ' ' + str(data_array[5])[:11]
except:
log.err('POS1 - Protocol: Data formatting error.')
intensity = 0.0
sigma_int = 0.0
err_code = 0.0
try:
# Analyze time difference between POS1 internal time and utc from PC
# Please note that the time difference between POS1-GPS (data recorded)
# and NTP (data received at PC) can be very large
# for our POS1 it is 6.2 seconds
gpstime = datetime.strptime(gps_time, "%Y-%m-%d %H:%M:%S.%f")
timelist = sorted([gpstime,currenttime])
timediff = timelist[1]-timelist[0]
delta = timediff.total_seconds()
if not delta in [0.0, np.nan, None]:
self.delaylist.append(delta)
self.delaylist = self.delaylist[-1000:]
if len(self.delaylist) > 100:
try:
self.timedelay = np.median(np.asarray(self.delaylist))
except:
self.timedelay = 0.0
if delta-self.ntp_gps_offset > self.timethreshold:
self.errorcnt['time'] +=1
if self.errorcnt.get('time') < 2:
log.msg("{} protocol: large time difference observed for {}: {} sec".format(self.sensordict.get('protocol'), sensorid, secdiff))
else:
self.errorcnt['time'] = 0
except:
pass
if self.sensordict.get('ptime','') in ['NTP','ntp']:
secondtime = gps_time
maintime = timestamp
else:
maintime = gps_time
secondtime = timestamp
try:
# extract time data
datearray = acs.timeToArray(maintime)
sectarray = acs.timeToArray(secondtime)
try:
datearray.append(int(intensity*1000))
datearray.append(int(sigma_int*1000))
datearray.append(err_code)
datearray.extend(sectarray)
data_bin = struct.pack('<'+packcode,datearray[0],datearray[1],datearray[2],datearray[3],datearray[4],datearray[5],datearray[6],datearray[7],datearray[8],datearray[9],datearray[10],datearray[11],datearray[12],datearray[13],datearray[14],datearray[15],datearray[16])
except:
log.msg('POS1 - Protocol: Error while packing binary data')
pass
if not self.confdict.get('bufferdirectory','') == '':
acs.dataToFile(self.confdict.get('bufferdirectory'), sensorid, filename, data_bin, header)
except:
log.msg('POS1 - Protocol: Error with binary save routine')
pass
return ','.join(list(map(str,datearray))), header
