def processArduinoData(self, sensorid, meta, data):
"""Convert raw ADC counts into SI units as per datasheets"""
currenttime = datetime.utcnow()
outdate = datetime.strftime(currenttime, "%Y-%m-%d")
actualtime = datetime.strftime(currenttime, "%Y-%m-%dT%H:%M:%S.%f")
outtime = datetime.strftime(currenttime, "%H:%M:%S")
timestamp = datetime.strftime(currenttime, "%Y-%m-%d %H:%M:%S.%f")
filename = outdate
datearray = acs.timeToArray(timestamp)
packcode = '6hL'
#sensorid = self.sensordict.get(idnum)
#events = self.eventdict.get(idnum).replace('evt','').split(',')[3:-1]
values = []
multiplier = []
for dat in data:
try:
values.append(float(dat))
datearray.append(int(float(dat) * 10000))
packcode = packcode + 'l'
multiplier.append(10000)
except:
log.msg(
'{} protocol: Error while appending data to file (non-float?): {}'
.format(self.sensordict.get('protocol'), dat))
try:
data_bin = struct.pack('<' + packcode, *datearray) #little endian
except:
log.msg('{} protocol: Error while packing binary data'.format(
self.sensordict.get('protocol')))
pass
key = '[' + str(meta.get('SensorKeys')).replace("'", "").strip() + ']'
ele = '[' + str(meta.get('SensorElements')).replace("'",
"").strip() + ']'
unit = '[' + str(meta.get('SensorUnits')).replace("'",
"").strip() + ']'
multplier = str(multiplier).replace(" ", "")
header = "# MagPyBin %s %s %s %s %s %s %d" % (
sensorid, key, ele, unit, multplier, packcode,
struct.calcsize('<' + packcode))
if not self.confdict.get('bufferdirectory', '') == '':
acs.dataToFile(self.confdict.get('bufferdirectory'), sensorid,
filename, data_bin, header)
return ','.join(list(map(str, datearray))), header
def processData(self, data):
"""Convert raw ADC counts into SI units as per datasheets"""
currenttime = datetime.utcnow()
# Correction for ms time to work with databank:
currenttime_ms = currenttime.microsecond / 1000000.
ms_rounded = round(float(currenttime_ms), 3)
if not ms_rounded >= 1.0:
currenttime = currenttime.replace(microsecond=int(ms_rounded *
1000000.))
else:
currenttime = currenttime.replace(microsecond=0) + timedelta(
seconds=1.0)
filename = datetime.strftime(currenttime, "%Y-%m-%d")
actualtime = datetime.strftime(currenttime, "%Y-%m-%dT%H:%M:%S.%f")
lastActualtime = currenttime
outtime = datetime.strftime(currenttime, "%H:%M:%S")
timestamp = datetime.strftime(currenttime, "%Y-%m-%d %H:%M:%S.%f")
sensorid = self.sensor
packcode = '6hLL'
header = "# MagPyBin %s %s %s %s %s %s %d" % (
self.sensor, '[f]', '[f]', '[nT]', '[1000]', packcode,
struct.calcsize('<' + packcode))
try:
intval = data[1].split(',')
value = float(intval[0].strip())
if 10000 < value < 100000:
intensity = value
else:
intensity = 88888.0
except ValueError:
log.err("CS - Protocol: Not a number. Instead found:", data[0])
intensity = 88888.0
try:
datearray = acs.timeToArray(timestamp)
datearray.append(int(intensity * 1000))
data_bin = struct.pack('<' + packcode, *datearray)
except:
log.msg('Error while packing binary data')
if not self.confdict.get('bufferdirectory', '') == '':
acs.dataToFile(self.confdict.get('bufferdirectory'), sensorid,
filename, data_bin, header)
return ','.join(list(map(str, datearray))), header
def processData(self, data):
"""Process Environment data """
currenttime = datetime.utcnow()
outdate = datetime.strftime(currenttime, "%Y-%m-%d")
filename = outdate
actualtime = datetime.strftime(currenttime, "%Y-%m-%dT%H:%M:%S.%f")
outtime = datetime.strftime(currenttime, "%H:%M:%S")
timestamp = datetime.strftime(currenttime, "%Y-%m-%d %H:%M:%S.%f")
packcode = '6hLllL'
sensorid = self.sensor
header = "# MagPyBin %s %s %s %s %s %s %d" % (
sensorid, '[t1,t2,var1]', '[T,DewPoint,RH]', '[degC,degC,per]',
'[1000,1000,1000]', packcode, struct.calcsize('<' + packcode))
valrh = re.findall(r'\d+', data[0])
if len(valrh) > 1:
temp = float(valrh[0] + '.' + valrh[1])
else:
temp = float(valrh[0])
valrh = re.findall(r'\d+', data[1])
if len(valrh) > 1:
rh = float(valrh[0] + '.' + valrh[1])
else:
rh = float(valrh[0])
valrh = re.findall(r'\d+', data[2])
if len(valrh) > 1:
dew = float(valrh[0] + '.' + valrh[1])
else:
dew = float(valrh[0])
try:
datearray = acs.timeToArray(timestamp)
datearray.append(int(temp * 1000))
datearray.append(int(dew * 1000))
datearray.append(int(rh * 1000))
data_bin = struct.pack('<' + packcode,
*datearray) #use little endian byte order
except:
log.msg('Error while packing binary data')
pass
if not self.confdict.get('bufferdirectory', '') == '':
acs.dataToFile(self.confdict.get('bufferdirectory'), sensorid,
filename, data_bin, header)
return ','.join(list(map(str, datearray))), header
def processOwData(self, sensorid, datadict):
"""Process OW data """
currenttime = datetime.utcnow()
outdate = datetime.strftime(currenttime, "%Y-%m-%d")
filename = outdate
actualtime = datetime.strftime(currenttime, "%Y-%m-%dT%H:%M:%S.%f")
outtime = datetime.strftime(currenttime, "%H:%M:%S")
timestamp = datetime.strftime(currenttime, "%Y-%m-%d %H:%M:%S.%f")
packcode = '6hL' + 'l' * len(datadict)
multplier = str([1000] * len(datadict)).replace(' ', '')
if sensorid.startswith('DS18'):
key = '[t1]'
ele = '[T]'
unit = '[degC]'
elif sensorid.startswith('DS2438'):
#'temperature','VAD','VDD','humidity','vis'
key = '[t1,var1,var2,var3,var4]'
ele = '[T,RH,VDD,VAD,VIS]'
unit = '[degC,per,V,V,V,V]'
header = "# MagPyBin %s %s %s %s %s %s %d" % (
sensorid, key, ele, unit, multplier, packcode,
struct.calcsize('<' + packcode))
data_bin = None
datearray = ''
try:
datearray = acs.timeToArray(timestamp)
paralst = typedef.get(sensorid.split('_')[0])
for para in paralst:
if para in datadict:
datearray.append(int(float(datadict[para]) * 1000))
data_bin = struct.pack('<' + packcode,
*datearray) # little endian
except:
log.msg('Error while packing binary data')
if not self.confdict.get('bufferdirectory', '') == '' and data_bin:
acs.dataToFile(self.confdict.get('bufferdirectory'), sensorid,
filename, data_bin, header)
#print ("Sending", ','.join(list(map(str,datearray))), header)
return ','.join(list(map(str, datearray))), header
def processData(self, data):
currenttime = datetime.utcnow()
date = datetime.strftime(currenttime, "%Y-%m-%d")
actualtime = datetime.strftime(currenttime, "%Y-%m-%dT%H:%M:%S.%f")
outtime = datetime.strftime(currenttime, "%H:%M:%S")
filename = date
timestamp = datetime.strftime(currenttime, "%Y-%m-%d %H:%M:%S.%f")
intensity = 88888.8
typ = "none"
dontsavedata = False
packcode = '6hLLl'
header = "# MagPyBin %s %s %s %s %s %s %d" % (
self.sensor, '[f,var1]', '[f,err]', '[nT,none]', '[1000,1000]',
packcode, struct.calcsize('<' + packcode))
try:
# Extract data
data_array = data
if len(data_array) == 2:
typ = "oldbase"
elif len(data_array) == 3:
typ = "valid"
# add other types here
except:
log.err(
'GSM19 - Protocol: Output format not supported - use either base, ... or mobile'
)
# Extracting the data from the station
# Extrat time info and use as primary if GPS is on (in this case PC time is secondary)
# PC is primary when a GPS is not connected
if typ == "valid" or typ == "oldbase": # Comprises Mobile and Base Station mode with single sensor and no GPS
intensity = float(data_array[1])
try:
systemtime = datetime.strptime(date + "-" + data_array[0],
"%Y-%m-%d-%H%M%S.%f")
except:
# This exception happens for old GSM19 because time is
# provided e.g. as 410356 instead of 170356 for 17:03:56 (Thursday)
# e.g 570301.0 instead of 09:03:01 (Friday)
try:
hournum = int(data_array[0][:-6])
rest = data_array[0][-6:]
factor = np.floor(hournum /
24.) # factor = days since starting
hour = int(hournum - factor * 24.)
systemtime = datetime.strptime(
date + "-" + str(hour) + rest, "%Y-%m-%d-%H%M%S.%f")
#print ("Got oldbase systemtime")
except:
systemtime = currenttime
self.timesource = 'NTP'
if len(data_array) == 2:
typ = "base"
errorcode = 99
elif len(data_array[2]) == 3:
typ = "base"
errorcode = int(data_array[2])
else:
typ = "gradient"
gradient = float(data_array[2])
elif typ == "none":
dontsavedata = True
pass
gpstime = datetime.strftime(systemtime, "%Y-%m-%d %H:%M:%S.%f")
try:
# Analyze time difference between GSM internal time and utc from PC
timelist = sorted([systemtime, currenttime])
timediff = timelist[1] - timelist[0]
#secdiff = timediff.seconds + timediff.microseconds/1E6
delta = timediff.total_seconds()
if not delta in [0.0, np.nan, None]:
self.delaylist.append(timediff.total_seconds())
self.delaylist = self.delaylist[-1000:]
if len(self.delaylist) > 100:
try:
self.timedelay = np.median(np.asarray(self.delaylist))
except:
self.timedelay = 0.0
if delta > self.timethreshold:
self.errorcnt['time'] += 1
if self.errorcnt.get('time') < 2:
log.msg(
"{} protocol: large time difference observed for {}: {} sec"
.format(self.sensordict.get('protocol'), sensorid,
secdiff))
else:
self.errorcnt['time'] = 0
except:
pass
if self.sensordict.get('ptime', '') in ['NTP', 'ntp']:
secondtime = gpstime
maintime = timestamp
else:
maintime = gpstime
secondtime = timestamp
try:
if not typ == "none":
# extract time data
datearray = acs.timeToArray(maintime)
try:
datearray.append(int(intensity * 1000.))
if typ == 'base':
datearray.append(int(errorcode * 1000.))
else:
datearray.append(int(gradient * 1000.))
data_bin = struct.pack('<' + packcode, *datearray)
except:
log.msg(
'GSM19 - Protocol: Error while packing binary data')
pass
except:
log.msg('GSM19 - Protocol: Error with binary save routine')
pass
if not self.confdict.get('bufferdirectory', '') == '':
acs.dataToFile(self.confdict.get('bufferdirectory'), self.sensor,
filename, data_bin, header)
return ','.join(list(map(str, datearray))), header
def on_message(client, userdata, msg):
global verifiedlocation
arrayinterpreted = False
sensorid = msg.topic.strip(stationid).replace('/','').strip('meta').strip('data').strip('dict')
# define a new data stream for each non-existing sensor
metacheck = identifier.get(sensorid+':packingcode','')
if msg.topic.endswith('meta') and metacheck == '':
print ("Found basic header:{}".format(str(msg.payload)))
print ("Quality od Service (QOS):{}".format(str(msg.qos)))
analyse_meta(str(msg.payload),sensorid)
if not sensorid in headdict:
headdict[sensorid] = msg.payload
# create stream.header dictionary and it here
headstream[sensorid] = create_head_dict(str(msg.payload),sensorid)
if debug:
print ("New headdict", headdict)
elif msg.topic.endswith('dict') and sensorid in headdict:
#print ("Found Dictionary:{}".format(str(msg.payload)))
head_dict = headstream[sensorid]
for elem in str(msg.payload).split(','):
keyvaluespair = elem.split(':')
try:
if not keyvaluespair[1] in ['-','-\n','-\r\n']:
head_dict[keyvaluespair[0]] = keyvaluespair[1].strip()
except:
pass
if debug:
print ("Dictionary now looks like", headstream[sensorid])
elif msg.topic.endswith('data'):
#if debug:
# print ("Found data:", str(msg.payload), metacheck)
if not metacheck == '':
if 'file' in destination:
# Import module for writing data from acquistion
# -------------------
#if debug:
# print (sensorid, metacheck, msg.payload) # payload can be split
# Check whether header is already identified
# -------------------
if sensorid in headdict:
header = headdict.get(sensorid)
packcode = metacheck.strip('<')[:-1] # drop leading < and final B
arrayelem = msg.payload.split(';')
for ar in arrayelem:
datearray = ar.split(',')
# identify string values in packcode
# -------------------
if not 's' in packcode:
datearray = list(map(int, datearray))
else:
stringidx = []
for i in range(len(packcode)):
if packcode[-i] == 's':
stringidx.append(i)
for i in range(len(datearray)):
if not i in stringidx:
datearray[-i] = int(datearray[-i])
# pack data using little endian byte order
data_bin = struct.pack('<'+packcode,*datearray)
# Check whether destination path has been verified already
# -------------------
if not verifiedlocation:
if not location in [None,''] and os.path.exists(location):
verifiedlocation = True
else:
print ("File: destination location {} is not accessible".format(location))
print (" -> please use option l (e.g. -l '/my/path') to define")
if verifiedlocation:
filename = "{}-{:02d}-{:02d}".format(datearray[0],datearray[1],datearray[2])
dataToFile(location, sensorid, filename, data_bin, header)
if 'stdout' in destination:
if not arrayinterpreted:
stream.ndarray = interprete_data(msg.payload, identifier, stream, sensorid)
#streamdict[sensorid] = stream.ndarray # to store data from different sensors
arrayinterpreted = True
for idx,el in enumerate(stream.ndarray[0]):
time = num2date(el).replace(tzinfo=None)
datastring = ','.join([str(val[idx]) for i,val in enumerate(stream.ndarray) if len(val) > 0 and not i == 0])
print ("{}: {},{}".format(sensorid,time,datastring))
if 'websocket' in destination:
if not arrayinterpreted:
stream.ndarray = interprete_data(msg.payload, identifier, stream, sensorid)
#streamdict[sensorid] = stream.ndarray # to store data from different sensors
arrayinterpreted = True
for idx,el in enumerate(stream.ndarray[0]):
time = num2date(el).replace(tzinfo=None)
datastring = ','.join([str(val[idx]) for i,val in enumerate(stream.ndarray) if len(val) > 0 and not i == 0])
wsserver.send_message_to_all("{}: {},{}".format(sensorid,time,datastring))
elif 'db' in destination:
if not arrayinterpreted:
stream.ndarray = interprete_data(msg.payload, identifier, stream, sensorid)
#streamdict[sensorid] = stream.ndarray # to store data from different sensors
arrayinterpreted = True
# create a stream.header
#if debug:
# print (stream.ndarray)
stream.header = headstream[sensorid]
#print ("header", stream.header)
writeDB(db,stream)
#sys.exit()
elif 'stringio' in destination:
if not arrayinterpreted:
stream.ndarray = interprete_data(msg.payload, identifier, stream, sensorid)
#streamdict[sensorid] = stream.ndarray # to store data from different sensors
arrayinterpreted = True
for idx,el in enumerate(stream.ndarray[0]):
time = num2date(el).replace(tzinfo=None)
date = datetime.strftime(time,"%Y-%m-%d %H:%M:%S.%f")
linelist = list(map(str,[el,date]))
linelist.extend([str(val[idx]) for i,val in enumerate(stream.ndarray) if len(val) > 0 and not i == 0])
line = ','.join(linelist)
eol = '\r\n'
output.write(line+eol)
else:
pass
else:
print(msg.topic + " " + str(msg.payload))
def sendRequest(self):
# TODO wohin mit debug?
debug = False
if self.reconnect.is_set():
log.msg('exiting, mutex locked!')
return
t = datetime.utcnow()
past = t - timedelta(seconds=3)
vals = self.device.get_data('SamplesEvery2s', past, t)
# vals[0] because we grap no older data, there is only one value in 2 seconds
# timestamp directly from datetime into array
# TODO Roman fragen, ob oder wie Vergleich mit Computerzeit
try:
darray = datetime2array(vals[0]['Datetime'])
# TODO "again" ist Provisorium
again = False
except:
again = True
try:
if again:
t = datetime.utcnow()
past = t - timedelta(seconds=3)
darray = datetime2array(vals[0]['Datetime'])
log.msg("IT TOOK A SECOND TIME TO GET DATA PROPERLY!")
except:
# there will be no log messages when the logger is turned off
return
# TODO reconnect Loesung, nur wenn sie sauber funktioniert!
log.msg('NO DATA FROM CR1000 !!! - vals:')
log.msg(vals)
port = self.confdict['serialport'] + self.sensordict.get(
'port')
baudrate = self.sensordict.get('baudrate')
self.reconnect.set()
connected = False
while not connected:
self.device.bye()
log.msg('reconnecting to device...')
time.sleep(5)
try:
self.device = CR1000.from_url('serial:{}:{}'.format(
port, baudrate))
tables = self.device.list_tables()
if tables == [
'Status', 'SamplesEvery2s',
'ValuesEveryMinute', 'Public'
]:
connected = True
log.msg('schaut ok aus...')
time.sleep(2)
except:
log.msg('reconnect wohl missglueckt!')
try:
past = t - timedelta(seconds=3)
vals = self.device.get_data('SamplesEvery2s',
past,
t,
debug=True)
log.msg(SENSOR_HEIGHT * 1000. -
vals[0]['DiffVolt'] * 250.)
except:
log.msg('...wohl doch nicht!')
connected = False
self.reconnect.clear()
log.msg('mutex released...')
return
if debug:
log.msg('getting data...')
# snowheight (1000mV is 250cm) - values from CR1000 in mV, factor 1000 for packing
snowheight = SENSOR_HEIGHT * 1000. - vals[0]['DiffVolt'] * 250.
darray.append(int(round(snowheight)))
# TODO weg
if debug:
log.msg(darray)
# preparations for file save
# date 6 short microsecond unsigned long snowheight signed long
# TODO alter packcode!
packcode = "6hLl"
#packcode = "<6hLl"
# header
sensorid = self.sensordict['sensorid']
header = "# MagPyBin %s %s %s %s %s %s %d" % (
sensorid, '[f]', '[JC]', '[cm]', '[1000]', packcode,
struct.calcsize(packcode))
data_bin = struct.pack(packcode, *darray)
# date of dataloggers timestamp
filedate = datetime.strftime(
datetime(darray[0], darray[1], darray[2]), "%Y-%m-%d")
if not self.confdict.get('bufferdirectory', '') == '':
acs.dataToFile(self.confdict.get('bufferdirectory'), sensorid,
filedate, data_bin, header)
if debug:
log.msg('Daten gesichert...')
# sending via MQTT
data = ','.join(list(map(str, darray)))
head = header
topic = self.confdict.get('station') + '/' + self.sensordict.get(
'sensorid')
coll = int(self.sensordict.get('stack'))
if coll > 1:
self.metacnt = 1 # send meta data with every block
if self.datacnt < coll:
self.datalst.append(data)
self.datacnt += 1
else:
senddata = True
data = ';'.join(self.datalst)
self.datalst = []
self.datacnt = 0
else:
senddata = True
if senddata:
self.client.publish(topic + "/data", data)
if self.count == 0:
self.client.publish(topic + "/meta", head)
self.count += 1
if self.count >= self.metacnt:
self.count = 0
# right now auxiliary data only in the log file
if t.second < 2:
# every minute aux data (battery voltage and logger temperature) will be available
# going 61s into the past to make sure there are already data
past = t - timedelta(seconds=61)
aux = self.device.get_data('ValuesEveryMinute', past, t)
log.msg('----- aux every minute:')
# timestamp directly from datetime into array
try:
darray = datetime2array(aux[0]['Datetime'])
except:
# following should never happen...
log.msg('AUXILIARY DATA NOT GOT PROPERLY! - aux:')
log.msg(aux)
log.msg('trying again...')
past = t - timedelta(seconds=62)
aux = self.device.get_data('ValuesEveryMinute', past, t)
try:
darray = datetime2array(aux[0]['Datetime'])
except:
log.msg('giving up...')
return
# battery voltage - factor 1000 for packing
BattV_Min = int(round(aux[0]['BattV_Min'] * 1000))
PTemp_C_Avg = int(round(aux[0]['PTemp_C_Avg'] * 1000))
darray.extend([BattV_Min, PTemp_C_Avg])
# alernative reading:
#aux = (aux.filter(('Datetime', 'BattV_Min','PTemp_C_Avg')).to_csv(header=False))
log.msg(darray)
packcode = "<6hLLl"
def processPos1Data(self, data):
"""Convert raw ADC counts into SI units as per datasheets"""
if len(data) != 44:
log.err('POS1 - Protocol: Unable to parse data of length %i' % len(data))
currenttime = datetime.utcnow()
outdate = datetime.strftime(currenttime, "%Y-%m-%d")
actualtime = datetime.strftime(currenttime, "%Y-%m-%dT%H:%M:%S.%f")
outtime = datetime.strftime(currenttime, "%H:%M:%S")
timestamp = datetime.strftime(currenttime, "%Y-%m-%d %H:%M:%S.%f")
filename = outdate
sensorid = self.sensor
packcode = '6hLLLh6hL'
header = "# MagPyBin %s %s %s %s %s %s %d" % (self.sensor, '[f,df,var1,sectime]', '[f,df,var1,GPStime]', '[nT,nT,none,none]', '[1000,1000,1,1]', packcode, struct.calcsize('<'+packcode))
try:
# Extract data
data_array = data.split()
intensity = float(data_array[0])/1000.
sigma_int = float(data_array[2])/1000.
err_code = int(data_array[3].strip('[').strip(']'))
dataelements = datetime.strptime(data_array[4],"%m-%d-%y")
newdate = datetime.strftime(dataelements,"%Y-%m-%d")
gps_time = newdate + ' ' + str(data_array[5])[:11]
except:
log.err('POS1 - Protocol: Data formatting error.')
intensity = 0.0
sigma_int = 0.0
err_code = 0.0
try:
# Analyze time difference between POS1 internal time and utc from PC
# Please note that the time difference between POS1-GPS (data recorded)
# and NTP (data received at PC) can be very large
# for our POS1 it is 6.2 seconds
gpstime = datetime.strptime(gps_time, "%Y-%m-%d %H:%M:%S.%f")
timelist = sorted([gpstime,currenttime])
timediff = timelist[1]-timelist[0]
delta = timediff.total_seconds()
if not delta in [0.0, np.nan, None]:
self.delaylist.append(delta)
self.delaylist = self.delaylist[-1000:]
if len(self.delaylist) > 100:
try:
self.timedelay = np.median(np.asarray(self.delaylist))
except:
self.timedelay = 0.0
if delta-self.ntp_gps_offset > self.timethreshold:
self.errorcnt['time'] +=1
if self.errorcnt.get('time') < 2:
log.msg("{} protocol: large time difference observed for {}: {} sec".format(self.sensordict.get('protocol'), sensorid, secdiff))
else:
self.errorcnt['time'] = 0
except:
pass
if self.sensordict.get('ptime','') in ['NTP','ntp']:
secondtime = gps_time
maintime = timestamp
else:
maintime = gps_time
secondtime = timestamp
try:
# extract time data
datearray = acs.timeToArray(maintime)
sectarray = acs.timeToArray(secondtime)
try:
datearray.append(int(intensity*1000))
datearray.append(int(sigma_int*1000))
datearray.append(err_code)
datearray.extend(sectarray)
data_bin = struct.pack('<'+packcode,datearray[0],datearray[1],datearray[2],datearray[3],datearray[4],datearray[5],datearray[6],datearray[7],datearray[8],datearray[9],datearray[10],datearray[11],datearray[12],datearray[13],datearray[14],datearray[15],datearray[16])
except:
log.msg('POS1 - Protocol: Error while packing binary data')
pass
if not self.confdict.get('bufferdirectory','') == '':
acs.dataToFile(self.confdict.get('bufferdirectory'), sensorid, filename, data_bin, header)
except:
log.msg('POS1 - Protocol: Error with binary save routine')
pass
return ','.join(list(map(str,datearray))), header
def processData(self, data):
""" GSM90 data """
currenttime = datetime.utcnow()
outdate = datetime.strftime(currenttime, "%Y-%m-%d")
actualtime = datetime.strftime(currenttime, "%Y-%m-%dT%H:%M:%S.%f")
outtime = datetime.strftime(currenttime, "%H:%M:%S")
timestamp = datetime.strftime(currenttime, "%Y-%m-%d %H:%M:%S.%f")
filename = outdate
sensorid = self.sensor
packcode = '6hLLL6hL'
header = "# MagPyBin %s %s %s %s %s %s %d" % (
self.sensor, '[f,var1,sectime]', '[f,errorcode,internaltime]',
'[nT,none,none]', '[1000,1,1]', packcode,
struct.calcsize('<' + packcode))
try:
# Extract data
# old data looks like 04-22-2015 142244 48464.53 99
data_array = data
if len(data) == 4:
intensity = float(data[2])
err_code = int(data[3])
try:
try:
internal_t = datetime.strptime(data[0] + 'T' + data[1],
"%m-%d-%YT%H%M%S.%f")
except:
internal_t = datetime.strptime(data[0] + 'T' + data[1],
"%m-%d-%YT%H%M%S")
internal_time = datetime.strftime(internal_t,
"%Y-%m-%d %H:%M:%S.%f")
except:
internal_time = timestamp #datetime.strftime(datetime.utcnow(), "%Y-%m-%d %H:%M:%S.%f")
#print internal_time
elif len(data) == 3: # GSM v7.0
intensity = float(data[1])
err_code = int(data[2])
try:
internal_t = datetime.strptime(outdate + 'T' + data[0],
"%Y-%m-%dT%H%M%S.%f")
internal_time = datetime.strftime(internal_t,
"%Y-%m-%d %H:%M:%S.%f")
except:
internal_time = timestamp #datetime.strftime(datetime.utcnow(), "%Y-%m-%d %H:%M:%S.%f")
else:
err_code = 0
intensity = float(data[0])
internal_time = timestamp #datetime.strftime(datetime.utcnow(), "%Y-%m-%d %H:%M:%S")
except:
log.err('{} protocol: Data formatting error. Data looks like: {}'.
format(self.sensordict.get('protocol'), data))
try:
# Analyze time difference between GSM internal time and utc from PC
timelist = sorted([internal_t, currenttime])
timediff = timelist[1] - timelist[0]
#secdiff = timediff.seconds + timediff.microseconds/1E6
#timethreshold = 3
delta = timediff.total_seconds()
if not delta in [0.0, np.nan, None]:
self.delaylist.append(timediff.total_seconds())
self.delaylist = self.delaylist[-1000:]
if len(self.delaylist) > 100:
try:
self.timedelay = np.median(np.asarray(self.delaylist))
except:
self.timedelay = 0.0
#if secdiff > timethreshold:
if delta > self.timethreshold:
self.errorcnt['time'] += 1
if self.errorcnt.get('time') < 2:
log.msg(
"{} protocol: large time difference observed for {}: {} sec"
.format(self.sensordict.get('protocol'), sensorid,
secdiff))
else:
self.errorcnt['time'] = 0
except:
pass
if self.sensordict.get('ptime', '') in ['NTP', 'ntp']:
secondtime = internal_time
maintime = timestamp
else:
maintime = internal_time
secondtime = timestamp
try:
## GSM90 does not provide any info on whether the GPS reading is OK or not
# extract time data
datearray = acs.timeToArray(maintime)
try:
datearray.append(int(intensity * 1000.))
datearray.append(err_code)
#print timestamp, internal_time
internalarray = acs.timeToArray(secondtime)
datearray.extend(internalarray)
data_bin = struct.pack('<' + packcode, *datearray)
except:
log.msg('{} protocol: Error while packing binary data'.format(
self.sensordict.get('protocol')))
if not self.confdict.get('bufferdirectory', '') == '':
acs.dataToFile(self.confdict.get('bufferdirectory'), sensorid,
filename, data_bin, header)
except:
log.msg('{} protocol: Error with binary save routine'.format(
self.sensordict.get('protocol')))
return ','.join(list(map(str, datearray))), header