def GetData(source,
path,
db,
dbcredentials,
sensorid,
amount,
startdate=None,
debug=False):
"""
DESCRIPTION:
read the appropriate amount of data from the data file, database or mqtt stream
"""
data = DataStream()
msg = ''
if not startdate:
startdate = datetime.utcnow()
endtime = None
else:
endtime = startdate
starttime = startdate - timedelta(seconds=int(amount))
if source in ['file', 'File']:
filepath = os.path.join(path, sensorid)
# TODO eventually check for existance and look for similar names
# expath = CheckPath(filepath)
# if expath:
if debug:
print("Trying to access files in {}: Timerange: {} to {}".format(
filepath, starttime, endtime))
try:
data = read(os.path.join(filepath, '*'),
starttime=starttime,
endtime=endtime)
except:
msg = "Could not access data for sensorid {}".format(sensorid)
if debug:
print(msg)
elif source in ['db', 'DB', 'database', 'Database']:
db = mysql.connect()
data = readDB(db, sensorid, starttime=starttime)
if debug:
print("Got {} datapoints".format(data.length()[0]))
return (data, msg)
def GetTestValue(data=DataStream(), key='x', function='average', debug=False):
"""
DESCRIPTION
Returns comparison value(e.g. mean, max etc)
"""
if debug:
print("Obtaining test value for key {} with function {}".format(
key, function))
func = 'mean'
testvalue = None
msg = ''
n = data.length()[0]
keys = data._get_key_headers()
if not key in keys:
print("Requested key not found")
return (testvalue, 'failure')
if function in ['mean', 'Mean', 'average', 'Average', 'Median', 'median']:
if n < 3:
print(
"not enough data points --- {} insignificant".format(function))
if function in ['mean', 'Mean', 'average', 'Average']:
func = 'mean'
elif function in ['Median', 'median']:
func = 'median'
testvalue = data.mean(key, meanfunction=func)
elif function in ['max', 'Max']:
testvalue = data._get_max(key)
elif function in ['min', 'Min']:
testvalue = data._get_min(key)
elif function in ['stddev', 'Stddev']:
mean, testvalue = data.mean(key, std=True)
else:
msg = 'selected test function not available'
if debug:
print(" ... got {}".format(testvalue))
return (testvalue, msg)
def on_message(client, userdata, msg):
if pyversion.startswith('3'):
msg.payload = msg.payload.decode('ascii')
global qos
global verifiedlocation
arrayinterpreted = False
if stationid in ['all', 'All', 'ALL']:
stid = msg.topic.split('/')[0]
else:
stid = stationid
try:
sensorind = msg.topic.split('/')[1]
sensorid = sensorind.replace('meta',
'').replace('data',
'').replace('dict', '')
except:
# Above will fail if msg.topic does not contain /
# TODO (previous version was without 1, first occurrence -> the following line should work as well although the code above is more general)
sensorid = msg.topic.replace(stid, "", 1).replace('/', '').replace(
'meta', '').replace('data', '').replace('dict', '')
# define a new data stream for each non-existing sensor
if not instrument == '':
if not sensorid.find(instrument) > -1:
return
if sensorid in blacklist:
if debug:
print("Sensor {} in blacklist - not collecting".format(sensorid))
return
## ################################################################################
## #### Eventually check for additional format libraries #########
## ################################################################################
identdic = {}
if addlib and len(addlib) > 0:
# Currently only one additioal library is supported
lib = addlib[0]
#for lib in addlib:
elemlist = []
for elem in topic_identifiers[lib]:
strelem = "msg.topic.{}('{}')".format(elem,
topic_identifiers[lib][elem])
elemlist.append(strelem)
if len(elemlist) > 1:
teststring = " and ".join(elemlist)
else:
teststring = "".join(elemlist)
if eval(teststring):
classref = class_reference.get(lib)
#print ("1", msg.payload)
try:
msg.payload, sensorid, headerline, headerdictionary, identdic = classref.GetPayload(
msg.payload, msg.topic)
except:
print("Interpretation error for {}".format(msg.topic))
return
#print (payload, sensorid, headerline)
headdict[sensorid] = headerline
headstream[sensorid] = create_head_dict(headerline, sensorid)
headstream[sensorid] = merge_two_dicts(headstream[sensorid],
headerdictionary)
msg.topic = msg.topic + '/data'
for el in identdic:
po.identifier[el] = identdic[el]
metacheck = po.identifier.get(sensorid + ':packingcode', '')
## ################################################################################
if msg.topic.endswith('meta') and metacheck == '':
log.msg("Found basic header:{}".format(str(msg.payload)))
log.msg("Quality of Service (QOS):{}".format(str(msg.qos)))
analyse_meta(str(msg.payload), sensorid)
if not sensorid in headdict:
headdict[sensorid] = msg.payload
# create stream.header dictionary and it here
headstream[sensorid] = create_head_dict(str(msg.payload), sensorid)
if debug:
log.msg("New headdict: {}".format(headdict))
elif msg.topic.endswith('dict') and sensorid in headdict:
#log.msg("Found Dictionary:{}".format(str(msg.payload)))
head_dict = headstream[sensorid]
for elem in str(msg.payload).split(','):
keyvaluespair = elem.split(':')
try:
if not keyvaluespair[1] in ['-', '-\n', '-\r\n']:
head_dict[keyvaluespair[0]] = keyvaluespair[1].strip()
except:
pass
if debug:
log.msg("Dictionary now looks like {}".format(
headstream[sensorid]))
elif msg.topic.endswith('data'): # or readable json
#if readable json -> create stream.ndarray and set arrayinterpreted :
# log.msg("Found data:", str(msg.payload), metacheck)
if not metacheck == '':
if 'file' in destination:
# Import module for writing data from acquistion
# -------------------
#if debug:
# log.msg(sensorid, metacheck, msg.payload) # payload can be split
# Check whether header is already identified
# -------------------
if sensorid in headdict:
header = headdict.get(sensorid)
if sys.version_info >= (3, 0):
metacheck = metacheck.decode()
if metacheck.endswith('B'):
packcode = metacheck.strip(
'<')[:-1] # drop leading < and final B
else:
packcode = metacheck.strip('<') # drop leading <
# temporary code - too be deleted when lemi protocol has been updated
if packcode.find('4cb6B8hb30f3Bc') >= 0:
header = header.replace(
'<4cb6B8hb30f3BcBcc5hL 169\n',
'6hLffflll {}'.format(
struct.calcsize('<6hLffflll')))
packcode = '6hLffflll'
arrayelem = msg.payload.split(';')
for ar in arrayelem:
datearray = ar.split(',')
# identify string values in packcode
# -------------------
# convert packcode numbers
cpack = []
for c in packcode:
if c.isdigit():
digit = int(c)
else:
cpack.extend([c] * digit)
digit = 1
cpackcode = "".join(cpack)
for i in range(len(cpackcode)):
if cpackcode[-i] == 's':
datearray[-i] = datearray[-i]
elif cpackcode[-i] == 'f':
datearray[-i] = float(datearray[-i])
else:
datearray[-i] = int(float(datearray[-i]))
# pack data using little endian byte order
data_bin = struct.pack('<' + packcode, *datearray)
# Check whether destination path has been verified already
# -------------------
if not verifiedlocation:
if not location in [None, ''
] and os.path.exists(location):
verifiedlocation = True
else:
log.msg(
"File: destination location {} is not accessible"
.format(location))
log.msg(
" -> please use option l (e.g. -l '/my/path') to define"
)
if verifiedlocation:
filename = "{}-{:02d}-{:02d}".format(
datearray[0], datearray[1], datearray[2])
acs.dataToFile(location, sensorid, filename,
data_bin, header)
if 'websocket' in destination:
if not arrayinterpreted:
stream.ndarray = interprete_data(msg.payload, stream,
sensorid)
#streamdict[sensorid] = stream.ndarray # to store data from different sensors
arrayinterpreted = True
for idx, el in enumerate(stream.ndarray[0]):
time = num2date(el).replace(tzinfo=None)
msecSince1970 = int(
(time - datetime(1970, 1, 1)).total_seconds() * 1000)
datastring = ','.join([
str(val[idx]) for i, val in enumerate(stream.ndarray)
if len(val) > 0 and not i == 0
])
if debug:
print("Sending {}: {},{} to webserver".format(
sensorid, msecSince1970, datastring))
wsserver.send_message_to_all("{}: {},{}".format(
sensorid, msecSince1970, datastring))
if 'diff' in destination:
global counter
counter += 1
global number
amount = int(number)
cover = 5
if not arrayinterpreted:
ar = interprete_data(msg.payload, stream, sensorid)
if not sensorid in senslst:
senslst.append(sensorid)
st.append(DataStream([], {}, ar))
idx = senslst.index(sensorid)
st[idx].extend(stream.container, {'SensorID': sensorid},
ar)
arrayinterpreted = True
st[idx].ndarray = np.asarray(
[np.asarray(el[-cover:]) for el in st[idx].ndarray])
if len(st) < 2:
print("Not enough streams for subtraction yet")
try:
if counter > amount:
counter = 0
sub = subtractStreams(st[0], st[1])
try:
part1 = (
st[0].header.get('SensorID').split('_')[1])
except:
part1 = 'unkown'
try:
part2 = (
st[1].header.get('SensorID').split('_')[1])
except:
part2 = 'unkown'
name = "Diff_{}-{}_0001".format(part1, part2)
# get head line for pub
#name = "diff_xxx_0001"
keys = sub._get_key_headers(numerical=True)
ilst = [KEYLIST.index(key) for key in keys]
keystr = "[{}]".format(",".join(keys))
#takeunits = ### take from st[0]
packcode = "6hL{}".format("".join(['l'] * len(keys)))
multi = "[{}]".format(",".join(['1000'] * len(keys)))
unit = "[{}]".format(",".join(['arb'] * len(keys)))
head = "# MagPyBin {} {} {} {} {} {} {}".format(
name, keystr, keystr, unit, multi, packcode,
struct.calcsize('<' + packcode))
#print (head)
# get data line for pub
time = sub.ndarray[0][-1]
timestr = (datetime.strftime(
num2date(float(time)).replace(tzinfo=None),
"%Y,%m,%d,%H,%M,%S,%f"))
val = [sub.ndarray[i][-1] for i in ilst]
if len(val) > 1:
valstr = ",".join(int(val * 1000))
else:
valstr = int(val[0] * 1000)
data = "{},{}".format(timestr, valstr)
#print (data)
topic = "wic/{}".format(name)
client.publish(topic + "/data", data, qos=qos)
client.publish(topic + "/meta", head, qos=qos)
except:
print("Found error in subtraction")
if 'stdout' in destination:
if not arrayinterpreted:
stream.ndarray = interprete_data(msg.payload, stream,
sensorid)
#streamdict[sensorid] = stream.ndarray # to store data from different sensors
arrayinterpreted = True
for idx, el in enumerate(stream.ndarray[0]):
time = num2date(el).replace(tzinfo=None)
datastring = ','.join([
str(val[idx]) for i, val in enumerate(stream.ndarray)
if len(val) > 0 and not i == 0
])
log.msg("{}: {},{}".format(sensorid, time, datastring))
elif 'db' in destination:
if not arrayinterpreted:
stream.ndarray = interprete_data(msg.payload, stream,
sensorid)
#streamdict[sensorid] = stream.ndarray # to store data from different sensors
arrayinterpreted = True
# create a stream.header
#if debug:
# log.msg(stream.ndarray)
stream.header = headstream[sensorid]
if debug:
log.msg("writing header: {}".format(headstream[sensorid]))
if revision != 'free':
writeDB(db,
stream,
tablename="{}_{}".format(sensorid, '0001'))
else:
writeDB(db, stream)
elif 'stringio' in destination:
if not arrayinterpreted:
stream.ndarray = interprete_data(msg.payload, stream,
sensorid)
#streamdict[sensorid] = stream.ndarray # to store data from different sensors
arrayinterpreted = True
for idx, el in enumerate(stream.ndarray[0]):
time = num2date(el).replace(tzinfo=None)
date = datetime.strftime(time, "%Y-%m-%d %H:%M:%S.%f")
linelist = list(map(str, [el, date]))
linelist.extend([
str(val[idx]) for i, val in enumerate(stream.ndarray)
if len(val) > 0 and not i == 0
])
line = ','.join(linelist)
eol = '\r\n'
output.write(line + eol)
elif 'serial' in destination:
if not arrayinterpreted:
stream.ndarray = interprete_data(msg.payload, stream,
sensorid)
#streamdict[sensorid] = stream.ndarray # to store data from different sensors
arrayinterpreted = True
"""
# send json like structures
collcount = 10
if sercount <= collcount:
for idx,col in enumerate(stream.ndarray):
if not len(col) == 0:
keyname = KEYLIST[idx]
if idx == 0:
time = num2date(col).replace(tzinfo=None)
col = int((time - datetime(1970,1,1)).total_seconds()*1000)
excol = datacol.get(keyname,[])
datacol[keyname] = excol.append(col)
sersount += 1
if sercount == collcount:
sercount = 0
jsonstr={}
jsonstr['sensorid'] = sensorid
jsonstr['nr'] = i
jsonstr['key'] = po.identifier[sensorid+':keylist'][i]
jsonstr['elem'] = po.identifier[sensorid+':elemlist'][i]
jsonstr['unit'] = po.identifier[sensorid+':unitlist'][i]
payload = json.dumps(jsonstr)
# write input to a another serial port (e.g. for radio transmisson)
# requires serdef = e.g. [115200,8,1,N]
# eventually create minimum 30 sec json blocks
#sendline='{"SensorID":"{}","Units":["Sec1970","degC"],"Keys":{"time":"time","x":"Temperature"}, "time":[{}],"x":[{}]}'.format(sensorid,time,x)
#{"SensorID":"ID","units":["Sec1970","degC"],"keys":{"time":"time","x":"Temperature"}, "data":{"time":[12,13,45],"x":[12,13,45]}}
#ser.write("{}: {},{}".format(sensorid,msecSince1970,datastring))
"""
else:
pass
else:
log.msg("Non-interpreted format: {} {}".format(
msg.topic, str(msg.payload)))
elif msg.topic.find('statuslog') > 0:
# json style statusinfo is coming
hostname = msg.topic.split('/')[-1]
#log.msg("---------------------------------------------------------------")
#log.msg("Receiving updated status information from {}".format(hostname))
#log.msg("---------------------------------------------------------------")
print("FOUND STATUS CHANGE", telegramconf)
statusdict = json.loads(msg.payload)
for elem in statusdict:
logmsg = "{}: {} - {}".format(hostname, elem, statusdict[elem])
# For Nagios - add in marcos.log
log.msg(logmsg)
# For Telegram
try:
# try to import telegram and telegram.cfg
##### Add the configuration to input and marcos.cfg
## Please note: requires anaconda2/bin/python on my test PC
## !!! Requires network connection !!!
if not telegramconf == '/telegram.conf':
martaslog = ml(receiver='telegram')
martaslog.receiveroptions('telegram',
options={'conf': telegramconf})
statusdict['Hostname'] = hostname
martaslog.notify(statusdict)
except:
pass
#telegram.send(msg)
if msg.topic.endswith('meta') and 'websocket' in destination:
# send header info for each element (# sensorid nr key elem unit)
analyse_meta(str(msg.payload), sensorid)
for (i, void) in enumerate(po.identifier[sensorid + ':keylist']):
jsonstr = {}
jsonstr['sensorid'] = sensorid
jsonstr['nr'] = i
jsonstr['key'] = po.identifier[sensorid + ':keylist'][i]
jsonstr['elem'] = po.identifier[sensorid + ':elemlist'][i]
jsonstr['unit'] = po.identifier[sensorid + ':unitlist'][i]
payload = json.dumps(jsonstr)
wsserver.send_message_to_all('# ' + payload)
# available since MagPy 0.3.99 in magpy.collector
# since MARTAS 0.1.9 in core
from core.websocket_server import WebsocketServer
except:
ws_available = False
# Some variable initialization
## -----------------------------------------------------------
global identifier # Thats probably wrong ... global should be used in functions
identifier = {} # used to store lists from header lines
global counter # use for diffcalc
counter = 0
qos = 0
streamdict = {}
stream = DataStream()
st = []
senslst = []
headdict = {
} # store headerlines for all sensors (headerline are firstline for BIN files)
headstream = {}
verifiedlocation = False
destination = 'stdout'
location = '/tmp'
credentials = 'cred'
stationid = 'WIC'
stid = stationid
webpath = './web'
webport = 8080
socketport = 5000
blacklist = []
def main(argv):
para = sp.parameterdict
conf = sp.configdict
debug = False
configfile = None
statusdict = {}
statuskeylist = []
travistestrun = False
usagestring = 'threshold.py -h <help> -m <configpath>'
try:
opts, args = getopt.getopt(argv,"hm:DT",["configpath="])
except getopt.GetoptError:
print ('Check your options:')
print (usagestring)
sys.exit(2)
for opt, arg in opts:
if opt == '-h':
print ('------------------------------------------------------')
print ('Usage:')
print (usagestring)
print ('------------------------------------------------------')
print ('Options:')
print ('-h help')
print ('-m Define the path for the configuration file.')
print (' Please note: a configuration file is obligatory')
print (' ----------------------------')
print (' configurationfile')
print (' ----------------------------')
print (' threhold.cfg: (looks like)')
print (' # MARTAS directory')
print (' martasdir : /home/cobs/MARTAS/')
print (' # Define data source (file, db)')
print (' source : file')
print (' # If source = db then define data base credentials created by addcred (MARTAS)')
print (' dbcredentials : None')
print (' # If source = file define the base path')
print (' sensorpath : /srv/mqtt/')
print (' # Notifaction (uses martaslog class, one of email, telegram, mqtt, log) ')
print (' notification : email')
print (' notificationconfig : /etc/martas/notification.cfg')
print (' # serial communication for switch commands (based on ardcomm.py (MARTAS/app)')
print (' serialcfg : None')
print (' #parameter (all given parameters are checked in the given order, use semicolons for parameter list):')
print (' # sensorid; timerange to check; key to check, value, lower or upper bound,statusmessage,resetby,switchcommand(optional)')
print (' 1 : DS18B20XX,1800,t1,5,low,average,on,swP:4:1')
print (' 2 : DS18B20XY,1800,t1,10,high,median,off,swP:4:0')
print (' 3 : DS18B20XZ,600,t2,20,high,max,alarm at date,None')
print (' #to be continued...')
print ('------------------------------------------------------')
print ('Example:')
print (' python threshold.py -m /etc/martas/threshold.cfg')
sys.exit()
elif opt in ("-m", "--configfile"):
configfile = arg
print ("Getting all parameters from configration file: {}".format(configfile))
conf = GetConf2(configfile, confdict=conf)
para = AssignParameterlist(sp.valuenamelist,conf)
elif opt in ("-D", "--debug"):
debug = True
elif opt in ("-T", "--Test"):
travistestrun = True
conf = GetConf2(os.path.join('..','conf','threshold.cfg'))
para = AssignParameterlist(sp.valuenamelist,conf)
# TODO activate in order prevent default values
#if not configfile or conf == {}:
# print (' !! Could not read configuration information - aborting')
# sys.exit()
if debug:
print ("Configuration dictionary: \n{}".format(conf))
print ("Parameter dictionary: \n{}".format(para))
if not (len(para)) > 0:
print ("No parameters given too be checked - aborting")
sys.exit()
try:
martaslogpath = os.path.join(conf.get('martasdir'), 'core')
sys.path.insert(1, martaslogpath)
from martas import martaslog as ml
logpath = conf.get('bufferpath')
except:
print ("Could not import martas logging routines - check MARTAS directory path")
# For each parameter
for i in range(0,1000):
valuedict = para.get(str(i),{})
content = ''
if not valuedict == {}:
if debug:
print ("Checking parameterset {}".format(i))
data = DataStream()
testvalue = None
evaluate = {}
# Obtain a magpy data stream of the respective data set
if debug:
print ("Accessing data from {} at {}: Sensor {} - Amount: {} sec".format(conf.get('source'),conf.get('bufferpath'),valuedict.get('sensorid'),valuedict.get('timerange') ))
(data,msg1) = GetData(conf.get('source'), conf.get('bufferpath'), conf.get('database'), conf.get('dbcredentials'), valuedict.get('sensorid'),valuedict.get('timerange'), debug=debug , startdate=conf.get('startdate') )
(testvalue,msg2) = GetTestValue( data, valuedict.get('key'), valuedict.get('function'), debug=debug) # Returns comparison value(e.g. mean, max etc)
if not testvalue and travistestrun:
print ("Testrun for parameterset {} OK".format(i))
elif not testvalue:
print ("Please check your test data set... are bufferfiles existing? Is the sensorid correct?")
elif is_number(testvalue):
(evaluate, msg) = CheckThreshold(testvalue, valuedict.get('value'), valuedict.get('state'), debug=debug) # Returns statusmessage
if evaluate and msg == '':
content = InterpreteStatus(valuedict,debug=debug)
# Perform switch and added "switch on/off" to content
if not valuedict.get('switchcommand') in ['None','none',None]:
if debug:
print ("Found switching command ... eventually will send serial command (if not done already) after checking all other commands")
content = '{} - switch: {}'.format(content, valuedict.get('switchcommand'))
# remember the switchuing command and only issue it if statusdict is changing
elif not msg == '':
content = msg
else:
content = ''
else:
content = msg1+' - '+msg2
if content:
statuskeylist.append('Sensor {} and key {}'.format(valuedict.get('sensorid'),valuedict.get('key')))
statusdict['Sensor {} and key {}'.format(valuedict.get('sensorid'),valuedict.get('key'))] = content
if debug:
print ("Finished parameterset {}".format(i))
if conf.get('reportlevel') == 'full':
# Get a unique status key list:
statuskeylist = list(dict.fromkeys(statuskeylist))
for elem in statuskeylist:
cont = statusdict.get(elem,'')
if cont == '':
statusdict[elem] = "Everything fine"
if debug:
print ("Statusdict: {}".format(statusdict))
receiver = conf.get('notification')
cfg = conf.get('notificationconfig')
logfile = conf.get('logfile')
if travistestrun:
print ("Skipping send routines in test run - finished successfully")
sys.exit()
if debug:
print ("New notifications will be send to: {} (Config: {})".format(receiver,cfg))
martaslog = ml(logfile=logfile,receiver=receiver)
if receiver == 'telegram':
martaslog.telegram['config'] = cfg
elif receiver == 'email':
martaslog.email['config'] = cfg
changes = martaslog.msg(statusdict)
if not len(changes) > 0:
print ("Nothing to report - threshold check successfully finished")
for element in changes:
line = changes.get(element)
if debug:
print ("Changes affecting:", element)
l = line.split('switch:')
if len(l) == 2:
print (" ... now dealing with switching serial command:")
comm = l[1].strip()
script = os.path.join(conf.get('martasdir'),'app','ardcomm.py')
pythonpath = sys.executable
arg1 = "-c {}".format(comm)
arg2 = "-p {}".format(conf.get('port'))
arg3 = "-b {}".format(conf.get('baudrate'))
arg4 = "-a {}".format(conf.get('parity'))
arg5 = "-y {}".format(conf.get('bytesize'))
arg6 = "-s {}".format(conf.get('stopbits'))
arg7 = "-t {}".format(conf.get('timeout'))
#arg8 = "-e {}".format(conf.get('eol')) # not used so far
command = "{} {} {} {} {} {} {} {} {}".format(pythonpath,script,arg1, arg2, arg3, arg4, arg5, arg6, arg7) ## Too be checked
command = "{} {} {}".format(pythonpath,script,arg1)
if debug:
print (" ... sending {}".format(command))
try:
import subprocess
p = subprocess.Popen(command, stdout=subprocess.PIPE, shell=True)
(output, err) = p.communicate()
mesg = "{}".format(output)
except subprocess.CalledProcessError:
mesg = "threshold: sending command didnt work"
except:
mesg = "threshold: sending command problem"
print (mesg)
print (" ... success")
def readJSON(filename, headonly=False, **kwargs):
"""
Reading JSON format data.
"""
stream = DataStream()
header = {}
array = [[] for key in KEYLIST]
with open(filename, 'r') as jsonfile:
dataset = json.load(jsonfile)
loggerlib.info('Read: %s, Format: %s ' % (filename, "JSON"))
fillkeys = ['var1', 'var2', 'var3', 'var4', 'var5', 'x', 'y', 'z', 'f']
datakeys = dataset[0]
keydict = {}
for i, key in enumerate(datakeys):
if 'time' in key:
keydict[i] = 'time'
elif key == 'density':
keydict[i] = 'var1'
fillkeys.pop(fillkeys.index('var1'))
elif key == 'speed':
keydict[i] = 'var2'
fillkeys.pop(fillkeys.index('var2'))
elif key == 'temperature':
keydict[i] = 'var3'
fillkeys.pop(fillkeys.index('var3'))
elif 'bx' in key.lower():
keydict[i] = 'x'
fillkeys.pop(fillkeys.index('x'))
elif 'by' in key.lower():
keydict[i] = 'y'
fillkeys.pop(fillkeys.index('y'))
elif 'bz' in key.lower():
keydict[i] = 'z'
fillkeys.pop(fillkeys.index('z'))
elif 'bt' in key.lower():
keydict[i] = 'f'
fillkeys.pop(fillkeys.index('f'))
else:
try:
keydict[i] = fillkeys.pop(0)
except IndexError:
loggerlib.warning(
"CAUTION! Out of available keys for data. {} will not be contained in stream."
.format(key))
print(
"CAUTION! Out of available keys for data. {} will not be contained in stream."
.format(key))
if 'time' in key:
data = [
date2num(testTimeString(str(x[i]))) for x in dataset[1:]
]
else:
data = [
np.nan if x[i] is None else float(x[i])
for x in dataset[1:]
]
array[KEYLIST.index(keydict[i])] = data
header['col-' + keydict[i]] = key
header['unit-col-' + keydict[i]] = ''
for idx, elem in enumerate(array):
array[idx] = np.asarray(array[idx])
stream = DataStream([], header, np.asarray(array))
return stream
def readNETCDF(filename, headonly=False, **kwargs):
"""
Reading NetCDF format data.
To see all attributes of file: print(ncdata.ncattrs())
"""
stream = DataStream([], {})
headers = {}
array = [[] for key in stream.KEYLIST]
timef = "%Y-%m-%dT%H:%M:%S.%fZ"
starttime = kwargs.get('starttime')
endtime = kwargs.get('endtime')
ncdata = Dataset(filename, 'r')
filestart = datetime.strptime(ncdata.time_coverage_start, timef)
fileend = datetime.strptime(ncdata.time_coverage_end, timef)
# Check if file is within defined time ranges:
getfile = True
try:
if starttime:
if not filestart >= datetime.date(stream._testtime(starttime)):
getfile = False
if endtime:
if not fileend <= datetime.date(stream._testtime(endtime)):
getfile = False
except:
getfile = True
# Read data into assigned columns:
if getfile:
logger.info("readNETCDF: Reading {}".format(filename))
if ncdata.program == 'DSCOVR':
logger.info(
"readNETCDF: File contains DSCOVR data. Using appropriate keys"
)
KEYDICT = DSCOVR_KEYDICT
if ncdata.variables[
'time'].units == 'milliseconds since 1970-01-01T00:00:00Z':
array[0] = np.array([
date2num(datetime.utcfromtimestamp(x / 1000.))
for x in ncdata.variables['time'][...]
])
else:
logger.warning(
"readNETCDF: Could not identify time format. Time array probably incorrect"
)
array[0] = ncdata.variables['time'][...]
for var in ncdata.variables:
if var in KEYDICT:
column = ncdata.variables[var]
coldata = column[...]
key = KEYDICT[var]
idx = stream.KEYLIST.index(key)
# Convert from MaskedArray
coldata = np.array(coldata, dtype=np.float)
# Replace masked values with NaNs:
coldata[np.where(
coldata == float(column.missing_value))] = np.NaN
array[idx] = coldata
headers['col-' + key] = var
headers['unit-col-' + key] = ncdata.variables[var].units
# Fill in additional header data:
for attr in ncdata.ncattrs():
headers[attr] = getattr(ncdata, attr)
ncdata.close()
return DataStream([], headers, np.asarray(array))
def main(argv):
path = ''
startdate = ''
enddate = ''
conf = ''
dummy = DataStream()
destination = ''
global energylist
global isotopelst
global colorlst
global intervallist
name = "{}-Projects-gamma".format(sn)
#global roi
try:
opts, args = getopt.getopt(argv, "hp:b:e:m:d:", [
"path=",
"begin=",
"end=",
"config=",
"destination=",
])
except getopt.GetoptError:
print(
'gamma.py -p <path> -b <begin> -e <end> -m <config> -d <destination>'
)
sys.exit(2)
for opt, arg in opts:
if opt == '-h':
print('-------------------------------------')
print('Description:')
print('-- gamma.py analyses gamma data --')
print('-------------------------------------')
print('Usage:')
print(
'gamma.py -p <path> -b <begin> -e <end> -m <config> -d <destination>'
)
print('-------------------------------------')
print('Options:')
print(
'-p : path to gamma spectral data like "/home/max/myarchive"'
)
print('-b : begin: default yesterday')
print('-e : end: default = datetime.utcnow()')
print('-d : path to store data')
print('-m : config file')
print('-------------------------------------')
print('Example:')
print(
'python gamma.py /media/Samsung/Observatory/data/ -b "2012-06-01" -m myconf.cfg'
)
sys.exit()
elif opt in ("-p", "--path"):
path = arg
elif opt in ("-b", "--begin"):
startdate = arg
elif opt in ("-e", "--end"):
enddate = arg
elif opt in ("-m", "--config"):
conf = arg
elif opt in ("-d", "--destination"):
destination = arg
if path == '':
path = r"/home/leon/CronScripts/MagPyAnalysis/RadonAnalysis/ScriptsThemisto"
#print ('Specify a path to gamma data:')
#print ('-- check gamma.py -h for more options and requirements')
#sys.exit()
if conf == '':
print('Specify a configuration file:')
print('-- check gamma.py -h for more options and requirements')
sys.exit()
# Test time
if not startdate == '':
starttime = dummy._testtime(startdate)
print(starttime)
else:
starttime = datetime.utcnow() - timedelta(days=2)
if not enddate == '':
endtime = dummy._testtime(enddate)
else:
endtime = datetime.utcnow()
# Read Config data
confdict = acs.GetConf(conf)
#print (confdict)
debug = False
if confdict.get('debug') in ['True', 'true']:
debug = True
if not confdict.get('plots') in ['Random', 'random']:
pl = confdict.get('plots').split(',')
try:
plotselection = [int(el) for el in pl]
except:
plotselection = []
else:
plotselection = 'random'
print("SELECTED for plotting:", plotselection)
graphdir = confdict.get('graphdir')
if not os.path.exists(graphdir):
os.makedirs(graphdir)
# Get analysis data
ok = True
if ok:
energylist = [int(el) for el in confdict.get('energies').split(',')]
if debug:
print("E:", energylist)
isotopelst = confdict.get('isotops').split(',')
if debug:
print("Isotops:", isotopelst)
colorlst = confdict.get('colors').split(',')
if debug:
print("Colors:", colorlst)
intervallist = [int(el) for el in confdict.get('intervals').split(',')]
if debug:
print("Intervals:", intervallist)
roi = []
roistrtmp = confdict.get('roi')
roistr = re.sub("\[[^]]*\]", lambda x: x.group(0).replace(',', ';'),
roistrtmp)
for el in roistr.split(','):
el = el.strip()
if not el.startswith('['):
roi.append(int(el))
else:
lelt = el.replace('[', '').replace(']', '')
lel = [int(ele) for ele in lelt.split(';')]
roi.append(lel)
if debug:
print("Rois:", roi)
confdict['roi'] = roi
stream = analyse_mca(path,
startdate=starttime,
enddate=endtime,
config=confdict)
sys.exit()
try:
stream = analyse_mca(path,
startdate=starttime,
enddate=endtime,
config=confdict)
stream.header['SensorSerialNum'] = confdict.get('sensorserial', '')
stream.header['SensorDataLogger'] = confdict.get(
'sensordatalogger', '')
stream.header['SensorDataLoggerSerialNum'] = confdict.get(
'sensordataloggerserialnum', '')
stream.header['SensorName'] = confdict.get('sensorname', '')
stream.header['SensorID'] = "{}_{}_0001".format(
confdict.get('sensorname', 'None'),
confdict.get('sensorserial', '12345'))
stream.header['DataID'] = "{}_{}_0001_0001".format(
confdict.get('sensorname', 'None'),
confdict.get('sensorserial', '12345'))
stream.header['StationID'] = confdict.get('StationID', 'WIC')
print(stream.length())
print("writing data")
if destination:
stream.write(destination,
coverage='year',
mode="replace",
format_type='PYSTR',
filenamebegins='{}_'.format(
stream.header['SensorID']),
dateformat='%Y')
#stream.write(destination, format_type='PYASCII', filenamebegins='{}_'.format(stream.header['SensorID']))
statusmsg[name] = 'gamma analysis successfully finished'
except:
statusmsg[name] = 'gamma analysis failed'
martaslog = ml(logfile=logpath, receiver='telegram')
martaslog.telegram['config'] = '/etc/martas/telegram.cfg'
martaslog.msg(statusmsg)
def analyse_mca(path, startdate=None, enddate=None, config={}):
"""
DESCRIPTION:
Anlyze MCA reads spectral data files with time stamp in filename (e.g. hour)
RETURN:
Datastream
config contains:
filename="Spectral_",
filedate="",
roi
sensorname : xx
sensorid : xx
voltage : 720 # -> datatype for eventually new datarevision
graphdir : /path
plots : random # selects 2 random time steps
plots : 427760 # list of filenumbers
notes : 20181118T000000,20181122T235900,Testlauf
notes : 20181123T000000,current,Active
notes : 20181118T000000,20181122T235900,background
roi :
energies :
isotops :
element :
"""
if not startdate:
startdate = datetime.utcnow() - timedelta(days=1)
if not enddate:
enddate = datetime.utcnow()
#analyzemca = False
#if analyzemca:
# Filter environment data
# #######################
filterenv = False
backgrdend = 0
accumulationend = 0
# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
# IMPORTANT: check times !!!!
# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
#path = r"/home/leon/CronScripts/MagPyAnalysis/RadonAnalysis/ScriptsThemisto"
path = path
#namedummy = "Spectral_"
namedummy = config.get('filename')
resolution = config.get('filedate')
numlist, dirs = NumList(path, namedummy)
#print (dirs)
print("limit filerange by starttime and endtime")
# min and max number within given timerange
if resolution in ['Day', 'day']:
div = 86400.
elif resolution in ['Minute', 'minute', 'minutes']:
div = 60.
else: # hour as default
div = 3600.
mintime = float(startdate.strftime("%s")) / div
maxtime = float(enddate.strftime("%s")) / div
validrange = [int(mintime), int(maxtime)]
filerange = [[el, i] for i, el in enumerate(numlist)
if el in range(min(validrange), max(validrange))]
if not len(filerange) > 0:
print("Did not find any spectrum files within the given time range")
return DataStream()
# get config data
if config.get('plots') in ['Random', 'random']:
# select one diagram randomly
import random
plotselection = [random.choice(filerange)]
if not config.get('plots') in ['Random', 'random']:
pl = config.get('plots').split(',')
try:
plotselection = [int(el) for el in pl]
except:
plotselection = []
roi = config.get('roi')
dataarray = [[] for i in range(len(roi) + 1)]
# Cycle through channels
validfilerange = []
print(
"# -------------------------------------------------------------------------"
)
print("# A. Main prog")
print(
"# -------------------------------------------------------------------------"
)
print("# Cycling through all spectrograms")
print(filerange)
for eleme in filerange:
# Define file name
i = eleme[0]
idx = eleme[1]
name = (namedummy + "%06i.Chn") % (i)
print(
"# -------------------------------------------------------------------------"
)
print(" - Analyzing {}".format(name))
print(
"# -------------------------------------------------------------------------"
)
filename = os.path.join(dirs[idx], name)
#test=True
#if test:
try:
temp = open(filename, 'rb')
line1 = temp.read(32)
data1 = struct.unpack("<bb3H2L8c4c2H", line1)
nchn = data1[-1]
line2 = temp.read(nchn * 4)
data2 = struct.unpack("<%sL" % nchn, line2)
# put zero values in front so that channel number correspond to indicies
data = [0] * 7
data = data + list(data2[6:])
data2 = np.asarray(data)
#roi = [63, 114, 231, 291, [197,398]]
testplot = False
if testplot:
print(" - Raw data plot")
# Determine the following channel energy relation from each analysis
fig, ax = plt.subplots(1, 1)
ax.set_yscale('log')
ax.set_xlim([0, 1050])
ax.plot(range(0, len(data2)), data2, '-')
plt.xlabel("channel []")
plt.ylabel("counts per hour [1/h]")
plt.grid()
plt.show()
print(
"# -------------------------------------------------------------------------"
)
print(" - Analysis of spectrum")
#try:
if i in plotselection:
print(" --> Plotting")
#result[str(i)] = singlespecanalysis(data2,roi=roi,plot=True,name=str(i))
singlespecanalysis(data2, roi=roi, plot=True, name=str(i))
plt.show()
else:
#result[str(i)] = singlespecanalysis(data2,roi=roi,plot=False,name=str(i))
singlespecanalysis(data2, roi=roi, plot=False, name=str(i))
#except:
# singlespecanalysis(data2,roi=roi,plot=False,name=str(i))
print(
"# -------------------------------------------------------------------------"
)
print(" - Extracting energy levels")
dataarray[0].append(
date2num(datetime.utcfromtimestamp(int(i) * 3600.)))
for idx, elem in enumerate(roi):
exec("roiname = str(roi[{}])".format(idx))
if isinstance(elem, (list, )):
roiname = roiname.replace("[", "").replace("]",
"").replace(
", ", "-")
#print (idx,roiname)
dataarray[idx + 1].append(result[roiname][3])
#exec("liste{}.append(result[roiname][3])".format(idx+1))
listemca.append(result[str(roi[3])][2])
listesca.append(result[str(roi[3])][5])
listeshift.append(result[str(roi[3])][6])
# Add flags to the datastream
# Notes should be converted to flags
except:
print("----------------------------")
print("Failed analysis for {}!".format(filename))
print(
"# -------------------------------------------------------------------------"
)
print("# Creating summary and MagPy file for storage")
starray = [np.asarray([]) for el in KEYLIST]
for idx, elem in enumerate(dataarray):
starray[idx] = np.asarray(elem)
#dataarray = np.asarray([np.asarray(elem) for elem in dataarray])
#print (dataarray)
print(KEYLIST[1:len(roi) + 1])
stream = DataStream()
stream.ndarray = np.asarray(starray)
#stream.header = header
stream = stream.sorting()
stream.header['SensorElements'] = ",".join(
["Counts{}".format(elem) for elem in energylist])
stream.header['SensorKeys'] = KEYLIST[1:len(roi) + 1]
return stream
"""
def CheckMARCOS(db,threshold=600, statusdict={},jobname='JOB',excludelist=[],acceptedoffsets={},debug=False):
testst = DataStream()
offset = {}
testname = '{}-DBactuality'.format(jobname)
cursor = db.cursor()
ok = True
if debug:
print ("1. Get all tables")
print ("-----------------------------------")
tablessql = 'SHOW TABLES'
try:
cursor.execute(tablessql)
except mpdb.mysql.IntegrityError as message:
print (message)
ok = False
except mpdb.mysql.Error as message:
print (message)
ok = False
except:
print ('check table: unkown error')
ok = False
if ok:
tables = cursor.fetchall()
tables = [el[0] for el in tables]
if not len(tables) > 0:
print ('check table: no tables found - stopping')
ok = False
else:
print ('check table: aborting')
#cursor.close()
ok = False
if ok:
if debug:
print ("2. Extract tables to be examined")
print ("-----------------------------------")
# get BLV tables
blv = [el for el in tables if el.startswith('BLV')]
excludelist.extend(blv)
if debug:
print ("Data to be excluded: {}".format(excludelist))
tables = [el for el in tables if not el in excludelist]
if ok:
if debug:
print ("3. Delete any existing timestamps which are in the future")
# classic problem of IWT
print ("-----------------------------------")
delsql = "DELETE FROM IWT_TILT01_0001_0001 WHERE time > NOW()"
if ok:
try:
cursor.execute(delsql)
except mpdb.mysql.IntegrityError as message:
print (' -- check: {}'.format(message))
except mpdb.mysql.Error as message:
print (' -- check: {}'.format(message))
except:
print (' -- check: unkown error')
# eventually an execute is necessary here
if ok:
if debug:
print ("4. Getting last input in each table")
print ("-----------------------------------")
for table in tables:
if debug:
print (' -> running for {}'.format(table))
lastsql = 'SELECT time FROM {} ORDER BY time DESC LIMIT 1'.format(table)
try:
cursor.execute(lastsql)
except mpdb.mysql.IntegrityError as message:
print (' -- check table: {}'.format(message))
except mpdb.mysql.Error as message:
print (' -- check table: {}'.format(message))
except:
print (' -- check table: unkown error')
value = cursor.fetchall()
try:
lasttime = value[0][0]
timetest = True
except:
timetest = False
pass
if timetest:
lastt = testst._testtime(lasttime)
# Get difference to current time
current = datetime.utcnow()
tdiff = np.abs((current-lastt).total_seconds())
offset[table] = tdiff
if debug:
print ("Difference: {}".format(tdiff))
if ok:
if debug:
print ("5. Check threshold information")
print ("-----------------------------------")
statusdict[testname] = 'possible'
for el in offset:
# determine threshold
usedthreshold = threshold
name = "{}-{}".format(testname,el.replace('_',''))
for elem in acceptedoffsets:
if el.find(elem) > -1:
usedthreshold = acceptedoffsets[elem]
if offset[el] > usedthreshold:
if debug:
print ("{} : data too old by {} seconds".format(el,offset[el]))
statusdict[name] = 'latest input older than {} sec'.format(usedthreshold)
else:
statusdict[name] = 'actual'
else:
statusdict[testname] = 'failure'
cursor.close()
return statusdict
def readJSON(filename, headonly=False, **kwargs):
"""
Reading JSON format data.
"""
stream = DataStream()
array = [[] for key in KEYLIST]
with open(filename, "r") as jsonfile:
dataset = json.load(jsonfile)
loggerlib.info("Read: %s, Format: %s " % (filename, "JSON"))
fillkeys = ["var1", "var2", "var3", "var4", "var5", "x", "y", "z", "f"]
datakeys = dataset[0]
keydict = {}
for i, key in enumerate(datakeys):
if "time" in key:
keydict[i] = "time"
elif key == "density":
keydict[i] = "var1"
fillkeys.pop(fillkeys.index("var1"))
elif key == "speed":
keydict[i] = "var2"
fillkeys.pop(fillkeys.index("var2"))
elif key == "temperature":
keydict[i] = "var3"
fillkeys.pop(fillkeys.index("var3"))
elif "bx" in key.lower():
keydict[i] = "x"
fillkeys.pop(fillkeys.index("x"))
elif "by" in key.lower():
keydict[i] = "y"
fillkeys.pop(fillkeys.index("y"))
elif "bz" in key.lower():
keydict[i] = "z"
fillkeys.pop(fillkeys.index("z"))
elif "bt" in key.lower():
keydict[i] = "f"
fillkeys.pop(fillkeys.index("f"))
else:
try:
keydict[i] = fillkeys.pop(0)
except IndexError:
loggerlib.warning(
"CAUTION! Out of available keys for data. {} will not be contained in stream.".format(key)
)
print("CAUTION! Out of available keys for data. {} will not be contained in stream.".format(key))
if "time" in key:
data = [date2num(testTimeString(str(x[i]))) for x in dataset[1:]]
else:
data = [np.nan if x[i] is None else float(x[i]) for x in dataset[1:]]
array[KEYLIST.index(keydict[i])] = data
stream.header["col-" + keydict[i]] = key
stream.header["unit-col-" + keydict[i]] = ""
for idx, elem in enumerate(array):
array[idx] = np.asarray(array[idx])
stream = DataStream([], stream.header, np.asarray(array))
return stream
def main(argv):
#para = sp.parameterdict
#conf = sp.configdict
para = {}
conf = {}
# necessary configs (may be overwritten):
conf['statusfile'] = '/var/log/magpy/statusfile.log'
debug = False
configfile = None
statusdict = {}
statuskeylist = []
MachineToReset = None
ListMachine = False
usagestring = 'threshold.py -h <help> -m <configpath> [-l][-r state machine number]'
try:
opts, args = getopt.getopt(argv, "hm:Ur:l", ["configpath=", "reset="])
except getopt.GetoptError:
print('Check your options:')
print(usagestring)
sys.exit(2)
for opt, arg in opts:
if opt == '-h':
print('------------------------------------------------------')
print('Usage:')
print(usagestring)
print('------------------------------------------------------')
print('Options:')
print('-h help')
print('-l List states of all state machines')
print(
'-r nr Reset state machine number nr into initial state'
)
print('-m Define the path for the configuration file.')
print(
' Please note: a configuration file is obligatory'
)
print(' ----------------------------')
print(' configurationfile')
print(' ----------------------------')
print(' statemachine.cfg: (looks like)')
print(' # MARTAS directory')
print(' martasdir : /home/cobs/MARTAS/')
print(' # Define data source (file, db, ...)')
print(' source : file')
print(
' # If source = db then define data base credentials created by addcred (MARTAS)'
)
print(' dbcredentials : None')
print(
' # If source = file define the MARTAS buffer base path'
)
print(' bufferpath : /srv/mqtt/')
print(
' # statusfile (a json style dictionary, which contains states)'
)
print(
' statusfile : /var/log/magpy/status.log'
)
print(' # Path of mail config file')
print(' emailconfig : /etc/martas/mail.cfg')
print(
' # serial communication for switch commands (based on ardcomm.py (MARTAS/app))'
)
print(' serialcfg : None')
print(
" #Nr. : 'sensorid';'timerange';'key';'value';'function';'operator';'statusmessage';'nextstatus'[;action;argument;action;argument...]"
)
print(
' # comment: timerange in seconds, statusmessage is until now a dummy'
)
print(' status : start')
print(
' 1 : SENSOR1_SERNR_0001;180;t1;20;min;below;;triggered;email;t1 below 20°'
)
print(
' 2 : SENSOR1_SERNR_0001;180;t1;20;min;below;;triggered;email;t1 below 20°'
)
print(' status : triggered')
print(
' 1 : SENSOR1_SERNR_0001;180;t1;21;min;above;;start;email;t1 above 21° - I like to send a lot of emails inspite of hystereses;-)'
)
print(
" 2 : SENSOR1_SERNR_0001;180;t1;20;min;above;;reentered;email;t1 above 20° again - but that doesn't mean everything is fine again..."
)
print(' status : reentered')
print(
' 2 : SENSOR1_SERNR_0001;120;t1;20;min;below;;stop;email;it seems like t1 is toggling around 20°C'
)
print(
' 2 : SENSOR1_SERNR_0001;120;t1;20;average;below;;stop;email;no more email'
)
print(' ----------------------------')
print('')
print('-r reset a state machine')
print('-l display states')
print('------------------------------------------------------')
print('Example:')
print(' python statemachine.py -m /etc/martas/statemachine.cfg')
sys.exit()
elif opt in ("-m", "--configfile"):
configfile = arg
if debug:
print(
"Getting all parameters of the state machine from configuration file: {}"
.format(configfile))
(conf, para) = readConfig(configfile)
elif opt in ("-r", "--reset"):
if is_number(arg):
MachineToReset = arg
else:
print("--reset must_be_a_number")
sys.exit()
elif opt in ("-l"):
ListMachine = True
elif opt in ("-U", "--debug"):
debug = True
if debug:
print("Configuration dictionary: \n{}".format(conf))
print("Parameter dictionary: \n{}".format(para))
if not (len(para)) > 0:
print("No parameters given to be checked - aborting")
sys.exit()
try:
martaslogpath = os.path.join(conf.get('martasdir'), 'core')
sys.path.insert(1, martaslogpath)
import martas
logpath = conf.get('bufferpath')
except:
print(
"Could not import martas logging routines - check MARTAS directory path"
)
statusdict = {}
if os.path.isfile(conf['statusfile']):
# read log if exists and exentually update changed information
# return changes
with open(conf['statusfile'], 'r') as file:
statusdict = json.load(file)
if debug:
print("Statusfile {} loaded".format(conf['statusfile']))
if ListMachine:
if statusdict == {}:
print('no states but start')
else:
for state in statusdict:
print(state + ": " + statusdict[state]['status'])
exit()
if MachineToReset and not statusdict == {}:
if MachineToReset in statusdict:
del statusdict[MachineToReset]
print(MachineToReset + " set to 'start'")
else:
print(MachineToReset + " not found")
with open(conf['statusfile'], 'w') as file:
if debug:
print('writing to ' + conf['statusfile'] + ' :')
print(statusdict)
file.write(
json.dumps(statusdict)) # use `json.loads` to do the reverse
exit()
# For each machine
for i in range(0, 1000):
valuedict = {}
values = []
if not str(i) in statusdict and str(i) in para['start']:
# machine is not yet in the statusfile, let's add it to the dict
statusdict[str(i)] = {}
statusdict[str(i)]['status'] = 'start'
#laststatusdict[str(i)]['sensorid'] = para['start'][str(i)]['sensorid']
#laststatusdict[str(i)]['key'] = para['start'][str(i)]['key']
if str(i) in statusdict:
status = statusdict[str(i)]['status']
# TODO handle states deleted from the config file!
values = para[status].get(str(i), [])
if not values == []:
if debug:
print("Checking state machine {}".format(i))
data = DataStream()
# Obtain a magpy data stream of the respective data set
for valuedict in values:
if debug:
print(
"Accessing data from {} at {}: Sensor {} - Amount: {} sec"
.format(conf.get('source'), conf.get('bufferpath'),
valuedict.get('sensorid'),
valuedict.get('timerange')))
(data, msg1) = GetData(conf.get('source'),
conf.get('bufferpath'),
conf.get('database'),
conf.get('dbcredentials'),
valuedict.get('sensorid'),
valuedict.get('timerange'),
debug=debug,
startdate=conf.get('startdate'))
testvalue = None
if data._get_key_headers() == []:
# there are no keys in the data
if debug:
print('no data for testvalue')
else:
(testvalue, msg2) = GetTestValue(
data,
valuedict.get('key'),
valuedict.get('function'),
debug=debug
) # Returns comparison value(e.g. mean, max etc)
if debug:
print("testvalue is {}".format(testvalue))
if is_number(testvalue):
(evaluate, msg) = CheckThreshold(
testvalue,
valuedict.get('value'),
valuedict.get('operator'),
debug=debug) # Returns statusmessage
if evaluate and msg == '':
# criteria are met - do something
# change status
if debug:
print("changing status of machine " + str(i) +
" from")
print(statusdict[str(i)]['status'])
print("to")
print(valuedict['nextstatus'])
statusdict[str(i)]['status'] = valuedict['nextstatus']
if 'action' in valuedict:
for action in valuedict['action']:
if action['action'] == 'email':
dic = readConfigFromFile(
conf.get('emailconfig'))
dic['Text'] = action['argument']
martas.sendmail(dic)
# TODO not implemented / not tested
if action['action'] == 'telegram':
dic = sm_support.readConfigFromFile(
conf.get('telegramconfig'))
dic['text'] = action['argument']
martas.sendtelegram(dic)
if action['action'] == 'switch:':
dic = conf
dic['comm'] = action['argument']
martas.sendswitchcommand(dic)
# TODO handle content resp. errors
content = InterpreteStatus(valuedict, debug=debug)
# Perform switch and added "switch on/off" to content
if not valuedict.get('switchcommand') in [
'None', 'none', None
]:
if debug:
print(
"Found switching command ... eventually will send serial command (if not done already) after checking all other commands"
)
content = '{} - switch: {}'.format(
content, valuedict.get('switchcommand'))
# remember the switchuing command and only issue it if statusdict is changing
elif not msg == '':
content = msg
else:
content = ''
else:
#content = msg1+' - '+msg2
pass
#if content:
if 0:
statuskeylist.append('Sensor {} and key {}'.format(
valuedict.get('sensorid'), valuedict.get('key')))
statusdict['Sensor {} and key {}'.format(
valuedict.get('sensorid'), valuedict.get('key'))] = content
if debug:
print("Finished state machine {}".format(i))
with open(conf['statusfile'], 'w') as file:
if debug:
print('writing to ' + conf['statusfile'] + ' :')
print(statusdict)
file.write(
json.dumps(statusdict)) # use `json.loads` to do the reverse
exit()
if conf.get('reportlevel') == 'full':
# Get a unique status key list:
statuskeylist = list(dict.fromkeys(statuskeylist))
for elem in statuskeylist:
cont = statusdict.get(elem, '')
if cont == '':
statusdict[elem] = "Everything fine"
if debug:
print("Statusdict: {}".format(statusdict))
receiver = conf.get('notification')
cfg = conf.get('notificationconfig')
logfile = conf.get('logfile')
if debug:
print("New notifications will be send to: {} (Config: {})".format(
receiver, cfg))
martaslog = ml(logfile=logfile, receiver=receiver)
if receiver == 'telegram':
martaslog.telegram['config'] = cfg
elif receiver == 'email':
martaslog.email['config'] = cfg
changes = martaslog.msg(statusdict)
if not len(changes) > 0:
print("Nothing to report - threshold check successfully finished")
for element in changes:
line = changes.get(element)
if debug:
print("Changes affecting:", element)
l = line.split('switch:')
if len(l) == 2:
print(" ... now dealing with switching serial command:")
comm = l[1].strip()
script = os.path.join(conf.get('martasdir'), 'app', 'ardcomm.py')
pythonpath = sys.executable
arg1 = "-c {}".format(comm)
arg2 = "-p {}".format(conf.get('port'))
arg3 = "-b {}".format(conf.get('baudrate'))
arg4 = "-a {}".format(conf.get('parity'))
arg5 = "-y {}".format(conf.get('bytesize'))
arg6 = "-s {}".format(conf.get('stopbits'))
arg7 = "-t {}".format(conf.get('timeout'))
#arg8 = "-e {}".format(conf.get('eol')) # not used so far
command = "{} {} {} {} {} {} {} {} {}".format(
pythonpath, script, arg1, arg2, arg3, arg4, arg5, arg6,
arg7) ## Too be checked
command = "{} {} {}".format(pythonpath, script, arg1)
if debug:
print(" ... sending {}".format(command))
try:
import subprocess
p = subprocess.Popen(command,
stdout=subprocess.PIPE,
shell=True)
(output, err) = p.communicate()
mesg = "{}".format(output)
except subprocess.CalledProcessError:
mesg = "threshold: sending command didnt work"
except:
mesg = "threshold: sending command problem"
print(mesg)
print(" ... success")
def readCOVJSON(filename, headonly=False, **kwargs):
"""
Reading CoverageJSON format data.
"""
header = {}
array = [[] for key in KEYLIST]
print("Reading coverage json")
with open(filename, 'r') as jsonfile:
dataset = json.load(jsonfile)
loggerlib.info('Read: {}, Format: {} '.format(filename,
"CoverageJSON"))
# Extract header and data
axes = dataset.get("domain").get("axes")
ranges = dataset.get("ranges")
parameters = dataset.get("parameters")
times = dataset.get("domain").get("axes").get("t").get("values")
times = [testTimeString(el) for el in times]
array[0] = date2num(times)
stream = DataStream([], header, array)
try:
stream.header['DataAcquisitionLatitude'] = dataset.get("domain").get(
"axes").get("x").get("values")
stream.header['DataAcquisitionLongitude'] = dataset.get("domain").get(
"axes").get("y").get("values")
stream.header['DataElevation'] = dataset.get("domain").get("axes").get(
"z").get("values")
except:
pass
print(dataset.get('context'))
def addelement(datastream, key, element, elementdict, parameterdict):
array = np.asarray(elementdict.get('values'))
datastream = datastream._put_column(array, key)
datastream.header['col-{}'.format(key)] = element
datastream.header['unit-col-{}'.format(key)] = parameterdict.get(
"unit").get("label")
numcnt = 0
strcnt = 1
AVAILKEYS = NUMKEYLIST
ELEMENTSTODO = []
fixedgroups = {
'x': ['x', 'X', 'H', 'I'],
'y': ['y', 'Y', 'D', 'E'],
'z': ['z', 'Z'],
'f': ['f', 'F', 'S'],
'df': ['g', 'G']
}
# Firstly assign data from fixed groups, then fill rest
for element in ranges:
print("Dealing with {}".format(element))
foundgroups = False
for group in fixedgroups:
if element in fixedgroups[group]:
print(" -> adding to {}".format(group))
addelement(stream, group, element, ranges[element],
parameters[element])
AVAILKEYS = ['USED' if x == group else x for x in AVAILKEYS]
foundgroups = True
break
if not foundgroups:
ELEMENTSTODO.append(element)
# Now assign all other elements to appropriate keys
for element in ELEMENTSTODO:
print("Now dealing with {}".format(element))
# assign element to key
if ranges.get(element).get('dataType') in ['float', 'double', 'int']:
# get the first key which is not yet used
index = min(
[idx for idx, el in enumerate(AVAILKEYS) if not el == 'USED'])
key = AVAILKEYS[index]
print(" -> adding to {}".format(key))
addelement(stream, key, element, ranges[element],
parameters[element])
AVAILKEYS[index] = 'USED'
else:
if strcnt <= 4:
key = "str{}".format(strcnt)
print(" -> adding to {}".format(key))
addelement(stream, key, element, ranges[element],
parameters[element])
strcnt += 1
return stream