def find_first_timeout(self):
nextTimeOut = dates.date2secs(
self.objectNode.get_child("nextTimeout").get_value())
now = dates.date2secs(dates.now_iso())
period = self.objectNode.get_child("periodSeconds").get_value()
while nextTimeOut < now:
nextTimeOut = nextTimeOut + period
self.objectNode.get_child("nextTimeout").set_value(
dates.epochToIsoString(nextTimeOut))
status = self.objectNode.get_child("isRunning").set_value(False)
def create(functionNode):
#hide(functionNode) # if any is already visible, hide it
model = functionNode.get_model()
logger = functionNode.get_logger()
logger.debug("create_envelope()")
motif = functionNode.get_parent().get_child("EnvelopeMiner").get_child(
"motif").get_target()
widget = functionNode.get_parent().get_child("EnvelopeMiner").get_child(
"widget").get_target()
name = motif.get_child("variable").get_target().get_name()
defaults = functionNode.get_parent().get_child(
"defaultParameters").get_value()
if motif and widget:
try:
model.disable_observers()
envelope = motif.create_child("envelope", type="folder")
envelope.create_child(name + "_limitMax", type="timeseries")
envelope.create_child(name + "_limitMin", type="timeseries")
envelope.create_child(name + "_expected", type="timeseries")
#adapt the sampling selection to the selected widget:
# we take the average sampling period *2 as the default and the 10*average sampling period as max
start = dates.date2secs(motif.get_child("startTime").get_value())
end = dates.date2secs(motif.get_child("endTime").get_value())
ts = motif.get_child("variable").get_target().get_time_series(
start, end)
avr = (end - start) / len(ts["values"])
logger.debug(f"sampling period in motif: {avr}")
defaults["samplingPeriod"] = [avr, 10 * avr, 2 * avr]
for k, v in defaults.items(
): #["samplingPeriod","filter","freedom","dynamicFreedom"]:
envelope.create_child(
k,
type="const",
value=v[2],
properties={"validation": {
"limits": [v[0], v[1]]
}})
finally:
model.enable_observers()
if not _connect(motif, widget):
logger.error("can't connect motif envelope to widget")
#also
#override the value with better values
#e.g the sampling period we take the average that we have and make 10 steps to the 10 times sampling rate
#call update from here
updateNode = functionNode.get_parent().get_child("update")
update(updateNode)
return True
def is_inside(annotation, annotations):
""" check if annotation is completely inside one of the annotations"""
start = dates.date2secs(annotation.get_child("startTime").get_value())
end = dates.date2secs(annotation.get_child("endTime").get_value())
for anno in annotations:
thisStart = dates.date2secs(anno.get_child("startTime").get_value())
thisEnd = dates.date2secs(anno.get_child("endTime").get_value())
if thisStart <= start and thisEnd >= end:
return True
return False
def alarm_clock(functionNode):
# this function never ends until we get the signal "stop"
# it works on an own thread
signalNode = functionNode.get_child("control").get_child("signal")
logger = functionNode.get_logger()
functionNode.get_child("isRunning").set_value(True)
while True:
if signalNode.get_value() == "stop":
signalNode.set_value("nosignal")
break
timeout = dates.date2secs(
functionNode.get_child("nextTimeout").get_value())
now = time.time()
if now > timeout:
#set the next timeout
period = functionNode.get_child("periodSeconds").get_value()
timeout = timeout + period
functionNode.get_child("nextTimeout").set_value(
dates.epochToIsoString(timeout))
for node in functionNode.get_child("target").get_leaves():
logger.debug(f"alarm_clock executes", node.get_name())
node.execute()
counterNode = functionNode.get_child("executionCounter")
counterNode.set_value(counterNode.get_value() + 1)
time.sleep(
5) #only sleep short to be able to break the loop with the signal
functionNode.get_child("isRunning").set_value(False)
return True
def thread_func(self):
while self.running:
status = self.objectNode.get_child("isRunning").get_value()
if not status:
self.objectNode.get_child("isRunning").set_value(True)
timeout = dates.date2secs(
self.objectNode.get_child("nextTimeout").get_value())
now = time.time()
if now > timeout:
# set the next timeout
period = self.objectNode.get_child("periodSeconds").get_value()
timeout = timeout + period
self.objectNode.get_child("nextTimeout").set_value(
dates.epochToIsoString(timeout))
for node in self.objectNode.get_child("target").get_leaves():
self.logger.debug(f"AutoTimerClass executes",
node.get_name())
node.execute()
counterNode = self.objectNode.get_child("executionCounter")
counterNode.set_value(counterNode.get_value() + 1)
time.sleep(
5
) # only sleep short to be able to break the loop with the signal
self.objectNode.get_child("isRunning").set_value(False)
def update(functionNode,startTime=0):
if functionNode.get_name()!="update":
functionNode = functionNode.get_parent().get_child("update")
motif = functionNode.get_parent().get_child("StumpyMASS").get_child("motif").get_target()
widget = functionNode.get_parent().get_child("StumpyMASS").get_child("widget").get_target()
logger = functionNode.get_logger()
start = dates.date2secs(motif.get_child("startTime").get_value())
end = dates.date2secs(motif.get_child("endTime").get_value())
times = numpy.arange(start, end)
ts = motif.get_child("variable").get_target().get_time_series(start,end,resampleTimes = times)
data = ts["values"]
if startTime!=0:
diff = startTime-start
times=times+diff
#value offset
ts = motif.get_child("variable").get_target().get_time_series(resampleTimes = times)
dataDiff = ts["values"][0]-data[0]
data = data +dataDiff
return True
def get_events():
evs = ["busy_start", "busy_end", "free_start", "free_end"]
times = [
"2015-02-12T09:16:00+01:00", "2015-02-12T15:46:00+01:00",
"2015-02-12T19:40:00+01:00", "2015-02-13T05:03:00+01:00"
]
return evs, numpy.asarray([dates.date2secs(tim) for tim in times],
dtype=numpy.float64)
def jump(functionNode):
widget = functionNode.get_parent().get_child("StumpyMASS").get_child("widget").get_target()
widgetStartTime = dates.date2secs(widget.get_child("startTime").get_value())
widgetEndTime = dates.date2secs(widget.get_child("endTime").get_value())
#now get the user selection, it will be the index of the results list
matchIndex=int(functionNode.get_child("match").get_value())
if matchIndex==-1:
motif = functionNode.get_parent().get_child("StumpyMASS").get_child("motif").get_target()
match = {}
match["epochStart"] = dates.date2secs(motif.get_child("startTime").get_value())
match["epochEnd"] = dates.date2secs(motif.get_child("endTime").get_value())
else:
results = functionNode.get_parent().get_child("StumpyMASS").get_child("results").get_value()
match = results[matchIndex]
middle = match["epochStart"]+(match["epochEnd"]-match["epochStart"])/2
newStart = middle - (widgetEndTime-widgetStartTime)/2
newEnd = middle + (widgetEndTime - widgetStartTime) / 2
widget.get_child("startTime").set_value(dates.epochToIsoString(newStart))
widget.get_child("endTime").set_value(dates.epochToIsoString(newEnd))
return True
def jump(functionNode):
widget = functionNode.get_parent().get_child("EnvelopeMiner").get_child(
"widget").get_target()
widgetStartTime = dates.date2secs(
widget.get_child("startTime").get_value())
widgetEndTime = dates.date2secs(widget.get_child("endTime").get_value())
#now get the user selection, it will be the index of the results list
matchIndex = int(functionNode.get_child("match").get_value())
if matchIndex == -1:
motif = functionNode.get_parent().get_child("EnvelopeMiner").get_child(
"motif").get_target()
match = {}
match["epochStart"] = dates.date2secs(
motif.get_child("startTime").get_value())
match["epochEnd"] = dates.date2secs(
motif.get_child("endTime").get_value())
#update(functionNode) # re-write the band: ! the match will not 100% align in time with the motif: this is because we are searching for motif only with a given step
else:
results = functionNode.get_parent().get_child(
"EnvelopeMiner").get_child("results").get_value()
match = results[matchIndex]
#update(functionNode,startTime=match["epochStart"])
middle = match["epochStart"] + (match["epochEnd"] -
match["epochStart"]) / 2
newStart = middle - (widgetEndTime - widgetStartTime) / 2
newEnd = middle + (widgetEndTime - widgetStartTime) / 2
widget.get_child("startTime").set_value(dates.epochToIsoString(newStart))
widget.get_child("endTime").set_value(dates.epochToIsoString(newEnd))
if matchIndex == -1:
update(functionNode)
else:
update(functionNode,
startTime=match["epochStart"],
offset=match["offset"])
return True
def data_cleaning(annotations, order=None, logger=None):
#condition: we need to have the right order of annotations
if type(order) is type(None):
order = ["Step" + str(no) for no in range(19)] #Step1,Step2,...Step18
#now take all annotations and order them in time
annoStartTimes = []
annos = []
for anno in annotations:
if anno.get_child("type").get_value() != "time":
continue
#filter out only the ones we want
if set(order).intersection(set(anno.get_child("tags").get_value())):
startTime = date2secs(anno.get_child("startTime").get_value())
annoStartTimes.append(startTime)
annos.append(anno)
#now sort them
sortedIndices = numpy.argsort(annoStartTimes)
sortedAnnos = list(
numpy.asarray(annos)[sortedIndices]) # use numpy for fancy indexing
#now check the order
invalidTimes = {
"startTime": [],
"endTime": []
} #list of {"startTime":333,"endTime":999}
validAnnos = []
nextAnnoIndex = 0
currentProcess = []
for anno in sortedAnnos:
thisTags = anno.get_child("tags").get_value()
if not order[nextAnnoIndex] in thisTags:
#the right order was not kept, reset the search
if logger:
logger.debug(
f'broken process @ {anno.get_child("startTime").get_value()}'
)
currentProcess = []
nextAnnoIndex = 0
if order[nextAnnoIndex] in thisTags:
currentProcess.append(anno)
nextAnnoIndex = nextAnnoIndex + 1
if nextAnnoIndex == len(order):
validAnnos.append(currentProcess)
currentProcess = []
nextAnnoIndex = 0
logger.debug(
f"Found {len(validAnnos)} with valid Processes with each {len(order)} subprocesses"
)
return validAnnos
def check_excalation(functionNode):
"""
check if we need to send any mail, also create the assessment result
"""
logger = functionNode.get_logger()
model = functionNode.get_model()
logger.info(f">>>> in check_excalation {functionNode.get_browse_path()}")
sendMailFunction = functionNode.get_child("sendMailFunction").get_target()
messages = functionNode.get_child("messages").get_leaves()
assessmentValues = functionNode.get_child("assessmentValues").get_value() #{"default":"green","continue":"yellow","critical":"red","unconfirmed":"red"}
lookBack = functionNode.get_child("assessmentLookback").get_value()
levels = {k:idx for idx,k in enumerate(assessmentValues)} # give confirm value get level
output = {idx:assessmentValues[k] for idx,k in enumerate(assessmentValues)} #give level get output string
assessment = 0 # take the least important as start point
now = time.time()
for msg in messages:
mustSend= msg.get_child("mustEscalate")
if mustSend and mustSend.get_value()== True:
#prepare a mail from the message
body = {child.get_name():child.get_value() for child in msg.get_children()}
logger.debug(f"email body {body}")
sendMailFunction.get_child("body").set_value(json.dumps(body,indent=4))
try:
model.disable_observers()
mustSend.set_value(False) # flag this message as done
finally:
model.enable_observers()
#now send the email and wait for ready
try:
sendMailFunction.execute_synchronous()
except:
model.log_error() # must grab the error here, otherwise the whole function breaks
alarmTime = dates.date2secs(msg.get_child("startTime").get_value())
if alarmTime>now-lookBack:
#must consider this alarm
msgConfirmedString = msg.get_child("confirmed").get_value()
if msgConfirmedString in levels:
level = levels[msgConfirmedString]
else:
level = 0 # ignore unknown
if level>assessment:
assessment = level
functionNode.get_child("assessment").set_value(output[assessment])
return True
def __convert_to_ids__(self,blob):
"""
convert incoming descriptors to ids
convert times to epoch
support the __events name for a default eventnode
"""
newBlob = {}
for k, v in blob.items():
if k == "__time":
if type(v) is not list:
v = [v]
if type(v[0]) is str:
v = [dates.date2secs(t) for t in v] # convert to epoch
newBlob[k] = numpy.asarray(v)
else:
# try to convert
if k in self.varNameLookup:
id = self.varNameLookup[k].get_id()
if type(v) is not list:
v=[v]
newBlob[id]=numpy.asarray(v)
else:
if self.autoCreate:
self.logger.warning(f"__convert_to_ids__: cant find {k}.. autocreate it")
startPath = self.functionNode.get_child("autoCreateFolder").get_target().get_browse_path()
path = startPath+"."+k
id = self.model.create_node_from_path(path,properties={"type":"timeseries"})
newNode = self.model.get_node(id)
self.varNameLookup.update({id:newNode,newNode.get_browse_path():newNode,k:newNode})
newBlob[id]=numpy.asarray(v)
else:
self.logger.warning(f"__convert_to_ids__: cant find {k}, ignore!")
return newBlob
def __convert_to_ids__(self, blob):
"""
convert incoming descriptors to ids
convert times to epoch
support the __events name for a default eventnode
"""
newBlob = {}
for k, v in blob.items():
if k == "__time":
if type(v) is not list:
v = [v]
if type(v[0]) is str:
v = [dates.date2secs(t) for t in v] # convert to epoch
newBlob[k] = numpy.asarray(v)
else:
# try to convert
if k in self.varNameLookup:
id = self.varNameLookup[k].get_id()
if type(v) is not list:
v = [v]
newBlob[id] = numpy.asarray(v)
else:
self.logger.error(f"__convert_to_ids__: cant find {k}")
return newBlob
def events_to_annotations(functionNode):
"""
this function creates annotations from events
"""
logger = functionNode.get_logger()
logger.info("==>>>> events_to_annotations "+functionNode.get_browse_path())
progressNode = functionNode.get_child("control").get_child("progress")
newAnnosNode = functionNode.get_child("annotations")
differential = functionNode.get_child("differential").get_value()
valueNotifyNodeIds = [] #this variable will hold node id to post a value notification (the nodes that were open before)
interleaved = functionNode.get_child("interleaved")
if interleaved:
#if the node is there
interleaved = interleaved.get_value()
if differential:
processedUntil = dates.date2secs(functionNode.get_child("processedUntil").get_value(),ignoreError=False)
if not processedUntil:
processedUntil = 0
else:
processedUntil = 0
lastTimeSeen = processedUntil # initialize, this variable with find the latest time entry
# eventSelection is a dict to translate and select the events from the event series
# eg {"machine1.init":"Preparation", "machine1.op2":"Printing"}
# we will select the "machine1.init and the "macine1.op2" events and name the annotation type as Preparation, Printing
eventSelection = functionNode.get_child("eventSelection").get_value()
progressNode.set_value(0)
m = functionNode.get_model()
startEvents = {v[0]:k for k,v in eventSelection.items()} # always the first event
endEvents = {v[1]:k for k, v in eventSelection.items()} # always the second event
evs = functionNode.get_child("eventSeries").get_targets() # a list of nodes where the events are located
if not differential:
#delete all old annos
annos = newAnnosNode.get_children()
if annos:
try:
m.disable_observers()
for anno in annos:
anno.delete()
finally:
m.enable_observers()
m.notify_observers(newAnnosNode.get_id(), "children")
openAnnos = [] # no open annotatations per definition
else:
#differential process: we don't delete the old but get all open annotations
openAnnos = []
annos = newAnnosNode.get_children()
for anno in annos:
if "open" in anno.get_child("tags").get_value():
openAnnotation = {
"tags": anno.get_child("tags").get_value(), #using "tag" to note that an open annotation has only one tag
"startTime": anno.get_child("startTime").get_value(),
"node": anno
}
openAnnos.append(openAnnotation)
#now collect all events filtered by the selection of events and the time (if differential process)
#we build up a list of dicts with ["event"."hello", "startTime":1234494., "endTime":2345346.:
filter = list(startEvents.keys())+list(endEvents.keys())
#from now on we use these lists
newAnnotations = [] # the annotations to be created new, a list of dicts, it will also contain "updates" for existing nodes
# those update entries will be found by the key "node" which holds the node that has to be updated
#openAnnos : a list of # {"key":"litu":{"startTime":132412334,"node"} # will also hold the node if we have one already
#put the open annotations in a fast look up table
if len(openAnnos) > 1 and not interleaved:
logger.error("there should be only one open annotation at max in non-interleaved mode")
#now iterate over all events
for ev in evs:
data = ev.get_event_series(eventFilter = filter) #["__time":... "eventStrings"....]
times = data["__time"]
eventStrings = data["eventStrings"]
if differential:
#also filter by time
new = times > processedUntil
times = times[new]
indexableEventStrings = numpy.asarray(eventStrings,dtype=numpy.str)
eventStrings = indexableEventStrings[new]
for index in range(len(times)):
evStr = eventStrings[index]
tim = times[index]
if tim>lastTimeSeen:
lastTimeSeen = tim
print(f"ev:{evStr}, tag:{tag}, open Annos {openAnnos}")
if evStr in startEvents:
tag = startEvents[evStr]
#this is a start of a new event
print("is start")
if openAnnos:
# we have at least on e annotation running already:
# in non-interleaved wo submitt all open annotations as anomaly
# in interleaved mode we only submit those with the same event as anomaly
# if the "open" entry was from an existing annotation in the tree, we also put that
# node handle to later use if for updating the annotation and not creating it new
newOpenAnnos = []
for openAnnotation in openAnnos:
if not interleaved or tag in openAnnotation["tags"]:
# we must close this open annotation as anomaly
# build the anomaly annotations
anno = {
"type": "time",
"endTime": dates.epochToIsoString(tim, zone='Europe/Berlin'),
"startTime": openAnnotation["startTime"],
"tags": ["anomaly",tag]
}
if "node" in openAnnotation:
anno["node"] = openAnnotation["node"]
newAnnotations.append(anno) # put the anomaly node there
else:
#keep this annotation
newOpenAnnos.append(openAnnotation)
openAnnos = newOpenAnnos
#now remember the current as the open one
openAnnotation = {
"startTime":dates.epochToIsoString(tim, zone='Europe/Berlin'),
"tags":[tag]
}
openAnnos.append(openAnnotation)
#not an else if, because it can be in both start and end events
if evStr in endEvents:
print("is end")
#this is an end event, see if we have a matching open annotation
tag = endEvents[evStr]
newOpenAnnos = []
for openAnnotation in openAnnos:
if tag in openAnnotation["tags"]:
#take this annotation, we can close it
anno = {
"type": "time",
"endTime": dates.epochToIsoString(tim, zone='Europe/Berlin'),
"startTime": openAnnotation["startTime"],
"tags": [tag]
}
if "node" in openAnnotation:
# if it was an existing annotation, we also put the anno handle to make an "update"
# instead of a new creation further down
anno["node"]=openAnnotation["node"]
newAnnotations.append(anno)
else:
newOpenAnnos.append(openAnnotation)#keep this one
if len(newOpenAnnos) == len(openAnnos):
logger.warning(f"annotation creation ende without start {tim} {evStr}")
openAnnos = newOpenAnnos
#print(f"open annotations {openAnnos} new annos {newAnnotations}")
#now create the annotations
logger.debug(f"creating {len(newAnnotations)} annotation, have {openAnnos} open annotations")
#print(f"creating {len(newAnnotations)} annotation, have {len(openAnnos)} open annotations")
m.disable_observers()
try:
if differential:
# in the open annotations list, we will find
# - open annotations that have existed before and have not found an update yet and were not closed
# - new open annotations that have started now
for openAnnotation in openAnnos:
nowIso = dates.epochToIsoString(time.time(),zone='Europe/Berlin')
if "node" in openAnnotation:
# this existed before, we just update the endtime
node = openAnnotation["node"]
node.get_child("endTime").set_value(nowIso) # set the current time as end time
valueNotifyNodeIds.append(node.get_child("endTime").get_id())
else:
#this is a new open anno:
entry = {
"type": "time",
"endTime": nowIso,
"startTime": openAnnotation["startTime"],
"tags": openAnnotation["tags"]+["open"]
}
newAnnotations.append(entry) # put it to the creation list
#create and update new annotations
for anno in newAnnotations:
if not "node" in anno:
newAnno = newAnnosNode.create_child(type="annotation")
for k, v in anno.items():
newAnno.create_child(properties={"name": k, "value": v, "type": "const"})
else:
#this is an update, typically a "close" or an extension of an open annotation
node = anno["node"]
node.get_child("endTime").set_value(anno["endTime"])
node.get_child("tags").set_value(anno["tags"])
valueNotifyNodeIds.append(node.get_child("endTime").get_id())
except Exception as ex:
logger.error(f"error in events_to_annotations {ex}")
m.enable_observers()
m.notify_observers(newAnnosNode.get_id(), "children")
#also notify the value changes of annotations that existed before
if valueNotifyNodeIds:
m.notify_observers(valueNotifyNodeIds,"value")# also notify the value change for the UI to adjust the annotation
#if time.time()>lastTimeSeen:
# lastTimeSeen=time.time()
isoDate = dates.epochToIsoString(lastTimeSeen,zone="Europe/Berlin")
functionNode.get_child("processedUntil").set_value(isoDate) # this is for me, next time I can check what has been processed
return True
def update(functionNode, startTime=0, offset=None):
if functionNode.get_name() != "update":
functionNode = functionNode.get_parent().get_child("update")
motif = functionNode.get_parent().get_child("EnvelopeMiner").get_child(
"motif").get_target()
widget = functionNode.get_parent().get_child("EnvelopeMiner").get_child(
"widget").get_target()
logger = functionNode.get_logger()
#fil = motif.get_child("envelopeFilter")
#sample = motif.get_child("envelopeSamplingPeriod")
lMax = None
lMin = None
exPe = None
#children = motif.get_get_children()
for child in motif.get_child("envelope").get_children():
if "_limitMax" in child.get_name():
lMax = child
elif "_limitMin" in child.get_name():
lMin = child
elif "_expected" in child.get_name():
exPe = child
#now get the data and write the new envelope
start = dates.date2secs(motif.get_child("startTime").get_value())
end = dates.date2secs(motif.get_child("endTime").get_value())
period = motif.get_child("envelope.samplingPeriod").get_value()
times = numpy.arange(start, end, period)
ts = motif.get_child("variable").get_target().get_time_series(
start, end, resampleTimes=times)
data = ts["values"]
#offset the times, values if necessary
if startTime != 0:
diff = startTime - start
times = times + diff
#value offset
ts = motif.get_child("variable").get_target().get_time_series(
resampleTimes=times)
if type(offset) is type(None):
dataDiff = ts["values"][0] - data[0]
else:
dataDiff = offset
data = data + dataDiff
freedom = motif.get_child("envelope.freedom").get_value()
dynFreedom = motif.get_child("envelope.dynamicFreedom").get_value()
diff = max(data) - min(data)
upper = data + diff * freedom * factorY # add x% of the diff to the upper
lower = data - diff * freedom * factorY
expect = data
#now also shift left and right to make it equally "wide" as "high"
if 1:
shift = int(float(len(times)) * freedom * factorX)
if shift == 0:
shift = 1 #at least 1
fillMax = numpy.min(upper)
fillMin = numpy.max(lower)
totalShift = shift
logger.debug(
f"envelope shift size is {shift}, freedom is {freedom} totallen is{len(times)} shiftsize/len ={float(shift)/float(len(times))}"
)
autoStepSize = float(shift) / float(len(times)) / 2
#then our step size is half the width to make sure we get the original motif
newupper = numpy.copy(upper)
newlower = numpy.copy(lower)
steps = set(list(numpy.linspace(1, totalShift, 8, dtype=int)))
for shift in steps:
right = numpy.append(upper[shift:], [fillMax] * shift)
left = numpy.append([fillMax] * shift, upper[:-shift])
newupper = numpy.max([left, newupper, right], axis=0)
right = numpy.append(lower[shift:], [fillMin] * shift)
left = numpy.append([fillMin] * shift, lower[:-shift])
newlower = numpy.min([left, newlower, right], axis=0)
upper = newupper
lower = newlower
#xxx todo dynamic freedom
model = functionNode.get_model() #get the model Api
try:
model.disable_observers()
numberSamples = len(times)
motif.get_child("envelope.numberSamples").set_value(
numberSamples) # the number of samples
#also set the possible step size
#step = motif.get_child("envelope.step").get_value()
#if step > numberSamples:
# step = numberSamples
#also set the limits
#motif.get_child("envelope.step").set_properties({"value":step,"validation":{"limits":[1,numberSamples]}})
if lMax:
lMax.set_time_series(upper, times)
if lMin:
lMin.set_time_series(lower, times)
if exPe:
exPe.set_time_series(expect, times)
finally:
model.enable_observers()
#model.notify_observers(motif.get_id(), "children")
autoStepNode = functionNode.get_child("autoStepSize")
if not autoStepNode or autoStepNode.get_value() == True:
logger.debug(f"autostep size enabled, set value {autoStepSize}")
motif.get_child("envelope.step").set_value(autoStepSize)
model.notify_observers(lMax.get_id(), "value")
return True
def replay(functionNode):
logger = functionNode.get_logger()
logger.info("==>>>> replay " + functionNode.get_browse_path())
progressNode = functionNode.get_child("control").get_child("progress")
progressNode.set_value(0)
functionNode.get_child("control.signal").set_value(None)
widget = functionNode.get_child("widget").get_target()
if widget:
#try to get the current anno
annotation = widget.get_child(
"hasAnnotation.selectedAnnotations").get_target()
if annotation:
#remember the anno
functionNode.get_child("annotation").add_references(annotation,
deleteAll=True)
annotation = functionNode.get_child("annotation").get_target()
if not annotation:
logger.error("replay:no annotation/widget given")
return False
startTime = dates.date2secs(annotation.get_child("startTime").get_value())
endTime = dates.date2secs(annotation.get_child("endTime").get_value())
#now get the data and build a table
varNodes = functionNode.get_child("input").get_leaves()
nodesLookup = {node.get_id(): node for node in varNodes}
#we build three array: timepoint, nodes in
table = pd.DataFrame({"__time": []})
latestTime = 0
for var in varNodes:
if var.get_type() == "timeseries":
data = var.get_time_series(startTime, endTime)
new = pd.DataFrame({
"__time": data["__time"],
var.get_id(): data["values"]
})
table = table.append(new, ignore_index=True)
#now find the last time
times = var.get_time_series()["__time"] #all the data
if len(times):
latestTime = max(latestTime, times[-1])
elif var.get_type() == "eventseries":
data = var.get_event_series(startTime, endTime)
new = pd.DataFrame({
"__time": data["__time"],
var.get_id(): data["eventstrings"]
})
table = table.append(new, ignore_index=True)
#now find the last time
times = var.get_event_series()["__time"] #all the data
if len(times):
latestTime = max(latestTime, times[-1])
#table.sort_values("__time", inplace=True)
table = remove_dublicate_times(table)
#print(table)
#now we have the data sorted and dublicates removed in the pd frame, let's replay
offset = latestTime - table["__time"].iloc[-1]
#logger.debug(f"latest Time {dates.epochToIsoString(latestTime)} , offset {offset}")
repeatoffset = table["__time"].iloc[-1] - table["__time"].iloc[0]
while 1:
offset = offset + repeatoffset
logger.debug(
f"latest Time {dates.epochToIsoString(latestTime)} , offset {offset}"
)
speed = functionNode.get_child("replaySpeed").get_value()
url = functionNode.get_child("url").get_value()
format = functionNode.get_child("format").get_value()
params = functionNode.get_child("parameters").get_value()
directApi = functionNode.get_child("directApi").get_value()
if directApi:
targetNode = functionNode.get_model().get_node(params)
for index, row in table.iterrows():
#build the data json
#all timeseries
if format == "21_execute":
timeblob = {}
eventblob = {}
for k, v in row.items():
if k == "__time":
timeblob["__time"] = v + offset
eventblob["__time"] = v + offset
elif numpy.isfinite(v):
if nodesLookup[k].get_type() == "timeseries":
timeblob[k] = v
else:
eventblob[k] = v
else:
logger.warning("data not finite")
body = {
"node": params,
"function": "feed",
"parameter": []
} #{"type": "timeseries", "data": blob}]}
if len(timeblob) > 1:
body["parameter"].append({
"type": "timeseries",
"data": timeblob
})
if len(eventblob) > 1:
body["parameter"].append({
"type": "eventseries",
"data": eventblob
})
else:
logger.warning("replay:unknown format")
body = {}
#try to send out
if not directApi:
try:
if body:
res = requests.post(url,
data=json.dumps(body),
timeout=5)
logger.debug("sent out replay packet")
except Exception as ex:
logger.error(ex)
else:
#send this directly to the node in the tree
targetNode.get_object().feed(body["parameter"])
time.sleep(speed)
if functionNode.get_child("control.signal").get_value() == "stop":
break
if functionNode.get_child("control.signal").get_value(
) == "stop" or not functionNode.get_child("loop").get_value():
break
else:
logger.debug("loop replay")
return True
def rca2(functionNode):
logger = functionNode.get_logger()
logger.info("==>>>> in rca2 (root cause analysis " +
functionNode.get_browse_path())
progressNode = functionNode.get_child("control").get_child("progress")
progressNode.set_value(0.1)
m = functionNode.get_model()
report = '<i>REPORT</i><br><div style="font-size:85%">'
annotations = functionNode.get_child("annotations").get_leaves()
#order = ["Step"+str(no) for no in range(1,19)]
order = ["Phase" + str(no) for no in range(3, 28)]
order = functionNode.get_child("annotationsOrder").get_value()
annotations = data_cleaning(annotations, order=order,
logger=logger) #Step1,Step2,...Step18
report += (f"found {len(annotations)} valid processes <br>")
#for now, flatten them out
annotations = [
subprocess for process in annotations for subprocess in process
]
algo = functionNode.get_child("selectedAlgorithm").get_value()
target = functionNode.get_child("selectedTarget").get_target()
progressNode.set_value(0.3)
#now we are building up the table by iterating all the children in "selection"
entries = functionNode.get_child("selection").get_children()
table = {"target": []}
firstVariable = True
for entry in entries:
logger.debug(f"entry {entry.get_name()}")
#each entry is a combination of variable, tags and feature
vars = entry.get_child("selectedVariables").get_targets()
tags = entry.get_child("selectedTags").get_value()
features = entry.get_child("selectedFeatures").get_value()
#for iterate over variables
for var in vars:
logger.debug(
f"processing variable: {var.get_name()} with tags {tags} and features {features}"
)
#columnName = var.get_name()+str(tags)+m.getRandomId()
for tag in tags:
row = 0
#table[columnName]=[]# make a column
for idx, anno in enumerate(annotations):
if anno.get_child("type").get_value() != "time":
continue
if tag in anno.get_child("tags").get_value():
startTime = anno.get_child("startTime").get_value()
endTime = anno.get_child("endTime").get_value()
data = var.get_time_series(startTime,
endTime)["values"]
#we take only the values "inside" the annotation
if len(data) > 2:
data = data[1:-1]
#now create the features
for feature in features:
feat = calc_feature(data, feature)
columnName = var.get_name(
) + "_" + tag + "_" + feature
if not columnName in table:
table[columnName] = []
table[columnName].append(feat)
targetValue = get_target(
target,
(date2secs(startTime) + date2secs(endTime)) / 2)
if targetValue:
if firstVariable:
#for the first variable we also write the target
table["target"].append(targetValue)
else:
#for all others we make sure we have the same target value for that case (sanity check)
if table["target"][row] != targetValue:
logger.warning(
f'problem target {table["target"][row]} !=> {targetValue}'
)
row = row + 1
else:
logger.warning(
f"no corrrect target value for {startTime} - {endTime}"
)
firstVariable = False
#now we have the table, plot it
import json
#print(json.dumps(table,indent=2))
progressNode.set_value(0.5)
#try a model
algo = functionNode.get_child("selectedAlgorithm").get_value()
if algo == "lasso":
reg = linear_model.LassoCV()
report += " using lasso Regression with auto-hyperparams <br>"
else:
#default
report += " using linear Regression <br>"
reg = linear_model.LinearRegression() #try rigde, lasso
columnNames = []
dataTable = []
for k, v in table.items():
if k == "target":
continue
dataTable.append(v)
columnNames.append(k)
dataTable = numpy.asarray(dataTable)
x = dataTable.T
y = table["target"]
x_train, x_test, y_train, y_test = train_test_split(x, y)
reg.fit(x_train, y_train)
print(reg.coef_)
y_hat = reg.predict(x_test)
y_repeat = reg.predict(x_train)
print(f"predict: {y_hat} vs real: {y_test}")
#check over/underfitting
r_train = r2_score(y_train, y_repeat)
r_test = r2_score(y_test, y_hat)
report += "R<sup>2</sup> train= %.4g, R<sup>2</sup> test = %.4g <br>" % (
r_train, r_test)
pearsons = []
for col in x.T:
pearsons.append(pearsonr(col, y)[0])
#and finally the correlations between y and yhat
y_pearson_train = pearsonr(y_train, y_repeat)[0]
y_pearson_test = pearsonr(y_test, y_hat)[0]
report += "pearsonCorr y/y_hat train:%.4g , test:%.4g <br>" % (
y_pearson_train, y_pearson_test)
report += "regression coefficients, pearsons correlations:<br>"
for col, coef, pear in zip(columnNames, reg.coef_, pearsons):
report += "    %s:%.4g,   %.4g <br>" % (col, coef, pear)
#write report
progressNode.set_value(0.8)
report += "<div>" #close the style div
functionNode.get_child("report").set_value(report)
#make a plot
hover1 = HoverTool(tooltips=[('x,y', '$x,$y')], mode='mouse')
hover1.point_policy = 'snap_to_data'
hover1.line_policy = "nearest"
tools = [
PanTool(),
WheelZoomTool(),
BoxZoomTool(),
ResetTool(),
SaveTool(), hover1
]
title = "prediction results on " + functionNode.get_child(
"selectedAlgorithm").get_value()
fig = figure(title=title, tools=tools, plot_height=400, plot_width=500)
fig.toolbar.logo = None
curdoc().theme = Theme(json=themes.darkTheme)
fig.xaxis.major_label_text_color = themes.darkTickColor
fig.yaxis.major_label_text_color = themes.darkTickColor
fig.xaxis.axis_label = target.get_name()
fig.xaxis.axis_label_text_color = "white"
fig.yaxis.axis_label = "predicted Values for " + target.get_name()
fig.yaxis.axis_label_text_color = "white"
fig.circle(y_train,
y_repeat,
size=4,
line_color="white",
fill_color="white",
name="train",
legend_label="train")
fig.circle(y_test,
y_hat,
line_color="#d9b100",
fill_color="#d9b100",
size=4,
name="test",
legend_label="test")
fileName = functionNode.get_child("outputFileName").get_value()
filePath = os.path.join(myDir, './../web/customui/' + fileName)
fig.legend.location = "top_left"
output_file(filePath, mode="inline")
save(fig)
return True
def rca(functionNode):
logger = functionNode.get_logger()
logger.info("==>>>> in rca (root cause analysis " +
functionNode.get_browse_path())
progressNode = functionNode.get_child("control").get_child("progress")
progressNode.set_value(0.1)
variables = functionNode.get_child("selectedVariables").get_leaves()
tag = functionNode.get_child("selectedTags").get_value() #only one tag
annotations = functionNode.get_child("annotations").get_leaves()
feature = functionNode.get_child("selectedFeatures").get_value()
algo = functionNode.get_child("selectedAlgorithms").get_value()
target = functionNode.get_child("selectedTarget").get_target()
p = Progress(progressNode)
p.set_divisor(len(annotations) / 0.5)
p.set_offset(0.1)
#now create the data as x-y
results = {"x": [], "y": []}
var = variables[0]
#now iterate over all annotations of the matching type and create feature
for idx, anno in enumerate(annotations):
p.set_progress(idx)
if (anno.get_child("type").get_value()
== "time") and (tag in anno.get_child("tags").get_value()):
startTime = anno.get_child("startTime").get_value()
endTime = anno.get_child("endTime").get_value()
data = var.get_time_series(startTime, endTime)
#now create the feature
feat = calc_feature(data["values"], feature)
targetValue = get_target(
target, (date2secs(startTime) + date2secs(endTime)) / 2)
if feat and targetValue and numpy.isfinite(
feat) and numpy.isfinite(targetValue):
results["x"].append(feat)
results["y"].append(targetValue)
else:
logger.warning(
f"no result for {var.get_name} @ {startTime}, anno:{tag}, feat:{feat}, target: {target}"
)
#now we have all the x-y
progressNode.set_value(0.7)
fig = figure(title="x-y Correlation Plot " + var.get_name(),
tools=[PanTool(),
WheelZoomTool(),
ResetTool(),
SaveTool()],
plot_height=300,
x_axis_label=feature + "(" + var.get_name() + ") @ " + tag,
y_axis_label=target.get_name())
fig.toolbar.logo = None
curdoc().theme = Theme(json=themes.darkTheme)
fig.xaxis.major_label_text_color = themes.darkTickColor
fig.yaxis.major_label_text_color = themes.darkTickColor
fig.scatter(x=results["x"],
y=results["y"],
size=5,
fill_color="#d9b100",
marker="o")
fileName = functionNode.get_child("outputFileName").get_value()
filePath = os.path.join(myDir, './../web/customui/' + fileName)
progressNode.set_value(0.8)
output_file(
filePath, mode="inline"
) #inline: put the bokeh .js into this html, otherwise the default cdn will be taken, might cause CORS problems)
save(fig)
#print(results)
return True
def envelope_miner(functionNode):
logger = functionNode.get_logger()
signal = functionNode.get_child("control.signal")
logger.info("==>>>> in envelope_miner " + functionNode.get_browse_path())
progressNode = functionNode.get_child("control").get_child("progress")
functionNode.get_child("results").set_value([])
progressNode.set_value(0)
signal.set_value(None)
# also make sure we are in the right jump
# when we have jumped to a different result, the envelope time series are set to different values
# (the ones from the according match)
# but now we want to use the envelope time series for the mining so we need to set it back to the
# motif time with the chosen parameters, we do that with the help of the update function
update(functionNode)
motif = functionNode.get_child("motif").get_target()
variable = motif.get_child("variable").get_target()
holeSize = functionNode.get_child("holeSize").get_value()
ts = variable.get_time_series()
samplePeriod = motif.get_child("envelope.samplingPeriod").get_value()
samplePointsPerWindow = motif.get_child(
"envelope.numberSamples").get_value()
stepSizePercent = motif.get_child(
"envelope.step").get_value() # in percent
if stepSizePercent == 0:
stepSize = 1 # choose one point for 0% settings
else:
stepSize = int(float(samplePointsPerWindow) * float(stepSizePercent))
if stepSize < 1:
stepSize = 1 # at least one point advances per step
samplePointsPerWindow = motif.get_child(
"envelope.numberSamples").get_value()
windowMaker = streaming.Windowing(
samplePeriod=samplePeriod,
stepSize=stepSize,
maxHoleSize=holeSize,
samplePointsPerWindow=samplePointsPerWindow)
numberOfWindows = (ts["__time"][-1] -
ts["__time"][0]) / samplePeriod / stepSize #approx
windowTime = samplePointsPerWindow * samplePeriod
logger.debug(
f"producing {numberOfWindows} windows, point per window ={samplePointsPerWindow}, stepsize {stepSizePercent*100}% => {stepSize} pt, sample Period {samplePeriod}"
)
windowMaker.insert(ts["__time"], ts["values"])
#old
#upper = motif.get_child("envelope."+variable.get_name()+"_limitMax").get_time_series()["values"]
#lower = motif.get_child("envelope."+variable.get_name()+"_limitMin").get_time_series()["values"]
#expected = motif.get_child("envelope."+variable.get_name()+"_expected").get_time_series()["values"]
# get the motif data with times
motifStart = dates.date2secs(motif.get_child("startTime").get_value())
motifEnd = dates.date2secs(motif.get_child("endTime").get_value())
upper = motif.get_child("envelope." + variable.get_name() +
"_limitMax").get_time_series(motifStart, motifEnd)
lower = motif.get_child("envelope." + variable.get_name() +
"_limitMin").get_time_series(motifStart, motifEnd)
expected = motif.get_child("envelope." + variable.get_name() +
"_expected").get_time_series(
motifStart, motifEnd)
if functionNode.get_child("maxNumberOfMatches"):
maxMatches = functionNode.get_child("maxNumberOfMatches").get_value()
else:
maxMatches = None
matches = []
i = 0
last = 0
for w in windowMaker.iterate():
# now we have the window w =[t,v] which is of correct length and resampled, let's compare it
# to the motif
# first the offset
offset = w[1][0] - expected["values"][0]
x = w[1] - offset
below = upper["values"] - x
above = x - lower["values"]
diff = numpy.sum(numpy.power(x - expected["values"], 2))
if numpy.all(below > 0) and numpy.all(above > 0):
logger.debug(
f"match @ {w[1][0]}, iteration: {float(i)/float(numberOfWindows)}"
)
matches.append({
"startTime":
dates.epochToIsoString(w[0][0], 'Europe/Berlin'),
"endTime":
dates.epochToIsoString(w[0][0] + windowTime, 'Europe/Berlin'),
"match":
diff,
"epochStart":
w[0][0],
"epochEnd":
w[0][0] + windowTime,
"offset":
offset,
"format":
my_date_format(w[0][0]) + "  (match=%2.3f)" % diff
#"below":list(numpy.copy(below)),
#"above":list(numpy.copy(above)),
#"x":list(numpy.copy(x)),
#"w":list(numpy.copy(w[1])),
#"upper":list(numpy.copy(upper["values"])),
#"lower":list(numpy.copy(lower["values"])),
#"expected":list(numpy.copy(expected["values"]))
})
if maxMatches and len(matches) == maxMatches:
break
i = i + 1
progress = round(float(i) / numberOfWindows *
20) #only 5% units on the progress bar
if progress != last:
progressNode.set_value(float(i) / numberOfWindows)
last = progress
if signal.get_value() == "stop":
break
#remove trivial matches inside half of the window len, we just take the best inside that area
cleanMatches = []
tNow = 0
for m in matches:
dif = m["epochStart"] - tNow
if dif < (windowTime / 2):
#check if this is a better match
if m["match"] < cleanMatches[-1]["match"]:
#exchange it to the better match
cleanMatches[-1] = m
continue
else:
cleanMatches.append(m)
tNow = m["epochStart"]
#now sort the matches by match value and rescale them
if cleanMatches == []:
functionNode.get_child("results").set_value([])
else:
matchlist = numpy.asarray([m["match"] for m in cleanMatches])
scaleMax = numpy.max(matchlist)
scaleMin = numpy.min(matchlist)
matchlist = (matchlist - scaleMin) / (scaleMax - scaleMin) * 100
sortIndices = numpy.argsort([m["match"] for m in cleanMatches])
sortMatches = []
for idx in sortIndices:
m = cleanMatches[idx]
m["format"] = my_date_format(
m["epochStart"]) + "  (distance=%.3g)" % matchlist[idx]
sortMatches.append(cleanMatches[idx])
#now create the annotations and notify them in one event
if functionNode.get_child(
"createAnnotations") and functionNode.get_child(
"createAnnotations").get_value():
myModel = functionNode.get_model()
myModel.disable_observers()
annoFolder = functionNode.get_child("annotations")
if maxMatches:
_create_annos_from_matches(annoFolder,
sortMatches,
maxMatches=maxMatches)
myModel.enable_observers()
if maxMatches != 0:
myModel.notify_observers(annoFolder.get_id(), "children")
if maxMatches != 0:
display_matches(functionNode, True) # turn on the annotations
functionNode.get_child("results").set_value(sortMatches)
progressNode.set_value(1)
return True
def varstatistics(functionNode):
logger = functionNode.get_logger()
logger.info("==>>>> statistics " + functionNode.get_browse_path())
progressNode = functionNode.get_child("control").get_child("progress")
progressNode.set_value(0)
#functionNode.get_child("control.signal").set_value(None)
vars = functionNode.get_child("variable").get_targets()
widget = functionNode.get_child("widget").get_target()
bins = functionNode.get_child("bins").get_value()
tags = functionNode.get_child("annotations").get_value()
startTime = date2secs(widget.get_child("startTime").get_value())
endTime = date2secs(widget.get_child("endTime").get_value())
vars = {var.get_id(): {"node": var} for var in vars}
#first 30% progress:
prog = Progress(progressNode)
progressNode.set_value(0.1)
prog.set_offset(0.1)
#prog.set_divisor()
if tags:
allAnnoNodes = widget.get_child(
"hasAnnotation.annotations").get_leaves()
allAnnos = []
prog.set_divisor(len(allAnnoNodes) / 0.2)
for index, node in enumerate(allAnnoNodes):
prog.set_progress(index)
if node.get_child("type").get_value() == "time":
thisTags = node.get_child("tags").get_value()
if any(tag in tags for tag in thisTags):
anno = {}
for child in node.get_children():
anno[child.get_name()] = child.get_value()
if date2secs(anno["startTime"]) >= startTime and date2secs(
anno["endTime"]
) <= endTime: #take this anno only if it is inside the current start/end time
allAnnos.append(anno)
if allAnnos == []:
give_up(functionNode, "no matching annotations in selected time")
return False
else:
allAnnos = []
progressNode.set_value(0.3)
logger.debug(f"statistics annotations to look at: {len(allAnnos)}")
prog.set_offset(0.3)
totalAnnos = max(len(allAnnos), 1)
totalCount = len(vars) * totalAnnos
prog.set_divisor(totalCount / 0.3)
totalValids = 0
for varIndex, var in enumerate(vars):
info = vars[var]
if tags:
#iterate over all start and end times
values = numpy.asarray([], dtype=numpy.float64)
for annoIndex, anno in enumerate(allAnnos):
thisValues = info["node"].get_time_series(
anno["startTime"], anno["endTime"])["values"]
values = numpy.append(values, thisValues)
myCount = varIndex * totalAnnos + annoIndex
prog.set_progress(myCount)
else:
values = info["node"].get_time_series(startTime, endTime)["values"]
valids = numpy.count_nonzero(~numpy.isfinite(values))
totalValids += valids
hist, edges = numpy.histogram(values, bins=bins)
hist = hist / len(values) #normalize
info["hist"] = hist
info["edges"] = edges
#make a plot
if totalValids == 0:
give_up(
functionNode,
"all Variables are have no data in the time and annotations selected"
)
return False
progressNode.set_value(0.6)
hover1 = HoverTool(tooltips=[('x,y', '$x,$y')], mode='mouse')
hover1.point_policy = 'snap_to_data'
hover1.line_policy = "nearest"
tools = [
PanTool(),
WheelZoomTool(),
BoxZoomTool(),
ResetTool(),
SaveTool(), hover1
]
title = "Statistics of " + str(
[info["node"].get_name() for var, info in vars.items()])
if tags:
title = title + " in annotation: " + str(tags)
fig = figure(title=title, tools=tools, plot_height=300)
fig.toolbar.logo = None
curdoc().theme = Theme(json=themes.darkTheme)
fig.xaxis.major_label_text_color = themes.darkTickColor
fig.yaxis.major_label_text_color = themes.darkTickColor
for index, var in enumerate(vars):
info = vars[var]
col = themes.darkLineColors[index]
hist = info["hist"]
edges = info["edges"]
fig.quad(top=hist,
bottom=0,
left=edges[:-1],
right=edges[1:],
fill_color=col,
line_color=col,
alpha=0.8,
legend_label=info["node"].get_name())
fig.legend.location = "top_left"
fileName = functionNode.get_child("fileName").get_value()
filePath = os.path.join(myDir, './../web/customui/' + fileName)
# now make the trend box plot, but only for tags
# for each variable we create statistics for the annotations and prepare the data
# {"node":Node(), "boxLower":[], "boxUpper", "mean", "limitUpper", "limitLower"}
#
startTime = date2secs(widget.get_child("startTime").get_value(
)) #we only take tags that are inside the current zoom of the widgets
endTime = date2secs(widget.get_child("endTime").get_value())
boxPlots = []
allTimes = []
if tags:
for index, var in enumerate(vars):
info = {
"node": vars[var]["node"],
"boxLower": [],
"boxUpper": [],
"median": [],
"time": [],
"limitUpper": [],
"limitLower": [],
"mean": []
}
for anno in allAnnos:
data = info["node"].get_time_series(anno["startTime"],
anno["endTime"])
if len(data["values"]):
data["values"] = data["values"][numpy.isfinite(
data["values"])]
#remove the nan
if len(data["values"]):
#make the statistics
info["time"].append(numpy.median(data["__time"]) * 1000)
allTimes.append(numpy.median(data["__time"]) * 1000)
info["limitLower"].append(
numpy.quantile(data["values"], 0.01))
info["limitUpper"].append(
numpy.quantile(data["values"], 0.99))
info["boxLower"].append(
numpy.quantile(data["values"], 0.25))
info["boxUpper"].append(
numpy.quantile(data["values"], 0.75))
info["median"].append(numpy.median(data["values"]))
info["mean"].append(numpy.mean(data["values"]))
boxPlots.append(info)
format = "%Y-%m-%d-T%H:%M:%S"
custom = """var local = moment(value).tz('UTC'); return local.format();""" #%self.server.get_settings()["timeZone"]
hover = HoverTool(tooltips=[('date', '@x{%F}')],
formatters={'@x': CustomJSHover(code=custom)},
mode='mouse')
hover.point_policy = 'snap_to_data'
hover.line_policy = "nearest"
tools = [
PanTool(),
BoxZoomTool(),
WheelZoomTool(),
ResetTool(), hover,
SaveTool()
]
fig2 = figure(title="trends",
tools=tools,
plot_height=300,
x_axis_type='datetime')
fig2.xaxis.major_label_text_color = themes.darkTickColor
fig2.yaxis.major_label_text_color = themes.darkTickColor
progressNode.set_value(0.7)
fig2.xaxis.formatter = DatetimeTickFormatter(years=format,
days=format,
months=format,
hours=format,
hourmin=format,
minutes=format,
minsec=format,
seconds=format)
fig2.toolbar.logo = None
#fig2.line([1,2,3],[1,2,3])
#calc with of vbars
if len(allAnnos) > 1:
xTimesStart = min(allTimes)
xTimesEnd = max(allTimes)
width = (xTimesEnd - xTimesStart) / 2 / len(allAnnos)
else:
width = 1000000
for index, info in enumerate(boxPlots):
#each info is for one variable
col = themes.darkLineColors[index]
fig2.segment(info["time"],
info["limitUpper"],
info["time"],
info["boxUpper"],
line_color=col)
fig2.segment(info["time"],
info["limitLower"],
info["time"],
info["boxLower"],
line_color=col)
width = 20
#fig2.vbar(info["time"],width=width,bottom=info["median"],top=info["boxUpper"],fill_color=col,line_color="black",width_units='screen')
#fig2.vbar(info["time"],width=width,bottom=info["boxLower"],top=info["median"],fill_color=col,line_color="black",width_units='screen')
#upper box
sizUpper = numpy.asarray(info["boxUpper"]) - numpy.asarray(
info["median"])
medUpper = numpy.asarray(info["median"]) + sizUpper / 2
fig2.rect(x=info["time"],
y=medUpper,
width_units='screen',
width=20,
height=sizUpper,
fill_color=col,
line_color="black")
#lower box
sizLower = numpy.asarray(info["median"]) - numpy.asarray(
info["boxLower"])
medLower = numpy.asarray(info["median"]) - sizLower / 2
fig2.rect(x=info["time"],
y=medLower,
width_units='screen',
width=20,
height=sizLower,
fill_color=col,
line_color="black")
#sort data for line
x = numpy.asarray(info["time"])
y = numpy.asarray(info["mean"])
order = numpy.argsort(x)
x = x[order]
y = y[order]
fig2.line(x, y, line_color=col)
progressNode.set_value(0.8)
else:
#no fig2
pass
output_file(
filePath, mode="inline"
) #inline: put the bokeh .js into this html, otherwise the default cdn will be taken, might cause CORS problems
if tags:
save(layout([[fig], [fig2]]))
else:
save(fig)
return True