コード例 #1
0
ファイル: rca.py プロジェクト: smartyal/21datalab
def rca(functionNode):
    logger = functionNode.get_logger()
    logger.info("==>>>> in rca (root cause analysis " +
                functionNode.get_browse_path())
    progressNode = functionNode.get_child("control").get_child("progress")
    progressNode.set_value(0.1)

    variables = functionNode.get_child("selectedVariables").get_leaves()
    tag = functionNode.get_child("selectedTags").get_value()  #only one tag
    annotations = functionNode.get_child("annotations").get_leaves()
    feature = functionNode.get_child("selectedFeatures").get_value()
    algo = functionNode.get_child("selectedAlgorithms").get_value()
    target = functionNode.get_child("selectedTarget").get_target()

    p = Progress(progressNode)
    p.set_divisor(len(annotations) / 0.5)
    p.set_offset(0.1)
    #now create the data as x-y

    results = {"x": [], "y": []}
    var = variables[0]
    #now iterate over all annotations of the matching type and create feature
    for idx, anno in enumerate(annotations):
        p.set_progress(idx)
        if (anno.get_child("type").get_value()
                == "time") and (tag in anno.get_child("tags").get_value()):
            startTime = anno.get_child("startTime").get_value()
            endTime = anno.get_child("endTime").get_value()
            data = var.get_time_series(startTime, endTime)
            #now create the feature
            feat = calc_feature(data["values"], feature)
            targetValue = get_target(
                target, (date2secs(startTime) + date2secs(endTime)) / 2)
            if feat and targetValue and numpy.isfinite(
                    feat) and numpy.isfinite(targetValue):
                results["x"].append(feat)
                results["y"].append(targetValue)
            else:
                logger.warning(
                    f"no result for {var.get_name} @ {startTime}, anno:{tag}, feat:{feat}, target: {target}"
                )

    #now we have all the x-y

    progressNode.set_value(0.7)
    fig = figure(title="x-y Correlation Plot " + var.get_name(),
                 tools=[PanTool(),
                        WheelZoomTool(),
                        ResetTool(),
                        SaveTool()],
                 plot_height=300,
                 x_axis_label=feature + "(" + var.get_name() + ") @ " + tag,
                 y_axis_label=target.get_name())
    fig.toolbar.logo = None
    curdoc().theme = Theme(json=themes.darkTheme)
    fig.xaxis.major_label_text_color = themes.darkTickColor
    fig.yaxis.major_label_text_color = themes.darkTickColor

    fig.scatter(x=results["x"],
                y=results["y"],
                size=5,
                fill_color="#d9b100",
                marker="o")
    fileName = functionNode.get_child("outputFileName").get_value()
    filePath = os.path.join(myDir, './../web/customui/' + fileName)
    progressNode.set_value(0.8)
    output_file(
        filePath, mode="inline"
    )  #inline: put the bokeh .js into this html, otherwise the default cdn will be taken, might cause CORS problems)
    save(fig)

    #print(results)

    return True
コード例 #2
0
ファイル: varstatistics.py プロジェクト: smartyal/21datalab
def varstatistics(functionNode):
    logger = functionNode.get_logger()
    logger.info("==>>>> statistics " + functionNode.get_browse_path())
    progressNode = functionNode.get_child("control").get_child("progress")
    progressNode.set_value(0)
    #functionNode.get_child("control.signal").set_value(None)

    vars = functionNode.get_child("variable").get_targets()
    widget = functionNode.get_child("widget").get_target()
    bins = functionNode.get_child("bins").get_value()
    tags = functionNode.get_child("annotations").get_value()
    startTime = date2secs(widget.get_child("startTime").get_value())
    endTime = date2secs(widget.get_child("endTime").get_value())

    vars = {var.get_id(): {"node": var} for var in vars}

    #first 30% progress:
    prog = Progress(progressNode)
    progressNode.set_value(0.1)
    prog.set_offset(0.1)
    #prog.set_divisor()

    if tags:
        allAnnoNodes = widget.get_child(
            "hasAnnotation.annotations").get_leaves()
        allAnnos = []
        prog.set_divisor(len(allAnnoNodes) / 0.2)
        for index, node in enumerate(allAnnoNodes):
            prog.set_progress(index)
            if node.get_child("type").get_value() == "time":
                thisTags = node.get_child("tags").get_value()
                if any(tag in tags for tag in thisTags):
                    anno = {}
                    for child in node.get_children():
                        anno[child.get_name()] = child.get_value()
                    if date2secs(anno["startTime"]) >= startTime and date2secs(
                            anno["endTime"]
                    ) <= endTime:  #take this anno only if it is inside the current start/end time
                        allAnnos.append(anno)
        if allAnnos == []:
            give_up(functionNode, "no matching annotations in selected time")
            return False
    else:
        allAnnos = []

    progressNode.set_value(0.3)

    logger.debug(f"statistics annotations to look at: {len(allAnnos)}")
    prog.set_offset(0.3)
    totalAnnos = max(len(allAnnos), 1)
    totalCount = len(vars) * totalAnnos

    prog.set_divisor(totalCount / 0.3)
    totalValids = 0
    for varIndex, var in enumerate(vars):
        info = vars[var]
        if tags:
            #iterate over all start and end times
            values = numpy.asarray([], dtype=numpy.float64)
            for annoIndex, anno in enumerate(allAnnos):
                thisValues = info["node"].get_time_series(
                    anno["startTime"], anno["endTime"])["values"]
                values = numpy.append(values, thisValues)
                myCount = varIndex * totalAnnos + annoIndex
                prog.set_progress(myCount)
        else:
            values = info["node"].get_time_series(startTime, endTime)["values"]

        valids = numpy.count_nonzero(~numpy.isfinite(values))
        totalValids += valids
        hist, edges = numpy.histogram(values, bins=bins)
        hist = hist / len(values)  #normalize
        info["hist"] = hist
        info["edges"] = edges

    #make a plot
    if totalValids == 0:
        give_up(
            functionNode,
            "all Variables are have no data in the time and annotations selected"
        )
        return False

    progressNode.set_value(0.6)

    hover1 = HoverTool(tooltips=[('x,y', '$x,$y')], mode='mouse')
    hover1.point_policy = 'snap_to_data'
    hover1.line_policy = "nearest"

    tools = [
        PanTool(),
        WheelZoomTool(),
        BoxZoomTool(),
        ResetTool(),
        SaveTool(), hover1
    ]

    title = "Statistics of " + str(
        [info["node"].get_name() for var, info in vars.items()])
    if tags:
        title = title + " in annotation: " + str(tags)

    fig = figure(title=title, tools=tools, plot_height=300)
    fig.toolbar.logo = None

    curdoc().theme = Theme(json=themes.darkTheme)
    fig.xaxis.major_label_text_color = themes.darkTickColor
    fig.yaxis.major_label_text_color = themes.darkTickColor

    for index, var in enumerate(vars):
        info = vars[var]
        col = themes.darkLineColors[index]
        hist = info["hist"]
        edges = info["edges"]

        fig.quad(top=hist,
                 bottom=0,
                 left=edges[:-1],
                 right=edges[1:],
                 fill_color=col,
                 line_color=col,
                 alpha=0.8,
                 legend_label=info["node"].get_name())

    fig.legend.location = "top_left"
    fileName = functionNode.get_child("fileName").get_value()
    filePath = os.path.join(myDir, './../web/customui/' + fileName)

    # now make the trend box plot, but only for tags
    # for each variable we create statistics for the annotations and prepare the data
    # {"node":Node(), "boxLower":[], "boxUpper", "mean", "limitUpper", "limitLower"}
    #

    startTime = date2secs(widget.get_child("startTime").get_value(
    ))  #we only take tags that are inside the current zoom of the widgets
    endTime = date2secs(widget.get_child("endTime").get_value())

    boxPlots = []
    allTimes = []
    if tags:
        for index, var in enumerate(vars):
            info = {
                "node": vars[var]["node"],
                "boxLower": [],
                "boxUpper": [],
                "median": [],
                "time": [],
                "limitUpper": [],
                "limitLower": [],
                "mean": []
            }
            for anno in allAnnos:
                data = info["node"].get_time_series(anno["startTime"],
                                                    anno["endTime"])
                if len(data["values"]):
                    data["values"] = data["values"][numpy.isfinite(
                        data["values"])]
                    #remove the nan
                if len(data["values"]):

                    #make the statistics
                    info["time"].append(numpy.median(data["__time"]) * 1000)
                    allTimes.append(numpy.median(data["__time"]) * 1000)
                    info["limitLower"].append(
                        numpy.quantile(data["values"], 0.01))
                    info["limitUpper"].append(
                        numpy.quantile(data["values"], 0.99))
                    info["boxLower"].append(
                        numpy.quantile(data["values"], 0.25))
                    info["boxUpper"].append(
                        numpy.quantile(data["values"], 0.75))
                    info["median"].append(numpy.median(data["values"]))
                    info["mean"].append(numpy.mean(data["values"]))
            boxPlots.append(info)

        format = "%Y-%m-%d-T%H:%M:%S"
        custom = """var local = moment(value).tz('UTC'); return local.format();"""  #%self.server.get_settings()["timeZone"]

        hover = HoverTool(tooltips=[('date', '@x{%F}')],
                          formatters={'@x': CustomJSHover(code=custom)},
                          mode='mouse')
        hover.point_policy = 'snap_to_data'
        hover.line_policy = "nearest"
        tools = [
            PanTool(),
            BoxZoomTool(),
            WheelZoomTool(),
            ResetTool(), hover,
            SaveTool()
        ]

        fig2 = figure(title="trends",
                      tools=tools,
                      plot_height=300,
                      x_axis_type='datetime')
        fig2.xaxis.major_label_text_color = themes.darkTickColor
        fig2.yaxis.major_label_text_color = themes.darkTickColor

        progressNode.set_value(0.7)

        fig2.xaxis.formatter = DatetimeTickFormatter(years=format,
                                                     days=format,
                                                     months=format,
                                                     hours=format,
                                                     hourmin=format,
                                                     minutes=format,
                                                     minsec=format,
                                                     seconds=format)
        fig2.toolbar.logo = None
        #fig2.line([1,2,3],[1,2,3])
        #calc with of vbars
        if len(allAnnos) > 1:
            xTimesStart = min(allTimes)
            xTimesEnd = max(allTimes)
            width = (xTimesEnd - xTimesStart) / 2 / len(allAnnos)
        else:
            width = 1000000

        for index, info in enumerate(boxPlots):
            #each info is for one variable
            col = themes.darkLineColors[index]
            fig2.segment(info["time"],
                         info["limitUpper"],
                         info["time"],
                         info["boxUpper"],
                         line_color=col)
            fig2.segment(info["time"],
                         info["limitLower"],
                         info["time"],
                         info["boxLower"],
                         line_color=col)

            width = 20
            #fig2.vbar(info["time"],width=width,bottom=info["median"],top=info["boxUpper"],fill_color=col,line_color="black",width_units='screen')
            #fig2.vbar(info["time"],width=width,bottom=info["boxLower"],top=info["median"],fill_color=col,line_color="black",width_units='screen')
            #upper box
            sizUpper = numpy.asarray(info["boxUpper"]) - numpy.asarray(
                info["median"])
            medUpper = numpy.asarray(info["median"]) + sizUpper / 2
            fig2.rect(x=info["time"],
                      y=medUpper,
                      width_units='screen',
                      width=20,
                      height=sizUpper,
                      fill_color=col,
                      line_color="black")

            #lower box
            sizLower = numpy.asarray(info["median"]) - numpy.asarray(
                info["boxLower"])
            medLower = numpy.asarray(info["median"]) - sizLower / 2
            fig2.rect(x=info["time"],
                      y=medLower,
                      width_units='screen',
                      width=20,
                      height=sizLower,
                      fill_color=col,
                      line_color="black")

            #sort data for line
            x = numpy.asarray(info["time"])
            y = numpy.asarray(info["mean"])
            order = numpy.argsort(x)
            x = x[order]
            y = y[order]
            fig2.line(x, y, line_color=col)

        progressNode.set_value(0.8)
    else:
        #no fig2
        pass

    output_file(
        filePath, mode="inline"
    )  #inline: put the bokeh .js into this html, otherwise the default cdn will be taken, might cause CORS problems
    if tags:
        save(layout([[fig], [fig2]]))
    else:
        save(fig)

    return True