def CalcPCAStats(Ts, Qs, NPCA, Vs, Regions): ''' Dummy calculate some key statisitics from anaylsis, Ts: The t squared hotelling statistic, Qs: The Q resisduals statistic, NPCA: The dimensionality of the model used, Vs: The fractional variance in each component over time Regions: The regions violating the limits. Return, Params: A dictionary of key statistics {"NRegions", "TsMean", "QsMean", "TsMax", "QsMax", "NPCA", "VarianceFracs"} ''' NRegions = Regions.shape[0] TsMean = np.nanmean(Ts["Ts"].values) QsMean = np.nanmean(Qs["Qs"].values) TsMax = np.nanmax(Ts["Ts"].values) QsMax = np.nanmax(Qs["Qs"].values) VFracs = np.nanmean(Vs.values, axis=0) return { "NRegions": NRegions, "TsMean": Misc.round_sig(TsMean, 2), "QsMean": Misc.round_sig(QsMean, 2), "TsMax": Misc.round_sig(TsMax, 2), "QsMax": Misc.round_sig(QsMax, 2), "NPCA": NPCA, "VarianceFracs": Misc.round_sig(sum(VFracs[:NPCA]), 2) }
def CalcBOCPDStats(R_Max, tol): ''' Calculate some key statisitics from the data provided, R_Max: Array of maximal propabilty sequences tol: BOCPD tol limit Return, Params: An empty dictionary of key statistics {"NoZeros", "NoOnes", "NoTwos", "MaxSeq", "MeanSeq", "tol"} ''' values, counts = np.unique(R_Max, return_counts=True) if len(counts) > 2: NoZeros = float(counts[0] - 1) NoOnes = float(counts[1]) NoTwos = float(counts[2]) elif len(counts) > 1: NoZeros = float(counts[0] - 1) NoOnes = float(counts[1]) NoTwos = 0.0 elif len(counts) > 0: NoZeros = float(counts[0] - 1) NoOnes = 0.0 NoTwos = 0.0 elif len(counts) == 0: NoZeros = 0.0 NoOnes = 0.0 NoTwos = 0.0 MaxSeq = float(np.nanmax(R_Max)) MeanSeq = np.nanmean(R_Max) return { "NoZeros": Misc.round_sig(NoZeros, 2), "NoOnes": Misc.round_sig(NoOnes, 2), "NoTwos": Misc.round_sig(NoTwos, 2), "MaxSeq": Misc.round_sig(MaxSeq, 2), "MeanSeq": Misc.round_sig(MeanSeq, 2), "tol": tol }
def CalcAlphaStats(Alphas): ''' Calculate some key statisitics from the data provided, Alphas: A dataframe Return, Params: A dictionary of key statistics {"Min", "Max", "Mean", "STD"} ''' Min = np.nanmin(Alphas.values) Max = np.nanmax(Alphas.values) Mean = np.nanmean(Alphas.values) STD = np.nanstd(Alphas.values, ddof=1) return { "Min": Misc.round_sig(Min, 2), "Max": Misc.round_sig(Max, 2), "Mean": Misc.round_sig(Mean, 2), "STD": Misc.round_sig(STD, 2) }
def CalcStats(data): ''' Calculate some key statisitics from the data provided, data: A dataframe Return, Params: A dictionary of key statistics {"Mean", "STD", "Start", "End", "Rate", "RateUnit", "Duration", "Min", "Max"} ''' Data_Mean = np.nanmean(data.values) Data_STD = np.nanstd(data.values, ddof=1) Data_StartDate = data.index[0] Data_EndDate = data.index[-1] if Data_StartDate != Data_EndDate: dT = abs(data.index[1] - data.index[0]) Compare = abs( np.array([ dT / timedelta(seconds=1), dT / timedelta(minutes=1), dT / timedelta(hours=1), dT / timedelta(days=1) ]) - 1) Index = np.argwhere(Compare == min(Compare))[0][0] if Index == 0: Unit = "S" Data_SampleRate = (data.index[1] - data.index[0]) / timedelta(seconds=1) elif Index == 1: Unit = "mins" Data_SampleRate = (data.index[1] - data.index[0]) / timedelta(minutes=1) elif Index == 2: Unit = "hours" Data_SampleRate = (data.index[1] - data.index[0]) / timedelta(hours=1) elif Index == 3: Unit = "days" Data_SampleRate = (data.index[1] - data.index[0]) / timedelta(days=1) else: dT = 0 Data_SampleRate = "N/A" Unit = "" Data_SampleDuration = (data.index[-1] - data.index[0]) / timedelta(days=1) Data_Min = np.nanmin(data.values) Data_Max = np.nanmax(data.values) return { "Mean": Misc.round_sig(Data_Mean, 2), "STD": Misc.round_sig(Data_STD, 2), "Start": Misc.unixTimeMillis(Data_StartDate), "End": Misc.unixTimeMillis(Data_EndDate), "Rate": Misc.round_sig(Data_SampleRate, 2), "RateUnit": Unit, "Duration": Misc.round_sig(Data_SampleDuration, 2), "Min": Misc.round_sig(Data_Min, 2), "Max": Misc.round_sig(Data_Max, 2) }
def CalcFourierStats(Fs, As, PPink, PPinkErrs): ''' Calculate some key statisitics from frequency spectrum fit, Fs: frequencies As: Amplitude of frequency component PPink: Fitted Spectrum [A, Alpha] PPinkErr: Fitted Spectrum Errors [A, Alpha] Return, Params: A dictionary of key statistics {"Alpha", "A0", "AlphaErr", "A0Err", "FreqMin", "FreqMax"} ''' Alpha = PPink[1] A0 = PPink[0] AlphaErr = PPinkErrs[1] A0Err = PPinkErrs[0] FreqMin = np.nanmin(Fs) FreqMax = np.nanmax(Fs) return { "Alpha": Misc.round_sig(Alpha, 2), "A0": Misc.round_sig(A0, 2), "AlphaErr": Misc.round_sig(AlphaErr, 2), "A0Err": Misc.round_sig(A0Err, 2), "FreqMin": Misc.round_sig(FreqMin, 2), "FreqMax": Misc.round_sig(FreqMax, 2) }
def GetRegions(df1, df2, index, dtMeasure, AlphaLim, TestCol=0, NDur=0): ''' Take two series and check for True and False concurance. df1: Panda Series one df2: Panda Series two index: Array of df1, df2 index dtMeasure: A timedelta object of distance between measurments AlphaLim: Required Pval threshold NDur: Minumum number of flag points. Return, Highlighted: A dataframe of regions which contain the True values across both series. dfRates: A dataframe of an estimator of the flag rates over time. ''' RollingWinSize = int( Misc.timedeltaOneUnit(1, Rolling_WindowUnit) * Rolling_Window / dtMeasure) #24*7 RatesUnitFactor = timedelta(days=1) / dtMeasure if TestCol == 0: df1 = df1 < AlphaLim df2 = df2 < AlphaLim df1TF = df1.iloc[:, 0] df2TF = df2.iloc[:, 0] else: df1Min = df1[TestCol].values == df1.iloc[:, 1:].min(axis=1).values df2Min = df2[TestCol].values == df2.iloc[:, 1:].min(axis=1).values df1 = df1 < AlphaLim df2 = df2 < AlphaLim df1TF = df1.iloc[:, 0] & df1[TestCol] df2TF = df2.iloc[:, 0] & df2[TestCol] dfFaults = pd.DataFrame(df1TF & df2TF, index=index) Col = dfFaults.columns[0] df = pd.DataFrame( index=index, data={"value_grp": (dfFaults[Col] != dfFaults[Col].shift(1)).cumsum()}) Starts = [] Stops = [] Durations = [] for i in range(1, df['value_grp'].max() + 1): D0 = df[df['value_grp'] == i].index[0] D1 = df[df['value_grp'] == i].index[-1] #abs((D1 - D0) / dtMeasure) > 0 and if abs( (D1 - D0) / dtMeasure ) > NDur and dfFaults[Col][D0] == True and dfFaults[Col][D1] == True: Starts.append(D0 - dtMeasure / 2) Stops.append(D1 + dtMeasure / 2) Durations.append((D1 - D0) / dtMeasure + 1) Highlighted = pd.DataFrame(data={ "Starts": Starts, "Stops": Stops, "Duration": Durations, }, ) dfFaults[Col] *= 0 for Ri in range(Highlighted.shape[0]): D0 = Highlighted.iloc[Ri]["Starts"] D1 = Highlighted.iloc[Ri]["Stops"] dfFaults[Col][D0:D1] = 1 dfFaults[Col].iloc[:7] = 0 dfRates = pd.DataFrame( data={ "Rate": (dfFaults[Col].rolling(RollingWinSize * 2, min_periods=1).mean().values * (RatesUnitFactor)), }, index=dfFaults.index, ) return Highlighted, dfRates
def CreateRatesFig(Label, Lim, RatesA, RatesS, RegionsA, RegionsS, TestCol, NDur, Height): ''' Example Alpha plots. Label: The y axis label, T or Q. Lim: The Criteria Limit, RatesA: Averaged Rolling rates, RatesS: Specific Rolling rates, RegionsA : Dataframe of Regions where alpha_crit violated overall RegionsS : Dataframe of Regions where requested feature is violated TestCol : The marginalised feature to examine name. String Height: Height in pixels required. (Width autoscales), Return, Fig: A figure html element.''' YRange = [0, np.max(RatesA.values) * 1.2] XRange = [RatesA.index[0], RatesA.index[-1]] Cols = RatesA.columns Fig = go.Figure( layout={ "xaxis_title": "Time, t", "yaxis_title": "Rolling Mean Rate / day", "autosize": True, "margin": { "l": 10, "r": 10, "t": 10, "b": 10 }, "height": Height, #"xaxis_type":'log', "xaxis_range": XRange, #"yaxis_type":'log', "yaxis_range": YRange, "plot_bgcolor": 'white', }) Fig.add_trace( go.Scatter( x=RatesA.index, y=RatesA[Label], name="", hoverinfo="name+x+y", #customdata=np.log10(Rates[Label]), hovertemplate="<br>N: %s" % (Cols[0]) + "<br>t: %{x}<br> y: %{y:.2f}", line={ "width": 1, "color": "black" }, marker={"size": 5}, mode="lines", showlegend=False, )) #Highlight flagged regions for Ri in range(RegionsA.shape[0]): if RegionsA.iloc[Ri]["Duration"] <= NDur: continue Fig.add_shape( fillcolor="orange", opacity=0.4, line={"width": 0}, type="rect", x0=Misc.unixToDatetime(RegionsA.iloc[Ri]["Starts"]), x1=Misc.unixToDatetime(RegionsA.iloc[Ri]["Stops"]), xref="x", y0=YRange[0], y1=YRange[1], yref="y", layer='above', ), if isinstance(RegionsS, (int)): pass else: for Ri in range(RegionsS.shape[0]): if RegionsS.iloc[Ri]["Duration"] <= NDur: continue Fig.add_shape( fillcolor="red", opacity=0.4, line={"width": 0}, type="rect", x0=Misc.unixToDatetime(RegionsS.iloc[Ri]["Starts"]), x1=Misc.unixToDatetime(RegionsS.iloc[Ri]["Stops"]), xref="x", y0=YRange[0], y1=YRange[1], yref="y", layer='above', ), Fig.add_trace( go.Scatter( x=RatesS.index, y=RatesS[Label], name="", hoverinfo="name+x+y", #customdata=np.log10(Rates[Label]), hovertemplate="<br>N: %s" % (TestCol) + "<br>t: %{x}<br> y: %{y:.2f}", line={ "width": 1, "color": "purple" }, marker={"size": 5}, mode="lines", showlegend=False, )) Fig.add_shape(type="line", x0=XRange[0], y0=Lim, x1=XRange[1], y1=Lim, line={ "width": 1, "color": "red", "dash": "dash", }) #Bounding box for double-click reset Fig.add_shape(type="line", x0=XRange[0], y0=YRange[0], x1=XRange[1], y1=YRange[0], line={ "width": 0, "color": "purple", "dash": "dash", }) Fig.add_shape(type="line", x0=XRange[1], y0=YRange[0], x1=XRange[1], y1=YRange[1], line={ "width": 0, }) #For range fixing only. (Invisable line) Fig.add_shape(type="line", x0=XRange[0], y0=YRange[0], x1=XRange[0], y1=YRange[1], line={ "width": 0, }) Fig.add_shape(type="line", x0=XRange[1], y0=YRange[0], x1=XRange[1], y1=YRange[1], line={ "width": 0, }) return Fig
def CreatePCAFig(Label, AlphaLim, Stats, RegionsA, RegionsS, Height): ''' Example Alpha plots. Label: The y axis label, T or Q. AlphaLim: The Criteria Limit Stats: The Pvalues of the T or Q stat RegionsA : Dataframe of Regions where alpha_crit violated overall RegionsS : Dataframe of Regions where requested feature is violated Height: Height in pixels required. (Width autoscales), Return, Fig: A figure html element.''' YRange = [max(1e-16, Stats[Label].min()), 2] XRange = [Stats[Label].index[0], Stats.index[-1]] Stats[Stats < YRange[0]] = YRange[0] Cols = Stats.columns Fig = go.Figure( layout={ "xaxis_title": "Time, t", "yaxis_title": Label, "autosize": True, "margin": { "l": 10, "r": 10, "t": 10, "b": 10 }, "height": Height, #"xaxis_type":'log', "xaxis_range": XRange, "yaxis_type": 'log', "yaxis_range": np.log10(YRange), "plot_bgcolor": 'white', }) #Error highlights Fig.add_trace( go.Scatter( x=Stats.index, y=Stats[Label], name="", hoverinfo="name+x+y", customdata=np.log10(Stats[Label]), hovertemplate="<br>N: %s" % (Cols[0]) + "<br>t: %{x}<br> y: %{customdata:.2f}", line={ "width": 1, "color": "purple" }, marker={"size": 5}, mode="lines", showlegend=False, )) #Highlight flagged regions for PCAi in range(1, len(Cols)): Fig.add_trace( go.Scatter( x=Stats.index, y=Stats.iloc[:, PCAi], name="", hoverinfo="name+x+y", customdata=np.log10(Stats.iloc[:, PCAi]), hovertemplate="<br>N: %s" % (Cols[PCAi]) + "<br>t: %{x}<br> y: %{customdata:.2f}", line={ "width": 1, "color": "black" }, marker={"size": 5}, mode="lines", showlegend=False, )) #Highlight flagged regions for Ri in range(RegionsA.shape[0]): Fig.add_shape( fillcolor="orange", opacity=0.4, line={"width": 0}, type="rect", x0=Misc.unixToDatetime(RegionsA.iloc[Ri]["Starts"]), x1=Misc.unixToDatetime(RegionsA.iloc[Ri]["Stops"]), xref="x", y0=YRange[0], y1=YRange[1], yref="y", layer='above', ), for Ri in range(RegionsS.shape[0]): Fig.add_shape( fillcolor="red", opacity=0.4, line={"width": 0}, type="rect", x0=Misc.unixToDatetime(RegionsS.iloc[Ri]["Starts"]), x1=Misc.unixToDatetime(RegionsS.iloc[Ri]["Stops"]), xref="x", y0=YRange[0], y1=YRange[1], yref="y", layer='above', ), Fig.add_shape(type="line", x0=XRange[0], y0=AlphaLim, x1=XRange[1], y1=AlphaLim, line={ "width": 1, "color": "red", "dash": "dash", }) #Bounding box for double-click reset Fig.add_shape(type="line", x0=XRange[0], y0=YRange[0], x1=XRange[1], y1=YRange[0], line={ "width": 0, "color": "purple", "dash": "dash", }) Fig.add_shape(type="line", x0=XRange[1], y0=YRange[0], x1=XRange[1], y1=YRange[1], line={ "width": 0, }) #For range fixing only. (Invisable line) Fig.add_shape(type="line", x0=XRange[0], y0=YRange[0], x1=XRange[0], y1=YRange[1], line={ "width": 0, }) Fig.add_shape(type="line", x0=XRange[1], y0=YRange[0], x1=XRange[1], y1=YRange[1], line={ "width": 0, }) return Fig
def CreatePCADataFig(Data, RegionsA, RegionsS, TestCol, NDur, Height): ''' Example Alpha plots. Data: Dataframe of T or Q statistics overall for all features then marginalised for each. RegionsA : Dataframe of Regions where alpha_crit violated overall RegionsS : Dataframe of Regions where requested feature is violated Height: Height in pixels required. (Width autoscales), Return, Fig: A figure html element.''' YRange = [-7.5, 7.5] XRange = [Data.index[0], Data.index[-1]] Cols = Data.columns Fig = go.Figure( layout={ "xaxis_title": "Time, t", "yaxis_title": "Feature", "autosize": True, "margin": { "l": 10, "r": 10, "t": 10, "b": 10 }, "height": Height, #"xaxis_type":'log', "xaxis_range": XRange, #"yaxis_type":'log', "yaxis_range": YRange, "plot_bgcolor": 'white', }) #Highlight flagged regions for PCAi in range(len(Cols)): Data.iloc[:, PCAi][Data.iloc[:, PCAi] > 7.5] = 7.5 Data.iloc[:, PCAi][Data.iloc[:, PCAi] < -7.5] = -7.5 if TestCol == None: for PCAi in range(len(Cols)): Fig.add_trace( go.Scatter( x=Data.index, y=Data.iloc[:, PCAi], name="", hoverinfo="name+x+y", hovertemplate="<br>N: %s" % (Cols[PCAi]) + "<br>t: %{x}<br> y: %{y:.2f}", line={ "width": 1, "color": "black" }, marker={"size": 5}, mode="lines", showlegend=False, )) else: Fig.add_trace( go.Scatter( x=Data.index, y=Data[TestCol], name="", hoverinfo="name+x+y", hovertemplate="<br>N: %s" % (TestCol) + "<br>t: %{x}<br> y: %{y:.2f}", line={ "width": 1, "color": "purple" }, marker={"size": 5}, mode="lines", showlegend=False, )) for Ri in range(RegionsA.shape[0]): if RegionsA.iloc[Ri]["Duration"] <= NDur: continue Fig.add_shape( fillcolor="orange", opacity=0.4, line={"width": 0}, type="rect", x0=Misc.unixToDatetime(RegionsA.iloc[Ri]["Starts"]), x1=Misc.unixToDatetime(RegionsA.iloc[Ri]["Stops"]), xref="x", y0=YRange[0], y1=YRange[1], yref="y", layer='above', ), if isinstance(RegionsS, (int)): pass else: for Ri in range(RegionsS.shape[0]): if RegionsS.iloc[Ri]["Duration"] <= NDur: continue Fig.add_shape( fillcolor="red", opacity=0.4, line={"width": 0}, type="rect", x0=Misc.unixToDatetime(RegionsS.iloc[Ri]["Starts"]), x1=Misc.unixToDatetime(RegionsS.iloc[Ri]["Stops"]), xref="x", y0=YRange[0], y1=YRange[1], yref="y", layer='above', ), #Bounding box for double-click reset Fig.add_shape(type="line", x0=XRange[0], y0=YRange[0], x1=XRange[1], y1=YRange[0], line={ "width": 0, "color": "purple", "dash": "dash", }) Fig.add_shape(type="line", x0=XRange[1], y0=YRange[0], x1=XRange[1], y1=YRange[1], line={ "width": 0, }) #For range fixing only. (Invisable line) Fig.add_shape(type="line", x0=XRange[0], y0=YRange[0], x1=XRange[0], y1=YRange[1], line={ "width": 0, }) Fig.add_shape(type="line", x0=XRange[1], y0=YRange[0], x1=XRange[1], y1=YRange[1], line={ "width": 0, }) return Fig
def CreateBOCPDFig(BOCPDdf, R_Max, Lines, WindowParameters, tol, Height): ''' Example Alpha plots. BOCPDdf:df "Alpha" and "Errs" values at each timestep R_Max: The Most likely sequence length at time t Lines: timestamps where R_Max is a 0 length sequence WindowParameters: A dictionary of key statistics {"Mean", "STD", "Start", "End", "Rate", "RateUnit", "Duration", "Min", "Max"} Sig: Error multiplier, tol: The P value truncation size Height: Height in pixels required. (Width autoscales) Return, Fig: A figure html element.''' Zmin = np.log10(tol) Zs = np.nan_to_num(np.log10(BOCPDdf.values.T), nan=-np.inf, posinf=np.inf, neginf=-np.inf) YRange = [0, np.max(R_Max) + 20] XRange = [ Misc.unixToDatetime(WindowParameters["Start"]), Misc.unixToDatetime(WindowParameters["End"]) ] Fig = go.Figure( layout={ "xaxis_title": "Time, t", "yaxis_title": "Seq length, l", "autosize": True, "margin": { "l": 10, "r": 10, "t": 10, "b": 10 }, "height": Height, "xaxis_range": XRange, "yaxis_range": YRange, "plot_bgcolor": 'white', }) Fig.add_trace( go.Heatmap(x=BOCPDdf.index, y=BOCPDdf.columns, z=Zs, customdata=10**Zs, name="", hoverinfo="x+y+z+text", hovertemplate="<br>t: %{x}<br>" + "y: %{y:.2f}<br>" + "z: %{customdata:.5f}", zmax=0, zmin=Zmin, showscale=False, colorscale='purples')) #Error highlights Fig.add_trace( go.Scatter( x=BOCPDdf.index, y=R_Max, name="", hoverinfo="x+y+text", hovertemplate="<br>t: %{x}<br>" + "y: %{y:.2f}", line={ "width": 2, "color": "black" }, marker={"size": 5}, mode="lines", showlegend=False, )) for X_i in Lines: Fig.add_shape(type="line", x0=X_i, y0=YRange[0], x1=X_i, y1=YRange[1], line={ "width": 1, "color": "black", "dash": "dash", }) #Bounding box for double-click reset Fig.add_shape(type="line", x0=XRange[0], y0=YRange[0], x1=XRange[1], y1=YRange[0], line={ "width": 1, "color": "purple", "dash": "dash", }) Fig.add_shape(type="line", x0=XRange[1], y0=YRange[0], x1=XRange[1], y1=YRange[1], line={ "width": 0, }) #For range fixing only. (Invisable line) Fig.add_shape(type="line", x0=XRange[0], y0=YRange[0], x1=XRange[0], y1=YRange[1], line={ "width": 0, }) Fig.add_shape(type="line", x0=XRange[1], y0=YRange[0], x1=XRange[1], y1=YRange[1], line={ "width": 0, }) return Fig
def CreateRawFig(data, sig, Data_i_Name, WindowParameters, Height): ''' Raw figure plots for tabs 1 and 2. data: A dataframe of data with columns [feature, error], sig: Error scaling factor, Data_i_Name: The Column feature name WindowParameters: A dictionary of key statistics {"Mean", "STD", "Start", "End", "Rate", "RateUnit", "Duration", "Min", "Max"} Height: Height in pixels required. (Width autoscales) Return, Fig: A figure html element.''' YRange = [ np.min(data[Data_i_Name] - data["Error"].mean() * 3), np.max(data[Data_i_Name] + data["Error"].mean() * 3) ] XRange = [data.index[0], data.index[-1]] if sig == None: sig = 0 Fig = go.Figure( layout={ "xaxis_title": "Time, t", "yaxis_title": "Amplitude, A", "autosize": True, "margin": { "l": 10, "r": 10, "t": 10, "b": 10 }, "height": Height, "yaxis_range": YRange, }) # Add traces Fig.add_trace( go.Scatter( x=data.index, y=data[Data_i_Name], name="", hoverinfo="x+y+text", hovertemplate="<br>t: %{x}<br>" + "y: %{y:.2f}", line={ "width": 1, "color": "black" }, marker={"size": 5}, mode="lines", showlegend=False, )) #Error highlights Fig.add_trace( go.Scatter( x=data.index, y=data[Data_i_Name] + data["Error"] * sig, name="", hoverinfo="x+y+text", hovertemplate="<br>t: %{x}<br>" + "y: %{y:.2f}", line={ "width": 0, "color": "red" }, marker={"size": 5}, mode="lines", showlegend=False, )) Fig.add_trace( go.Scatter( x=data.index, y=data[Data_i_Name] - data["Error"] * sig, name="", hoverinfo="x+y+text", hovertemplate="<br>t: %{x}<br>" + "y: %{y:.2f}", line={ "width": 0, "color": "red" }, marker={"size": 5}, mode="lines", showlegend=False, fill='tonexty', )) Fig.add_shape( fillcolor="#68246D", opacity=0.2, line={"width": 0}, type="rect", x0=Misc.unixToDatetime(WindowParameters["Start"]), x1=Misc.unixToDatetime(WindowParameters["End"]), xref="x", #y0=WindowParameters["Min"], #y1=WindowParameters["Max"], y0=YRange[0], y1=YRange[1], yref="y", layer='below', ), return Fig