def windowed_gaussian(ts_obj, gaussian_window_size, step_size, plot_anomaly_score=False): if ts_obj.miss: raise ValueError("Missing time steps. Cannot use Windowed Gaussian.") start = time.time() anomaly_scores = ah.determine_anomaly_scores_error(ts_obj.dataframe["value"].values, [0] * ts_obj.get_length(), ts_obj.get_length(), gaussian_window_size, step_size=step_size) # why is the below here? # anomaly_scores = np.nan_to_num(anomaly_scores) end = time.time() if plot_anomaly_score: plt.subplot(211) plt.title("Anomaly Scores") plt.plot(anomaly_scores) plt.ylim([.99,1]) plt.subplot(212) plt.title("Time Series") plt.plot(ts_obj.dataframe["value"].values) plt.axvline(ts_obj.get_probationary_index(), color="black", label="probationary line") plt.tight_layout() plt.show() return {"Anomaly Scores": anomaly_scores, "Time": end - start}
def matrixprofile(ts_obj, subseq_len, gaussian_window_size, step_size, plot_matrixprofile=False, plot_anomaly_score=False): start = time.time() # see line 53 # https://github.com/target/matrixprofile-ts/blob/master/matrixprofile/matrixProfile.py if ts_obj.miss: ref_date_range = ch.get_ref_date_range(ts_obj.dataframe, ts_obj.dateformat, ts_obj.timestep) gaps = ref_date_range[~ref_date_range.isin(ts_obj. dataframe["timestamp"])] filled_df = ch.fill_df(ts_obj.dataframe, ts_obj.timestep, ref_date_range, "fill_nan") # print("NaNs exist?: ",filled_df['value'].isnull().values.any()) matrix_profile = matrixProfile.stamp(filled_df["value"].values, subseq_len) else: matrix_profile = matrixProfile.stamp(ts_obj.dataframe["value"].values, subseq_len) # Append np.nan to Matrix profile to enable plotting against raw data matrix_profile = np.append(matrix_profile[0], np.zeros(subseq_len - 1) + np.nan) anomaly_scores = ah.determine_anomaly_scores_error( matrix_profile, np.zeros_like(matrix_profile), ts_obj.get_length(), gaussian_window_size, step_size) end = time.time() if plot_matrixprofile: plt.subplot(211) plt.title("Matrix Profile") plt.plot(matrix_profile) plt.subplot(212) plt.title("Time Series") plt.plot(ts_obj.dataframe["value"].values) plt.axvline(ts_obj.get_probationary_index(), color="black", label="probationary line") plt.tight_layout() plt.show() if plot_anomaly_score: plt.subplot(211) plt.title("Anomaly Scores") plt.plot(anomaly_scores) plt.ylim([.99, 1]) plt.subplot(212) plt.title("Time Series") plt.plot(ts_obj.dataframe["value"].values) plt.axvline(ts_obj.get_probationary_index(), color="black", label="probationary line") plt.tight_layout() plt.show() return { 'Anomaly Scores': anomaly_scores, 'Time': end - start, "Matrix Profile": matrix_profile }
name = ts.name result_128 = joblib.load("stl_scores_" + str(score_number) + "_gaussian_window_128_" + name + "_swindow_" + str(swindow) + "_sdegree_" + str(sdegree) + "_twindow_" + str(twindow) + "_tdegree_" + str(tdegree) + "_inner_" + str(inner) + "_outer_" + str(outer)) stl_remainder = list(result_128["STL Remainder"]) actual = list(ts.dataframe["value"].values) step_size = int(gaussian_window_size / 2) anomaly_scores = ah.determine_anomaly_scores_error( stl_remainder, [0] * ts.get_length(), ts.get_length(), gaussian_window_size, step_size) result_dict = { "Anomaly Scores": anomaly_scores, "Time": result_128["Time"], "STL Remainder": stl_remainder } joblib.dump( result_dict, "stl_scores_" + str(score_number) + "_gaussian_window_" + str(gaussian_window_size) + "_" + name + "_swindow_" + str(swindow) + "_sdegree_" + str(sdegree) + "_twindow_" + str(twindow) + "_tdegree_" + str(tdegree) + "_inner_" + str(inner) + "_outer_" + str(outer))
mcmc_iteration = 10 latent_dim = 5 name = ts.name result_128 = joblib.load("vae_scores_" + str(score_number) + "_gaussian_window_128_" + name + "_window_size_" + str(window_size) + "_mcmc_iteration_" + str(mcmc_iteration) + "_latent_dim_" + str(latent_dim)) reconstruction_probabilities = list( result_128["Reconstruction Probabilities"]) step_size = int(gaussian_window_size / 2) anomaly_scores = ah.determine_anomaly_scores_error( reconstruction_probabilities, np.zeros_like(reconstruction_probabilities), ts.get_length(), gaussian_window_size, step_size) result_dict = { "Anomaly Scores": anomaly_scores, "Time": result_128["Time"], "Reconstruction Probabilities": reconstruction_probabilities } joblib.dump( result_dict, "vae_scores_" + str(score_number) + "_gaussian_window_" + str(gaussian_window_size) + "_" + name + "_window_size_" + str(window_size) + "_mcmc_iteration_" + str(mcmc_iteration) + "_latent_dim_" + str(latent_dim))
for f in listdir(mypath): for score_number in range(0,num_scores): for gaussian_window_size in gaussian_window_sizes: if "ts_object" in f: ts = joblib.load(mypath + f) if ts.seasonality: subseq_len = ts.period else: subseq_len = 100 if subseq_len < 5: subseq_len = 100 name = ts.name result_128 = joblib.load("matrix_profile_scores_" + str(score_number) + "_gaussian_window_128_" + name + "_subseq_len_" + str(subseq_len)) matrix_profile = list(result_128["Matrix Profile"]) step_size = int(gaussian_window_size/2) anomaly_scores = ah.determine_anomaly_scores_error(matrix_profile, np.zeros_like(matrix_profile), ts.get_length(), gaussian_window_size, step_size) result_dict = {"Anomaly Scores": anomaly_scores, "Time": result_128["Time"], "Matrix Profile": matrix_profile} joblib.dump(result_dict, "matrix_profile_scores_" + str(score_number) + "_gaussian_window_" + str(gaussian_window_size) + "_" + name + "_subseq_len_" + str(subseq_len))
ts = joblib.load(mypath + f) if ts.name == all_datasets[dataset_index]: name = ts.name result_128 = joblib.load("sarimax_scores_" + str(score_number) + "_gaussian_window_128_" + name + "_step_size_" + str(64)) forecast = list(result_128["Forecast"]) actual = list(ts.dataframe["value"].values) step_size = int(gaussian_window_size / 2) anomaly_scores = ah.determine_anomaly_scores_error( actual, forecast, len(forecast), gaussian_window_size, step_size) # plt.plot(anomaly_scores) # plt.show() result_dict = { "Anomaly Scores": anomaly_scores, "Time": result_128["Time"], "Predictions": forecast } joblib.dump( result_dict, "sarimax_scores_" + str(score_number) + "_gaussian_window_" + str(gaussian_window_size) + "_" + name + "_step_size_" + str(step_size))
mypath = "../jair_work_step_one_determine_characteristics/" for f in listdir(mypath): for score_number in range(0,num_scores): for gaussian_window_size in gaussian_window_sizes: if "ts_object" in f: ts = joblib.load(mypath + f) best_df = glim_grid_search_df.loc[[glim_grid_search_df.loc[glim_grid_search_df["TS Name"] == ts.name, 'RMSE'].idxmin()]] lambda_ = best_df["Lambda"].values[0] eta = best_df["Eta"].values[0] family = best_df["Family"].values[0] name = ts.name result_128 = joblib.load("glim_scores_" + str(score_number) + "_gaussian_window_128_" + name + "_lambda_" + str(lambda_) + "_eta_" + str(eta) + "_family_" + family) predictions = list(result_128["Predictions"]) actual = list(ts.dataframe["value"].values) step_size = int(gaussian_window_size/2) anomaly_scores = ah.determine_anomaly_scores_error(actual, predictions, ts.get_length(), gaussian_window_size, step_size) result_dict = {"Anomaly Scores": anomaly_scores, "Time": result_128["Time"], "Predictions": predictions} joblib.dump(result_dict, "glim_scores_" + str(score_number) + "_gaussian_window_" + str(gaussian_window_size) + "_" + name + "_lambda_" + str(lambda_) + "_eta_" + str(eta) + "_family_" + family)
def stl(ts_obj, gaussian_window_size, step_size, swindow, sdegree, twindow, tdegree, inner, outer, grid_search_mode=False, plot_components=False, plot_anomaly_score=False): # this method can deal with missing time steps if ts_obj.get_period() < 4: raise ValueError("n_periods must be at least 4.") start = time.time() if ts_obj.miss: ref_date_range = ch.get_ref_date_range(ts_obj.dataframe, ts_obj.get_dateformat(), ts_obj.get_timestep()) nan_data = ch.fill_df(ts_obj.dataframe, ts_obj.get_timestep, ref_date_range, method="fill_nan") ts_values = nan_data["value"].values ts_timestamps = list(nan_data["timestamp"].values) ts_timestamps = np.array([str(item) for item in ts_timestamps]) result = stlplus(ts_values, ts_timestamps, ts_obj.get_period(), swindow, sdegree, twindow, tdegree, inner, outer) stl_remainder = list(list(result)[0]["remainder"]) nan_data["stl remainder"] = stl_remainder nan_data = nan_data.dropna() stl_remainder = nan_data["stl remainder"].values else: ts_values = ts_obj.dataframe["value"].values ts_timestamps = list(ts_obj.dataframe["timestamp"].values) ts_timestamps = np.array([str(item) for item in ts_timestamps]) result = stlplus(ts_values, ts_timestamps, ts_obj.get_period(), swindow, sdegree, twindow, tdegree, inner, outer) stl_remainder = list(result)[0]["remainder"] if list(stl_remainder).count(0) >= int(.9 * ts_obj.get_length()): raise ValueError("Remainders are mostly zero") if grid_search_mode: if plot_components: # print(list(result)[0]["remainder"].values) plt.subplot(311) plt.title("Seasonality") plt.plot(list(result)[0]["seasonal"].values) plt.subplot(312) plt.title("Trend") plt.plot(list(result)[0]["trend"].values) plt.subplot(313) plt.title("remainder") plt.plot(list(result)[0]["remainder"].values) plt.show() the_sum = 0 for remainder in stl_remainder: the_sum += abs(remainder) print("Sum of STL Remainders: ", the_sum) return the_sum anomaly_scores = ah.determine_anomaly_scores_error( stl_remainder, [0] * ts_obj.get_length(), ts_obj.get_length(), gaussian_window_size, step_size) end = time.time() if plot_components: # print(list(result)[0]["remainder"].values) plt.subplot(311) plt.title("Seasonality") plt.plot(list(result)[0]["seasonal"].values) plt.subplot(312) plt.title("Trend") plt.plot(list(result)[0]["trend"].values) plt.subplot(313) plt.title("remainder") plt.plot(list(result)[0]["remainder"].values) plt.tight_layout() plt.show() if plot_anomaly_score: plt.subplot(211) plt.title("Anomaly Scores") plt.plot(anomaly_scores) plt.ylim([.99, 1]) plt.subplot(212) plt.title("Time Series") plt.plot(ts_obj.dataframe["value"].values) plt.axvline(ts_obj.get_probationary_index(), color="black", label="probationary line") plt.tight_layout() plt.show() return { "Anomaly Scores": anomaly_scores, "Time": end - start, "STL Remainder": stl_remainder }
def fbprophet(ts_obj, gaussian_window_size, step_size, changepoint_prior_scale=.05, growth='linear', yearly_seasonality='auto', weekly_seasonality='auto', daily_seasonality='auto', holidays=None, seasonality_mode='additive', seasonality_prior_scale=10, holidays_prior_scale=10, plot_anomaly_score=False, plot_forecast=False, grid_search_mode=False): start = time.time() fb_prophet_model = Prophet(changepoint_prior_scale=changepoint_prior_scale, growth=growth, yearly_seasonality=yearly_seasonality, weekly_seasonality=weekly_seasonality, daily_seasonality=daily_seasonality, holidays=holidays, seasonality_mode=seasonality_mode, seasonality_prior_scale=seasonality_prior_scale, holidays_prior_scale=holidays_prior_scale) if ts_obj.miss: # https://facebook.github.io/prophet/docs/outliers.html # Prophet has no problem with missing data # You set the missing values to NaNs in the training data # But you LEAVE the dates in the prediction ref_date_range = ch.get_ref_date_range(ts_obj.dataframe, ts_obj.dateformat, ts_obj.timestep) data_copy = copy.deepcopy(ts_obj.dataframe) data_copy["timestamp"] = pd.to_datetime(data_copy["timestamp"], format=ts_obj.dateformat) data_copy.set_index('timestamp', inplace=True) data_copy = data_copy.reindex(ref_date_range, fill_value=np.nan) # use entire time series for training counts = [i for i in range(len(data_copy))] fb_df_train = pd.DataFrame({ "count": counts, "ds": ref_date_range, "y": data_copy["value"] }) else: # use entire time series for training fb_df_train = pd.DataFrame({ "ds": ts_obj.dataframe["timestamp"], "y": ts_obj.dataframe["value"] }) fb_prophet_model.fit(fb_df_train, verbose=False) # periods=how much further you want to extend from the training dataset # this is not periodicity relating to seasonality future = fb_prophet_model.make_future_dataframe(periods=0, freq=ts_obj.timestep) # make a forecast over the entire time series fcst = fb_prophet_model.predict(future) predictions = fcst["yhat"].values # get RMSE if grid_search_mode: if ts_obj.miss: # remove the predictions from missing time steps inds = fb_df_train.loc[ pd.isna(fb_df_train["y"]), :]["count"].values print(inds) nonmissing_predictions = [] for i in range(len(predictions)): if i not in inds: nonmissing_predictions.append(predictions[i]) rmse = mean_squared_error(ts_obj.dataframe["value"].values, nonmissing_predictions, squared=False) print("RMSE: ", rmse) else: rmse = mean_squared_error(ts_obj.dataframe["value"].values, predictions, squared=False) print("RMSE: ", rmse) return rmse # get anomaly scores else: if ts_obj.miss: # you HAVE to interpolate to get a gaussian window new_ts_obj = copy.deepcopy(ts_obj) new_ts_obj.set_miss(fill=True) actual = list(new_ts_obj.dataframe["value"]) else: actual = ts_obj.dataframe["value"] anomaly_scores = ah.determine_anomaly_scores_error( actual, predictions, ts_obj.get_length(), gaussian_window_size, step_size) end = time.time() if plot_forecast: plt.plot([i for i in range(len(fcst))], fcst["yhat"]) plt.fill_between([i for i in range(len(fcst))], fcst["yhat_lower"], fcst["yhat_upper"], facecolor='blue', alpha=.3) if ts_obj.miss: plt.plot([i for i in range(len(predictions))], data_copy["value"], alpha=.5) else: plt.plot([i for i in range(len(predictions))], ts_obj.dataframe["value"], alpha=.5) plt.xticks(rotation=90) plt.show() if plot_anomaly_score: plt.subplot(211) plt.title("Anomaly Scores") plt.plot(anomaly_scores) plt.ylim([.99, 1]) plt.subplot(212) plt.title("Time Series") plt.plot(ts_obj.dataframe["value"].values) plt.axvline(ts_obj.get_probationary_index(), color="black", label="probationary line") plt.tight_layout() plt.show() return { "Anomaly Scores": anomaly_scores, "Time": end - start, "Predictions": predictions }
def vae_donut(ts_obj, window_size, mcmc_iteration, latent_dim, gaussian_window_size, step_size, plot_reconstruction=False, plot_anomaly_score=False): # authors use window_size = 120 # mcmc_iteration = 10 # https://github.com/kratzert/finetune_alexnet_with_tensorflow/issues/8 tf.reset_default_graph() start = time.time() # if there are missing time steps, we DO NOT fill them with NaNs because donut will replace them with 0s # using complete_timestamp # see line 6 in https://github.com/NetManAIOps/donut/blob/master/donut/preprocessing.py timestamp, values, labels = ts_obj.dataframe[ "timestamp"].values, ts_obj.dataframe["value"].values, np.zeros_like( ts_obj.dataframe["value"].values, dtype=np.int32) # print(len(timestamp)) # print(len(values)) # print(len(labels)) # Complete the timestamp, and obtain the missing point indicators # replaces missing with 0s. # donut cannot handle this date format for some reason if ts_obj.dateformat == "%Y-%m": rng = pd.date_range('2000-01-01', periods=len(values), freq='T') timestamp, missing, (values, labels) = complete_timestamp( rng, (values, labels)) else: timestamp, missing, (values, labels) = complete_timestamp( timestamp, (values, labels)) # print(len(timestamp)) # print(len(values)) # print(len(labels)) # print(sum(missing)) # Standardize the training and testing data. values, mean, std = standardize_kpi(values, excludes=np.logical_or( labels, missing)) with tf.variable_scope('model') as model_vs: model = Donut( h_for_p_x=Sequential([ K.layers.Dense(100, kernel_regularizer=K.regularizers.l2(0.001), activation=tf.nn.relu), K.layers.Dense(100, kernel_regularizer=K.regularizers.l2(0.001), activation=tf.nn.relu), ]), h_for_q_z=Sequential([ K.layers.Dense(100, kernel_regularizer=K.regularizers.l2(0.001), activation=tf.nn.relu), K.layers.Dense(100, kernel_regularizer=K.regularizers.l2(0.001), activation=tf.nn.relu), ]), x_dims=window_size, z_dims=latent_dim, ) trainer = DonutTrainer(model=model, model_vs=model_vs) predictor = DonutPredictor(model) with tf.Session().as_default(): trainer.fit(values, labels, missing, mean, std) score = predictor.get_score(values, missing) # if time series is [1,2,3,4...] and ts_length is 3 # this gives us [[1,2,3],[2,3,4]...] ts_strided = ah.as_sliding_window(values, window_size) ts_strided = my_func_float(np.array(ts_strided, dtype=np.float32)) missing_strided = ah.as_sliding_window(missing, window_size) missing_strided = my_func_int( np.array(missing_strided, dtype=np.int32)) # print(ts_strided) # print(missing_strided) x = model.vae.reconstruct( iterative_masked_reconstruct(reconstruct=model.vae.reconstruct, x=ts_strided, mask=missing_strided, iter_count=mcmc_iteration, back_prop=False)) # `x` is a :class:`tfsnippet.stochastic.StochasticTensor`, from which # you may derive many useful outputs, for example: # print(x.tensor.eval()) # the `x` samples # print(x.log_prob(group_ndims=0).eval()) # element-wise log p(x|z) of sampled x # print(x.distribution.log_prob(ts_strided).eval()) # the reconstruction probability # print(x.distribution.mean.eval(), x.distribution.std.eval()) # mean and std of p(x|z) tensor_reconstruction_probabilities = x.distribution.log_prob( ts_strided).eval() # because of the way strided works, we use the first 120 anomaly scores in the first slide # and then for remaining slides, we use the last point/score reconstruction_probabilities = list( tensor_reconstruction_probabilities[0]) for i in range(len(tensor_reconstruction_probabilities)): if i != 0: slide = tensor_reconstruction_probabilities[i] reconstruction_probabilities.append(slide[-1]) # print(len(reconstruction_probabilities)) # print(len(ts_obj.dataframe)) if ts_obj.miss: ref_date_range = ch.get_ref_date_range(ts_obj.dataframe, ts_obj.dateformat, ts_obj.timestep) gaps = ref_date_range[~ref_date_range.isin(ts_obj. dataframe["timestamp"])] filled_df = ch.fill_df(ts_obj.dataframe, ts_obj.timestep, ref_date_range, "fill_nan") # print("NaNs exist?: ",filled_df['value'].isnull().values.any()) filled_df[ "reconstruction_probabilities"] = reconstruction_probabilities # remove nans filled_df = filled_df.dropna() reconstruction_probabilities = list( filled_df["reconstruction_probabilities"].values) # print(len(reconstruction_probabilities)) # print(len(ts_obj.dataframe)) reconstruction_probabilities = [ abs(item) for item in reconstruction_probabilities ] anomaly_scores = anomaly_scores = ah.determine_anomaly_scores_error( reconstruction_probabilities, np.zeros_like(reconstruction_probabilities), ts_obj.get_length(), gaussian_window_size, step_size) end = time.time() if plot_reconstruction: plt.subplot(211) # see lines 98 to 100 of https://github.com/NetManAIOps/donut/blob/master/donut/prediction.py plt.title("Negative of Reconstruction Probabilities") plt.plot(reconstruction_probabilities) # plt.ylim([.99,1]) plt.subplot(212) plt.title("Time Series") plt.plot(ts_obj.dataframe["value"].values) plt.axvline(ts_obj.get_probationary_index(), color="black", label="probationary line") plt.tight_layout() plt.show() if plot_anomaly_score: plt.subplot(211) plt.title("Anomaly Scores") plt.plot(anomaly_scores) plt.ylim([.998, 1]) plt.subplot(212) plt.title("Time Series") plt.plot(ts_obj.dataframe["value"].values) plt.axvline(ts_obj.get_probationary_index(), color="black", label="probationary line") plt.tight_layout() plt.show() return { "Anomaly Scores": anomaly_scores, "Time": end - start, "Reconstruction Probabilities": reconstruction_probabilities }
def sarimax_mini(ts_obj, gaussian_window_size, step_size, plot_anomaly_score=False, plot_forecast=False): start = time.time() # SARIMAX SHOULD BE ABLE TO HANDLE MISSING VALUES theoretically and code wise: # --theoretically: https://stats.stackexchange.com/questions/346225/fitting-arima-to-time-series-with-missing-values # "ARIMA models are state space models and the Kalman filter, # which is used to fit state space models, deals with missing values exactly # by simply skipping the update phase." # --code wise: https://www.statsmodels.org/devel/examples/notebooks/generated/statespace_sarimax_internet.html # "The novel feature is the ability of the model to work on datasets with missing values." # Unfortunately, python pyramid autoarima cannot handle missing values # I tested this using ''' import pmdarima as pm from pmdarima import model_selection import numpy as np data = pm.datasets.load_wineind() data[50] = np.nan data[160] = np.nan train, test = model_selection.train_test_split(data, train_size=150) arima = pm.auto_arima(train, error_action='ignore', trace=True, suppress_warnings=True, maxiter=10,seasonal=True, m=12) ''' # which resulted in ValueError: Input contains NaN, infinity or a value too large for dtype('float64'). # R's autoarima can handle missing values BUT # you can force a seasonality with python pyramid with a specified periodicity # you cannot force a seasonality in R autoarima # I have tried upping parameters like in: # https://stackoverflow.com/questions/24390859/why-does-auto-arima-drop-my-seasonality-component-when-stepwise-false-and-approx # but it does not work # I will instead fill in missing values using interpolation and use python pyramid and FIX s # this will give me p,d,q,P,D,Q # then I will fill in missing values with NaNs and then use Statsmodels' fit if ts_obj.miss: ref_date_range = ch.get_ref_date_range(ts_obj.dataframe, ts_obj.dateformat, ts_obj.timestep) gaps = ref_date_range[~ref_date_range.isin(ts_obj. dataframe["timestamp"])] filled_df_value = ch.fill_df(ts_obj.dataframe, ts_obj.timestep, ref_date_range, "fill_value") filled_df_value = pd.DataFrame({ "timestamp": filled_df_value.index, "value": filled_df_value["value"] }) endogenous_values_filled_interpolate = filled_df_value.set_index( 'timestamp')['value'] filled_df_nan = ch.fill_df(ts_obj.dataframe, ts_obj.timestep, ref_date_range, "fill_nan") endogenous_values_filled_nan = filled_df_nan.set_index( 'timestamp')['value'] exogenous_values_filled_interpolate = ah.get_exogenous( endogenous_values_filled_interpolate, ts_obj.get_dateformat()).drop("Intercept", axis=1, errors='ignore') exogenous_values_filled_nan = ah.get_exogenous( endogenous_values_filled_nan, ts_obj.get_dateformat()).drop("Intercept", axis=1, errors='ignore') # fill NaNs with values using interpolation to use Pyramid try: arima = pm.auto_arima( y=endogenous_values_filled_interpolate, exogenous=exogenous_values_filled_interpolate, max_p=3, max_q=3, error_action='ignore', trace=True, suppress_warnings=True, maxiter=1, maxorder=5, seasonal=ts_obj.seasonality, m=ts_obj.period) # http://alkaline-ml.com/pmdarima/seasonal-differencing-issues.html except ValueError: arima = pm.auto_arima( y=endogenous_values_filled_interpolate, exogenous=exogenous_values_filled_interpolate, D=0, max_p=3, max_q=3, error_action='ignore', trace=True, suppress_warnings=True, maxiter=1, maxorder=5, seasonal=ts_obj.seasonality, m=ts_obj.period) order = arima.order seasonal_order = arima.seasonal_order # order = (2,1,2) # seasonal_order = (0,0,1,3) # print(order) # print(seasonal_order) # use NaNs with sarimax fitting with statsmodels try: fit_result = sm.tsa.SARIMAX(endogenous_values_filled_nan, exogenous_values_filled_nan, order=order, seasonal_order=seasonal_order, time_varying=True, mle_regression=False).fit() # https://github.com/statsmodels/statsmodels/issues/5459 # https://github.com/statsmodels/statsmodels/issues/5374 except np.linalg.LinAlgError as err: print("\n\n!!!!") print("enforce_stationarity = False") fit_result = sm.tsa.SARIMAX(endogenous_values_filled_nan, exogenous_values_filled_nan, order=order, seasonal_order=seasonal_order, time_varying=True, mle_regression=False, enforce_stationarity=False).fit() model = sm.tsa.SARIMAX(endogenous_values_filled_nan, exogenous_values_filled_nan, order=order, seasonal_order=seasonal_order, time_varying=True, mle_regression=False) model.initialize_known(fit_result.filtered_state[..., -1], fit_result.filtered_state_cov[..., -1]) model.update(model.start_params) filter_result = model.ssm.filter() response = filter_result.forecasts.squeeze(0) # print(len(ts_obj.dataframe["value"])) # print(len(response)) filled_df_nan["response"] = response filled_df_nan = filled_df_nan.dropna() response = filled_df_nan["response"].values # print(len(ts_obj.dataframe["value"])) # print(len(response)) anomaly_scores = ah.determine_anomaly_scores_error( ts_obj.dataframe["value"], response, len(response), gaussian_window_size, step_size) else: endogenous_values = ts_obj.dataframe.set_index('timestamp')['value'] exogenous_values = ah.get_exogenous(endogenous_values, ts_obj.get_dateformat()).drop( "Intercept", axis=1, errors='ignore') try: arima = pm.auto_arima(y=endogenous_values, exogenous=exogenous_values, max_p=3, max_q=3, error_action='ignore', trace=True, suppress_warnings=True, maxiter=1, maxorder=5, seasonal=ts_obj.seasonality, m=ts_obj.period) # http://alkaline-ml.com/pmdarima/seasonal-differencing-issues.html except ValueError: arima = pm.auto_arima(y=endogenous_values, exogenous=exogenous_values, D=0, max_p=3, max_q=3, error_action='ignore', trace=True, suppress_warnings=True, maxiter=1, maxorder=5, seasonal=ts_obj.seasonality, m=ts_obj.period) order = arima.order seasonal_order = arima.seasonal_order # print("!!!!!!!!!!!!!!!") # print(ts_obj.name) # print(order) # print(seasonal_order) # print("!!!!!!!!!!!!!!!") try: fit_result = sm.tsa.SARIMAX(endogenous_values, exogenous_values, order=order, seasonal_order=seasonal_order, time_varying=True, mle_regression=False).fit() model = sm.tsa.SARIMAX(endogenous_values, exogenous_values, order=order, seasonal_order=seasonal_order, time_varying=True, mle_regression=False) except np.linalg.LinAlgError as err: print("\n\n!!!!") print("enforce_stationarity = False") fit_result = sm.tsa.SARIMAX(endogenous_values, exogenous_values, order=order, seasonal_order=seasonal_order, time_varying=True, mle_regression=False, enforce_stationarity=False, simple_differencing=True).fit() model = sm.tsa.SARIMAX(endogenous_values, exogenous_values, order=order, seasonal_order=seasonal_order, time_varying=True, mle_regression=False, enforce_stationarity=False, simple_differencing=True) model.initialize_known(fit_result.filtered_state[..., -1], fit_result.filtered_state_cov[..., -1]) model.update(model.start_params) filter_result = model.ssm.filter() response = filter_result.forecasts.squeeze(0) anomaly_scores = ah.determine_anomaly_scores_error(endogenous_values, response, len(response), gaussian_window_size, step_size) end = time.time() if plot_forecast: plt.plot(response, alpha=.7, label="Predictions") plt.plot(ts_obj.dataframe["value"].values, alpha=.5, label="Data") plt.legend() plt.show() if plot_anomaly_score: plt.subplot(211) plt.title("Anomaly Scores") plt.plot(anomaly_scores) plt.ylim([.99, 1]) plt.subplot(212) plt.title("Time Series") plt.plot(ts_obj.dataframe["value"].values) plt.axvline(ts_obj.get_probationary_index(), color="black", label="probationary line") plt.tight_layout() plt.show() return { "Anomaly Scores": np.asarray(anomaly_scores), "Time": end - start, "Forecast": response }