def test_kalman2(): from pykalman import UnscentedKalmanFilter ukf = UnscentedKalmanFilter(lambda x, w: x + np.sin(w), lambda x, v: x + v, transition_covariance=0.1) (filtered_state_means, filtered_state_covariances) = ukf.filter([0, 1, 2]) (smoothed_state_means, smoothed_state_covariances) = ukf.smooth([0, 1, 2])
def kalman_filter(seq): def f(state, noise): return state + np.sin(noise) def g(state, noise): return state + np.cos(noise) # kf = KalmanFilter(initial_state_mean=0, n_dim_obs=2) ukf = UnscentedKalmanFilter(f, g, 0.001) return ukf.smooth(seq)[0].T[0]
def kalman_feat(df, cols): for col in cols: ukf = UnscentedKalmanFilter(lambda x, w: x + np.sin(w), lambda x, v: x + v, observation_covariance=0.1) (filtered_state_means, filtered_state_covariances) = ukf.filter(df[col]) (smoothed_state_means, smoothed_state_covariances) = ukf.smooth(df[col]) df[col + "_UKFSMOOTH"] = smoothed_state_means.flatten() df[col + "_UKFFILTER"] = filtered_state_means.flatten() return df
def kalmanFilter(X): ukf = UnscentedKalmanFilter() X_fil = np.zeros(np.shape(X)) for i in range(np.shape(X)[1]): X_fil[:, i] = ukf.smooth(X[:, i])[0].ravel() return X_fil #if __name__ == "__main__": # # subject = 'qing_frontal_3.7' # X,y= fetchDataset(subject) # # seq_x, seq,y = split_sequences(s, n_steps): ## X_train,X_test,y_train,y_test = split_train_test(X,y)
def kalman_filter(video_file, dep0, v0): # get feature points from video points, init_image = get_feature_points_from_video(video_file) i, o = feature_points_to_observation_and_initilization(points, v0, dep0) # number of the features points observations = np.asarray(o) n = observations.shape[1] print('initilization shape is ', i.shape) print('observation shape is ', observations.shape) # initialize the vovariance and mean transition_covariance = np.eye(2*n+6) random_state = np.random.RandomState(0) observation_covariance = np.eye(n) + abs(random_state.randn(n, n) * 0.005) initial_state_mean = i covariance_init = random_state.randn(2*n+6, 2*n+6) * 0.2 covariance_init[0:3,0:3] = 0.005 initial_state_covariance = np.eye(2*n+6) + abs(covariance_init) # set Unscented kalman filter kf = UnscentedKalmanFilter( transition_function, observation_function, transition_covariance, observation_covariance, initial_state_mean, initial_state_covariance, random_state=random_state ) """ kf = AdditiveUnscentedKalmanFilter( additive_transition_function, additive_observation_function, transition_covariance, observation_covariance, initial_state_mean, initial_state_covariance ) """ # get result filtered_state_estimates = kf.filter(observations) smoothed_state_estimates = kf.smooth(observations) return filtered_state_estimates, smoothed_state_estimates
n_dim_obs=n_dim_kalman_observed, random_state=None) n = 100 while n: n -= 1 # Find the means and covariances warped_obs = np.zeros(hidden_time_length) for obs in observed_trajectories: # warp trajectory to make it to the same length of hidden trajectory warped_obs += get_time_warped_series(obs, hidden_trajectory) warped_obs = warped_obs / hidden_time_length smooth_means, smooth_cov = ukf.smooth(warped_obs) filter_means, filter_cov = ukf.filter(warped_obs) # Find new Q and R Q = ukf.transition_covariance R = ukf.observation_covariance new_Q = 0 new_R = 0 for t in range(hidden_time_length-1): d_mu = smooth_means[t+1] - _transition(smooth_means[t]) A = jacobian_transition(smooth_means[i]) L = filter_cov[t] * A.T / filter_cov[t+1] P = smooth_cov[t+1] - smooth_cov[t+1] * L.T * A.T - A.T * L.T * smooth_cov[t+1] new_Q += d_mu * d_mu.T + A * smooth_cov[t] * A.T + P
intial_covariance = [[1, 0.1], [-0.1, 1]] # UKF kf = UnscentedKalmanFilter(transition_function, observation_function, transition_covariance, observation_covariance, Initial_state, intial_covariance, random_state=noise_generator) sample = 200 states, observations = kf.sample(sample, Initial_state) # estimate state with filtering and smoothing filtered_state_estimates = kf.filter(observations)[0] smoothed_state_estimates = kf.smooth(observations)[0] # True line t = np.linspace(0, sample * 0.1, sample) y = np.sin(t) plt.plot(filtered_state_estimates[:, 0], filtered_state_estimates[:, 1], color='r', ls='-', label='UKF') plt.plot(smoothed_state_estimates[:, 0], smoothed_state_estimates[:, 1], color='g', ls='-.', label='UKF Smother')
def test_unscented_kalman(self): ukf = UnscentedKalmanFilter(lambda x, w: x + np.sin(w), lambda x, v: x + v, transition_covariance=0.1) (filtered_state_means, filtered_state_covariances) = ukf.filter([0, 1, 2]) (smoothed_state_means, smoothed_state_covariances) = ukf.smooth([0, 1, 2]) return filtered_state_means
from pykalman import UnscentedKalmanFilter import numpy as np ukf = UnscentedKalmanFilter(lambda x, w: x + np.sin(w), lambda x, v: x + v, observation_covariance=0.1) (filtered_state_means, filtered_state_covariance) = ukf.filter([0, 0, 0]) (smoothed_state_means, smoothed_state_covariance) = ukf.smooth([0, 0, 0]) def smothingData(args): delta = ukf.smooth(args)[0] return delta def filteringData(args): alpha = ukf.filter(args) return alpha
transition_covariance = np.eye(2) random_state = np.random.RandomState(0) observation_covariance = np.eye(2) + random_state.randn(2, 2) * 0.1 initial_state_mean = [0, 0] initial_state_covariance = [[1, 0.1], [-0.1, 1]] # sample from model kf = UnscentedKalmanFilter( transition_function, observation_function, transition_covariance, observation_covariance, initial_state_mean, initial_state_covariance, random_state=random_state ) states, observations = kf.sample(50, initial_state_mean) # estimate state with filtering and smoothing filtered_state_estimates = kf.filter(observations)[0] smoothed_state_estimates = kf.smooth(observations)[0] # draw estimates pl.figure() lines_true = pl.plot(states, color='b') lines_filt = pl.plot(filtered_state_estimates, color='r', ls='-') lines_smooth = pl.plot(smoothed_state_estimates, color='g', ls='-.') pl.legend((lines_true[0], lines_filt[0], lines_smooth[0]), ('true', 'filt', 'smooth'), loc='lower left' ) pl.show()
def filtering(timeseries=None, filterType='highPassRealTime'): ''' filterType can be highPassRealTime highPassBetweenRuns UnscentedKalmanFilter_filter # documentation: https://pykalman.github.io/ UnscentedKalmanFilter_smooth KalmanFilter_filter KalmanFilter_smooth noFilter ''' timeseries = timeseries.astype(np.float) oldShape = timeseries.shape timeseries = timeseries.reshape(timeseries.shape[0], -1) if filterType == 'highPassRealTime': # from highpassFunc import highPassRealTime, highPassBetweenRuns from highpass import highpass def highPassRealTime(A_matrix, TR, cutoff): full_matrix = np.transpose( highpass(np.transpose(A_matrix), cutoff / (2 * TR), True)) return full_matrix[-1, :] filtered_timeseries = [] for currTR in range(timeseries.shape[0]): filtered_timeseries.append( highPassRealTime(timeseries[:(currTR + 1)], 1.5, 56)) filtered_timeseries = np.asarray(filtered_timeseries) elif filterType == 'highPassBetweenRuns': # from highpassFunc import highPassRealTime, highPassBetweenRuns from highpass import highpass def highPassBetweenRuns(A_matrix, TR, cutoff): return np.transpose( highpass(np.transpose(A_matrix), cutoff / (2 * TR), False)) filtered_timeseries = highPassBetweenRuns(timeseries, 1.5, 56) elif filterType == 'UnscentedKalmanFilter_filter': from pykalman import UnscentedKalmanFilter ukf = UnscentedKalmanFilter(lambda x, w: x + np.sin(w), lambda x, v: x + v, observation_covariance=0.1) filtered_timeseries = np.zeros(timeseries.shape) for curr_voxel in range(timeseries.shape[1]): (filtered_timeseries_state_means, filtered_timeseries_state_covariances) = ukf.filter( timeseries[:, curr_voxel]) filtered_timeseries[:, curr_voxel] = filtered_timeseries_state_means.reshape( -1) elif filterType == 'UnscentedKalmanFilter_smooth': from pykalman import UnscentedKalmanFilter ukf = UnscentedKalmanFilter(lambda x, w: x + np.sin(w), lambda x, v: x + v, observation_covariance=0.1) filtered_timeseries = np.zeros(timeseries.shape) for curr_voxel in range(timeseries.shape[1]): (smoothed_state_means, smoothed_state_covariances) = ukf.smooth(data) filtered_timeseries[:, curr_voxel] = smoothed_state_means.reshape( -1) elif filterType == 'KalmanFilter_filter': from pykalman import KalmanFilter kf = KalmanFilter(transition_matrices=None, observation_matrices=None) filtered_timeseries = np.zeros(timeseries.shape) for curr_voxel in range(timeseries.shape[1]): measurements = np.asarray(timeseries[:, curr_voxel]) kf = kf.em(measurements, n_iter=5) (filtered_state_means, filtered_state_covariances) = kf.filter(measurements) filtered_timeseries[:, curr_voxel] = filtered_state_means.reshape( -1) elif filterType == 'KalmanFilter_smooth': from pykalman import KalmanFilter kf = KalmanFilter(transition_matrices=None, observation_matrices=None) filtered_timeseries = np.zeros(timeseries.shape) for curr_voxel in range(timeseries.shape[1]): measurements = np.asarray(timeseries[:, curr_voxel]) kf = kf.em(measurements, n_iter=5) (smoothed_state_means, smoothed_state_covariances) = kf.smooth(measurements) filtered_timeseries[:, curr_voxel] = smoothed_state_means.reshape( -1) elif filterType == 'noFilter': filtered_timeseries = timeseries else: raise Exception('filterType wrong') filtered_timeseries = filtered_timeseries.reshape(oldShape) return filtered_timeseries
def main(argv): config = read_parser(argv, Inputs, InputsOpt_Defaults) if config['mode'] == 'fuse_kalman': print('Select MASTER Features xls') root = Tk() root.withdraw() root.update() filepath = filedialog.askopenfilename() root.destroy() mydict = pd.read_excel(filepath) rownames = list(mydict.index.values) length_data = len(rownames) mydict = mydict.to_dict(orient='list') newdict = {} for key, values in mydict.items(): newdict[key] = movil_avg(mydict[key], config['n_mov_avg']) Features = [] for i in range(length_data): example = [] for feature in config['feature_array']: example.append(newdict[feature][i]) Features.append(example) Features = np.array(Features) plt.plot(Features) plt.show() scaler_model = StandardScaler() scaler_model.fit(Features) Features = scaler_model.transform(Features) plt.plot(Features) plt.show() kf = UnscentedKalmanFilter(n_dim_state=1, n_dim_obs=len(config['feature_array']), random_state=1) # kf = AdditiveUnscentedKalmanFilter(n_dim_state=1, n_dim_obs=6, random_state=1) # kf = KalmanFilter(n_dim_state=1, n_dim_obs=len(config['feature_array']), random_state=1) measurements = [i for i in Features] # kf.em(measurements) z = kf.smooth(measurements) z = z[0] plt.plot(z) plt.show() elif config['mode'] == 'testtt': if config['mypath'] == None: print('Select XLS Files with Feature') root = Tk() root.withdraw() root.update() Filepaths = filedialog.askopenfilenames() root.destroy() else: # Filepath = config['mypath'] Filepaths = [ join(config['mypath'], f) for f in listdir(config['mypath']) if isfile(join(config['mypath'], f)) if f[-4:] == 'xlsx' ] Feature = [] for filepath in Filepaths: mydict = pd.read_excel(filepath) mydict = mydict.to_dict(orient='list') Feature += mydict[config['feature']] Feature = np.array(Feature) for i in range(len(Feature)): count = 0 while np.isnan(Feature[i]) == True: count += 1 Feature[i] = Feature[i + count] # Feature = np.nan_to_num(Feature) Feature = movil_avg(Feature, config['n_mov_avg']) plt.plot(Feature) plt.show() elif config['mode'] == 'fuse_factor': print('Select MASTER Features xls') root = Tk() root.withdraw() root.update() filepath = filedialog.askopenfilename() root.destroy() mydict = pd.read_excel(filepath) rownames = list(mydict.index.values) length_data = len(rownames) mydict = mydict.to_dict(orient='list') newdict = {} for key, values in mydict.items(): newdict[key] = movil_avg(mydict[key], config['n_mov_avg']) Features = [] for i in range(length_data): example = [] for feature in config['feature_array']: example.append(newdict[feature][i]) Features.append(example) Features = np.array(Features) plt.plot(Features) plt.show() scaler_model = StandardScaler() scaler_model.fit(Features) Features = scaler_model.transform(Features) plt.plot(Features) plt.show() fa = FactorAnalysis(n_components=1, random_state=1) fa.fit(Features) z = fa.transform(Features) plt.plot(z) plt.show() elif config['mode'] == 'fuse_pca': print('Select MASTER Features xls') root = Tk() root.withdraw() root.update() filepath = filedialog.askopenfilename() root.destroy() mydict = pd.read_excel(filepath) rownames = list(mydict.index.values) length_data = len(rownames) mydict = mydict.to_dict(orient='list') newdict = {} for key, values in mydict.items(): newdict[key] = movil_avg(mydict[key], config['n_mov_avg']) Features = [] for i in range(length_data): example = [] for feature in config['feature_array']: example.append(newdict[feature][i]) Features.append(example) Features = np.array(Features) plt.plot(Features) plt.show() scaler_model = StandardScaler() scaler_model.fit(Features) Features = scaler_model.transform(Features) plt.plot(Features) plt.show() from sklearn.cluster import FeatureAgglomeration from sklearn.decomposition import FastICA from sklearn.decomposition import KernelPCA FeatureAgglomeration # model = PCA(n_components=len(config['feature_array'])) model = PCA(n_components=2) # model = FeatureAgglomeration(n_clusters=1) # model = FastICA(n_components=1) # model = KernelPCA(n_components=1, kernel='rbf') # model = KernelPCA(n_components=len(config['feature_array']), kernel='rbf') model.fit(Features) Features = model.transform(Features) print(model.explained_variance_) print(model.explained_variance_ratio_) plt.plot(Features) plt.show() # TFeatures = np.transpose(Features) # plt.plot(TFeatures[0]) # plt.show() # mydict_out = {} # mydict_out['FuFFT_ma1'] = TFeatures[0] # writer = pd.ExcelWriter('MASTER_Features.xlsx') # DataFr = pd.DataFrame(data=mydict_out, index=rownames) # DataFr.to_excel(writer, sheet_name='AE_Features') # writer.close() elif config['mode'] == 'fuse_corr': print('Select MASTER Features xls') root = Tk() root.withdraw() root.update() filepath = filedialog.askopenfilename() root.destroy() mydict = pd.read_excel(filepath) rownames = list(mydict.index.values) length_data = len(rownames) mydict = mydict.to_dict(orient='list') Features = {} for key, values in mydict.items(): Features[key] = movil_avg(mydict[key], config['n_mov_avg']) CorrCoefs = {} for feature1 in config['feature_array']: mylist = [] for feature2 in config['feature_array']: value = np.corrcoef(Features[feature1], Features[feature2])[0][1] mylist.append(value) CorrCoefs[feature1] = mylist writer = pd.ExcelWriter('Corr_Features.xlsx') DataFr = pd.DataFrame(data=CorrCoefs, index=config['feature_array']) DataFr.to_excel(writer, sheet_name='Corr_Features') writer.close() elif config['mode'] == 'test_cluster': print('Select MASTER Features xls') root = Tk() root.withdraw() root.update() filepath = filedialog.askopenfilename() root.destroy() mydict = pd.read_excel(filepath) rownames = list(mydict.index.values) length_data = len(rownames) mydict = mydict.to_dict(orient='list') newdict = {} for key, values in mydict.items(): newdict[key] = movil_avg(mydict[key], config['n_mov_avg']) Features = [] for i in range(length_data): example = [] for feature in config['feature_array']: example.append(newdict[feature][i]) Features.append(example) Features = np.array(Features) from sklearn.cluster import DBSCAN, AffinityPropagation model = AffinityPropagation() print(model.fit_predict(Features)) elif config['mode'] == 'fuse_mlp': print('Select MASTER Features xls') root = Tk() root.withdraw() root.update() filepath = filedialog.askopenfilename() root.destroy() mydict = pd.read_excel(filepath) rownames = list(mydict.index.values) length_data = len(rownames) mydict = mydict.to_dict(orient='list') newdict = {} for key, values in mydict.items(): newdict[key] = movil_avg(mydict[key], config['n_mov_avg']) Features = [] for i in range(length_data): example = [] for feature in config['feature_array']: example.append(newdict[feature][i]) Features.append(example) Features = np.array(Features) plt.plot(Features) plt.show() scaler_model = StandardScaler() scaler_model.fit(Features) Features = scaler_model.transform(Features) plt.plot(Features) plt.show() nn = MLPRegressor(hidden_layer_sizes=(5), activation='tanh', solver='lbfgs0', alpha=1.e1) nn.fit(X=Features, y=np.linspace(0, 1, length_data)) z = nn.predict(Features) # Features = pca.transform(Features) # corr = [] # TFeatures = np.transpose(Features) # for feature_pca in TFeatures: # corr.append(np.corrcoef(np.ravel(feature_pca), np.arange(len(feature_pca)))[0][1]) # z = TFeatures[np.argmax(np.absolute(corr))] # print(corr) plt.plot(z) plt.show() elif config['mode'] == 'predict_from_xls_mlp': print('Select xls') root = Tk() root.withdraw() root.update() Filepaths = filedialog.askopenfilenames() root.destroy() Feature = [] for filepath in Filepaths: mydict = pd.read_excel(filepath, sheetname=config['sheet']) mydict = mydict.to_dict(orient='list') # Feature += mydict[config['feature']][:-2] Feature += mydict[config['feature']] Feature = list(np.nan_to_num(Feature)) Feature = movil_avg(Feature, config['n_mov_avg']) Feature = np.array(Feature) # fig, ax = plt.subplots(nrows=2, ncols=1, sharex=True, sharey=True) # ax[0].plot(Feature, 'r') Feature = median_filter(data=Feature, points=5, same_length=True) # ax[1].plot(Feature, 'b') # plt.show() x_Feature = np.arange(len(Feature)) Train = Feature[0:int(config['train'] * len(Feature))] x_Train = np.arange(float(len(Train))) x_Predict = np.linspace(len(Train), len(Feature), num=len(Feature) - len(Train), endpoint=False) # scaler = StandardScaler() # scaler = RobustScaler() # scaler.fit(Train) # Train = scaler.transform(Train) clf = MLPRegressor(solver=config['solver'], alpha=config['alpha'], hidden_layer_sizes=config['layers'], random_state=config['rs'], activation=config['activation'], tol=config['tol'], verbose=True, max_iter=config['max_iter']) # from sklearn.tree import DecisionTreeRegressor # clf = DecisionTreeRegressor() n_pre = int(config['n_pre'] * len(Train)) m_post = int(config['m_post'] * len(Train)) n_ex = len(Train) - n_pre - m_post print('+++++++++++++Info: Input points n = ', n_pre) print('+++++++++++++Info: Output points m = ', m_post) print('+++++++++++++Info: Training examples = ', n_ex) a = input('enter to continue...') T_Inputs = [] T_Outputs = [] # plt.plot(Train, 'k') # plt.show() for k in range(n_ex + 1): T_Inputs.append(Train[k:k + n_pre]) T_Outputs.append(Train[k + n_pre:k + n_pre + m_post]) # aa = np.arange(len(Train[k : k + n_pre])) # plt.plot(aa, Train[k : k + n_pre], 'b') # bb = np.max(aa) + np.arange(len(Train[k + n_pre : k + n_pre + m_post])) # plt.plot(bb, Train[k + n_pre : k + n_pre + m_post], 'r') # plt.show() # sys.exit() from sklearn.model_selection import KFold, GroupKFold kf = KFold(n_splits=100, shuffle=True) # X = np.array([[1, 2], [3, 4], [5, 6], [7, 8], [9, 10], [11, 12], [13, 14], [15, 16], [17, 19]]) # y = np.array([[3], [5], [7], [9], [11], [13], [15], [17], [19]]) # # X = T_Inputs # # y = T_Outputs # for i in range(3): # for train_index, test_index in kf.split(X): # print("TRAIN:", train_index, "TEST:", test_index) # # X_train, X_test = X[train_index], X[test_index] # # y_train, y_test = y[train_index], y[test_index] # sys.exit() T_Inputs = np.array(T_Inputs) T_Outputs = np.array(T_Outputs) count = 0 epochs = 10 for i in range(epochs): print( '+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++Epoch', count) numb = 0 for train_index, test_index in kf.split(T_Inputs): print('+++++++++++Batch', numb) T_Inputs_train, T_Inputs_test = T_Inputs[ train_index], T_Inputs[test_index] T_Outputs_train, T_Outputs_test = T_Outputs[ train_index], T_Outputs[test_index] clf.partial_fit(T_Inputs_test, T_Outputs_test) numb += 1 count += 1 # clf.fit(T_Inputs, T_Outputs) print( '+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n' ) Predict = [] It_Train = list(Train) for k in range(len(x_Predict) + m_post - 1): P_Input = It_Train[n_ex + k + 1:n_ex + n_pre + k + 1] P_Output = clf.predict([P_Input]) P_Output = P_Output[0] Predict.append(P_Output[-1]) It_Train.append(P_Output[-1]) Predict = Predict[:-(m_post - 1)] fig, ax = plt.subplots() ax.set_xlabel('Accumulated Operating Hours', fontsize=13) ax.set_ylabel('Health Index', fontsize=13) ax.set_title('Linear Regression', fontsize=13) fact = 5. / 3600. ax.plot(x_Feature * fact, Feature, 'b', label='Real') ax.plot(x_Predict * fact, Predict, 'r', label='Prediction') ax.plot(x_Train * fact, Train, 'k', label='Training') ax.legend() plt.show() plt.plot(x_Feature, Feature, 'b', x_Predict, Predict, 'r', x_Train, Train, 'k') plt.show() # elif config['mode'] == 'predict_from_xls_mlp': # print('Select xls') # root = Tk() # root.withdraw() # root.update() # Filepaths = filedialog.askopenfilenames() # root.destroy() # Feature = [] # for filepath in Filepaths: # mydict = pd.read_excel(filepath, sheetname=config['sheet']) # mydict = mydict.to_dict(orient='list') # # Feature += mydict[config['feature']][:-2] # Feature += mydict[config['feature']] # Feature = list(np.nan_to_num(Feature)) # Feature = movil_avg(Feature, config['n_mov_avg']) # Feature = np.array(Feature) # # fig, ax = plt.subplots(nrows=2, ncols=1, sharex=True, sharey=True) # # ax[0].plot(Feature, 'r') # Feature = median_filter(data=Feature, points=5, same_length=True) # # ax[1].plot(Feature, 'b') # # plt.show() # x_Feature = np.arange(len(Feature)) # Train = Feature[0:int(config['train']*len(Feature))] # x_Train = np.arange(float(len(Train))) # x_Predict = np.linspace(len(Train), len(Feature), num=len(Feature) - len(Train), endpoint=False) # # scaler = StandardScaler() # # scaler = RobustScaler() # # scaler.fit(Train) # # Train = scaler.transform(Train) # clf = MLPRegressor(solver=config['solver'], alpha=config['alpha'], hidden_layer_sizes=config['layers'], random_state=config['rs'], activation=config['activation'], tol=config['tol'], verbose=True, max_iter=config['max_iter']) # # from sklearn.tree import DecisionTreeRegressor # # clf = DecisionTreeRegressor() # n_pre = int(config['n_pre']*len(Train)) # m_post = int(config['m_post']*len(Train)) # n_ex = len(Train) - n_pre - m_post # print('+++++++++++++Info: Input points n = ', n_pre) # print('+++++++++++++Info: Output points m = ', m_post) # print('+++++++++++++Info: Training examples = ', n_ex) # a = input('enter to continue...') # T_Inputs = [] # T_Outputs = [] # # plt.plot(Train, 'k') # # plt.show() # for k in range(n_ex + 1): # T_Inputs.append(Train[k : k + n_pre]) # T_Outputs.append(Train[k + n_pre : k + n_pre + m_post]) # # aa = np.arange(len(Train[k : k + n_pre])) # # plt.plot(aa, Train[k : k + n_pre], 'b') # # bb = np.max(aa) + np.arange(len(Train[k + n_pre : k + n_pre + m_post])) # # plt.plot(bb, Train[k + n_pre : k + n_pre + m_post], 'r') # # plt.show() # # sys.exit() # from sklearn.model_selection import KFold, GroupKFold # kf = KFold(n_splits=100, shuffle=True) # # X = np.array([[1, 2], [3, 4], [5, 6], [7, 8], [9, 10], [11, 12], [13, 14], [15, 16], [17, 19]]) # # y = np.array([[3], [5], [7], [9], [11], [13], [15], [17], [19]]) # # # X = T_Inputs # # # y = T_Outputs # # for i in range(3): # # for train_index, test_index in kf.split(X): # # print("TRAIN:", train_index, "TEST:", test_index) # # # X_train, X_test = X[train_index], X[test_index] # # # y_train, y_test = y[train_index], y[test_index] # # sys.exit() # T_Inputs = np.array(T_Inputs) # T_Outputs = np.array(T_Outputs) # count = 0 # epochs = 10 # for i in range(epochs): # print('+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++Epoch', count) # numb = 0 # for train_index, test_index in kf.split(T_Inputs): # print('+++++++++++Batch', numb) # T_Inputs_train, T_Inputs_test = T_Inputs[train_index], T_Inputs[test_index] # T_Outputs_train, T_Outputs_test = T_Outputs[train_index], T_Outputs[test_index] # clf.partial_fit(T_Inputs_test, T_Outputs_test) # numb += 1 # count += 1 # # clf.fit(T_Inputs, T_Outputs) # print('+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n') # Predict = [] # It_Train = list(Train) # for k in range(len(x_Predict) + m_post - 1): # P_Input = It_Train[n_ex + k + 1 : n_ex + n_pre + k + 1] # P_Output = clf.predict([P_Input]) # P_Output = P_Output[0] # Predict.append(P_Output[-1]) # It_Train.append(P_Output[-1]) # Predict = Predict[:-(m_post-1)] # fig, ax = plt.subplots() # ax.set_xlabel('Accumulated Operating Hours', fontsize=13) # ax.set_ylabel('Health Index', fontsize=13) # ax.set_title('Linear Regression', fontsize=13) # fact = 5./3600. # ax.plot(x_Feature*fact, Feature, 'b', label='Real') # ax.plot(x_Predict*fact, Predict, 'r', label='Prediction') # ax.plot(x_Train*fact, Train, 'k', label='Training') # ax.legend() # plt.show() # plt.plot(x_Feature, Feature, 'b', x_Predict, Predict, 'r', x_Train, Train, 'k') # plt.show() elif config['mode'] == 'predict_from_xls_svr': print('Select xls') root = Tk() root.withdraw() root.update() Filepaths = filedialog.askopenfilenames() root.destroy() Feature = [] for filepath in Filepaths: mydict = pd.read_excel(filepath, sheetname=config['sheet']) mydict = mydict.to_dict(orient='list') Feature += mydict[config['feature']] Feature = list(np.nan_to_num(Feature)) Feature = movil_avg(Feature, config['n_mov_avg']) Feature = np.array(Feature) x_Feature = np.arange(len(Feature)) Train = Feature[0:int(config['train'] * len(Feature))] x_Train = np.arange(float(len(Train))) x_Predict = np.linspace(len(Train), len(Feature), num=len(Feature) - len(Train), endpoint=False) # scaler = StandardScaler() # scaler = RobustScaler() # scaler.fit(Train) # Train = scaler.transform(Train) # clf = NuSVR(nu=0.5, C=1.0, kernel='poly') clf = SVR(kernel='poly', verbose=True, degree=6, C=0.5) n_pre = int(config['n_pre'] * len(Train)) m_post = 1 # m_post = int(config['m_post']*len(Train)) n_ex = len(Train) - n_pre - m_post print('+++++++++++++Info: Input points n = ', n_pre) print('+++++++++++++Info: Output points m = ', m_post) print('+++++++++++++Info: Training examples = ', n_ex) a = input('enter to continue...') T_Inputs = [] T_Outputs = [] for k in range(n_ex + 1): T_Inputs.append(Train[k:k + n_pre]) T_Outputs.append(Train[k + n_pre:k + n_pre + m_post]) clf.fit(T_Inputs, T_Outputs) print( '+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++\n' ) Predict = [] It_Train = list(Train) for k in range(len(x_Predict) + m_post - 1): P_Input = It_Train[n_ex + k + 1:n_ex + n_pre + k + 1] P_Output = clf.predict([P_Input]) P_Output = P_Output[0] Predict.append(P_Output) It_Train.append(P_Output) # Predict = Predict[:-(m_post-1)] fig, ax = plt.subplots() ax.set_xlabel('Accumulated Operating Hours', fontsize=13) ax.set_ylabel('Health Index', fontsize=13) ax.set_title('Linear Regression', fontsize=13) fact = 5. / 3600. ax.plot(x_Feature * fact, Feature, 'b', label='Real') ax.plot(x_Predict * fact, Predict, 'r', label='Prediction') ax.plot(x_Train * fact, Train, 'k', label='Training') ax.legend() plt.show() # plt.plot(x_Feature, Feature, 'b', x_Predict, Predict, 'r', x_Train, Train, 'k') # plt.show() elif config['mode'] == 'predict_from_xls_lin': print('Select xls') root = Tk() root.withdraw() root.update() Filepaths = filedialog.askopenfilenames() root.destroy() Feature = [] for filepath in Filepaths: mydict = pd.read_excel(filepath, sheetname=config['sheet']) mydict = mydict.to_dict(orient='list') Feature += mydict[config['feature']] Feature = list(np.nan_to_num(Feature)) Feature = movil_avg(Feature, config['n_mov_avg']) # Feature = median_filter(data=Feature, points=5, same_length=True) Feature = np.array(Feature) x_Feature = np.arange(len(Feature)) Train = Feature[0:int(config['train'] * len(Feature))] x_Train = np.arange(float(len(Train))) slope, intercept, r_value, p_value, std_err = stats.linregress( x_Train, Train) x_Predict = np.linspace(len(Train), len(Feature), num=len(Feature) - len(Train), endpoint=False) Predict = slope * x_Predict + intercept fig, ax = plt.subplots() ax.set_xlabel('Accumulated Operating Hours', fontsize=13) ax.set_ylabel('Health Index', fontsize=13) # ax.set_title('Linear Regression', fontsize=13) ax.set_title('Neural Network', fontsize=13) fact = 5. / 3600. ax.plot(x_Feature * fact, Feature, 'b', label='Real') ax.plot(x_Predict * fact, Predict, 'r', label='Prediction') ax.plot(x_Train * fact, Train, 'k', label='Training') ax.legend() plt.show() print(len(Feature)) else: print('unknown mode') sys.exit() return
def calc_offset(s0, sr0, s1, sr1, ax=None, start_seconds=None, length_seconds=None, chroma_weight=1.0, sf_weight=1.0, cf_weight=1.0): try: # Acutally calc the offset lookahead_ms = 100 lookbehind_ms = 600 hop_lengths = [256, 512, 1024, 2048] times = np.arange(-lookahead_ms, lookbehind_ms, 10) all_chroma_errors = [] all_sf_errors = [] all_cf_errors = [] for hop_length in hop_lengths: hop_length_seconds = hop_length / SAMPLE_RATE sf0, cf0, chroma_s0 = gen_features( s0, sr0, hop_length_seconds=hop_length_seconds) sf1, cf1, chroma_s1 = gen_features( s1, sr1, hop_length_seconds=hop_length_seconds) if start_seconds is not None: start_frames = int(start_seconds // hop_length_seconds) sf0 = sf0.copy()[start_frames:] sf1 = sf1.copy()[start_frames:] cf0 = cf0.copy()[start_frames:] cf1 = cf1.copy()[start_frames:] chroma_s0 = chroma_s0.copy()[start_frames:] chroma_s1 = chroma_s1.copy()[start_frames:] if length_seconds is not None: length_frames = int(length_seconds // hop_length_seconds) sf0 = sf0.copy()[:length_frames] sf1 = sf1.copy()[:length_frames] cf0 = cf0.copy()[:length_frames] cf1 = cf1.copy()[:length_frames] chroma_s0 = chroma_s0.copy()[:length_frames] chroma_s1 = chroma_s1.copy()[:length_frames] map_sf0 = gen_peak_map(sf0) map_sf1 = gen_peak_map(sf1) map_cf0 = gen_peak_map(cf0) map_cf1 = gen_peak_map(cf1) chroma_errors = calc_errors(chroma_s0, chroma_s1, times, hop_length, exact=True) std = np.std(chroma_errors) std = 1 if std == 0 else std chroma_errors = (chroma_errors - np.mean(chroma_errors)) / std chroma_errors *= chroma_weight if np.isfinite(chroma_errors).all(): all_chroma_errors.append(chroma_errors) sf_errors = calc_errors(sf0, sf1, times, hop_length) std = np.std(sf_errors) std = 1 if std == 0 else std sf_errors = (sf_errors - np.mean(sf_errors)) / std sf_errors *= sf_weight if np.isfinite(sf_errors).all(): all_sf_errors.append(sf_errors) cf_errors = calc_errors(cf0, cf1, times, hop_length) std = np.std(cf_errors) std = 1 if std == 0 else std cf_errors = (cf_errors - np.mean(cf_errors)) / std cf_errors *= cf_weight if np.isfinite(cf_errors).all(): all_cf_errors.append(cf_errors) # Normalise to_stack = [] if len(all_chroma_errors): all_chroma_errors = np.stack(all_chroma_errors) kf = KalmanFilter(initial_state_mean=0, n_dim_obs=all_chroma_errors.shape[0]) median_chroma_errors = kf.smooth( all_chroma_errors.transpose())[0].flatten() to_stack.append(median_chroma_errors) if len(all_sf_errors): all_sf_errors = np.stack(all_sf_errors) kf = KalmanFilter(initial_state_mean=0, n_dim_obs=all_sf_errors.shape[0]) median_sf_errors = kf.smooth( all_sf_errors.transpose())[0].flatten() to_stack.append(median_sf_errors) if len(all_cf_errors): all_cf_errors = np.stack(all_cf_errors) kf = KalmanFilter(initial_state_mean=0, n_dim_obs=all_cf_errors.shape[0]) median_cf_errors = kf.smooth( all_cf_errors.transpose())[0].flatten() to_stack.append(median_cf_errors) total = np.sum(np.stack(to_stack), axis=0) peaks, _ = calc_peaks(-total, height=1.0, prominence=1.0) if len(peaks) > 0: offsets = times[peaks] offset_ms = offsets[0] else: offsets = [] offset_ms = 0 if ax: if len(all_chroma_errors): for chroma_errors in all_chroma_errors: ax.plot(times, chroma_errors, color='r', alpha=0.3) ax.plot(times, median_chroma_errors, label='Chroma', color='r') if len(all_sf_errors): for sf_errors in all_sf_errors: ax.plot(times, sf_errors, color='g', alpha=0.3) ax.plot(times, median_sf_errors, label='Spectral Flux', color='g') if len(all_cf_errors): for cf_errors in all_cf_errors: ax.plot(times, cf_errors, color='b', alpha=0.3) ax.plot(times, median_cf_errors, label='Crest Factor', color='b') plt.plot(times, total, label='Overall', color='k', linewidth=5) for offset in offsets: alpha = 1 if offset == offset_ms else 0.2 ax.axvline(x=offset, color='r', alpha=alpha, linestyle='--') ax.legend() except Exception as e: raise print("Could not sync audio", e) offset_ms = 0 return int(offset_ms)