def _process_kalman(df): kf1 = KalmanFilter(transition_matrices=[1], observation_matrices=[1], initial_state_mean=0, initial_state_covariance=1, observation_covariance=1, transition_covariance=.01) x_state_means, _ = kf1.filter(df.x.values) y_state_means, _ = kf1.filter(df.y.values) df['smooth_x'] = x_state_means df['smooth_y'] = y_state_means obs_mat = np.vstack([df.smooth_x, np.ones(df.smooth_x.shape)]).T[:, np.newaxis] delta = 1e-5 trans_cov = delta / (1 - delta) * np.eye(2) kf2 = KalmanFilter(n_dim_obs=1, n_dim_state=2, initial_state_mean=np.zeros(2), initial_state_covariance=np.ones((2, 2)), transition_matrices=np.eye(2), observation_matrices=obs_mat, observation_covariance=1.0, transition_covariance=trans_cov) reg_state, _ = kf2.filter(df.smooth_y.values) df['beta'], df['alpha'] = reg_state[:, 0], reg_state[:, 1] df['new_x'] = df['smooth_x'] * df['beta'] + df['alpha'] df['spread'] = df['smooth_y'] - df['new_x'] df['stdev'] = df.expanding().spread.std() df['z'] = df['spread'] / df['stdev']
def rebalance(context, data): x = np.asarray([data.current(context.ewa, 'price'), 1.0]) y = data.current(context.ewc, 'price') delta = 0.0001 trans_cov = delta / (1 - delta) * np.eye(2) obs_mat = np.expand_dims(np.vstack([[x], [np.ones(x.shape[0])]]).T, axis=1) kf = KalmanFilter(n_dim_obs=1, n_dim_state=2, initial_state_mean=[0, 0], initial_state_covariance=np.ones((2, 2)), transition_matrices=np.eye(2), observation_matrices=obs_mat, observation_covariance=2, transition_covariance=trans_cov) state_means = kf.filter(y)[0] curx = data.current(context.ewa, 'price') cury = data.current(context.ewc, 'price') sm = state_means[0, -1] est = (cury - (sm * curx)) threshold = est - sm if threshold <= 0.095: order_target_percent(context.ewc, est) order_target_percent(context.ewa, -1 * est) else: order_target_percent(context.ewa, est) order_target_percent(context.ewc, -1 * est)
class KalmanRegression(object): def __init__(self, x_state_mean, y_state_mean, delta=1e-5): trans_cov = delta / (1 - delta) * np.eye(2) obs_mat = np.expand_dims(np.vstack([[x_state_mean], [np.ones(len(x_state_mean))]]).T, axis=1) self.kf = KalmanFilter(n_dim_obs=1, n_dim_state=2, initial_state_mean=np.zeros(2), initial_state_covariance=np.ones((2, 2)), transition_matrices=np.eye(2), observation_matrices=obs_mat, observation_covariance=1.0, transition_covariance=trans_cov) means, cov = self.kf.filter(y_state_mean) self.mean = means[-1] self.cov = cov[-1] def update(self, x, y): self.mean, self.cov = self.kf.filter_update( self.mean, self.cov, y, observation_matrix=np.array([[x, 1.0]])) @property def state_mean(self): return self.mean
def KalmanSequence(size, a): # a = transition matrix's (0,0) and (1,1) coordinate ########## # specify parameters random_state = np.random.RandomState(0) transition_matrix = [[a, 0], [0, a]] transition_offset = [0, 0] observation_matrix = np.eye(2) #+ random_state.randn(2, 2) * 0.1 observation_offset = [0, 0] transition_covariance = np.eye(2) observation_covariance = np.eye(2) #+ random_state.randn(2, 2) * 0.1 initial_state_mean = [5, -5] initial_state_covariance = [[1, 0], [0, 1]] # sample from model kf = KalmanFilter( transition_matrix, observation_matrix, transition_covariance, observation_covariance, transition_offset, observation_offset, initial_state_mean, initial_state_covariance, random_state=random_state ) states, observations = kf.sample(n_timesteps=size, initial_state=initial_state_mean) # estimate state with filtering and smoothing filtered_state_estimates = kf.filter(observations)[0] # smoothed_state_estimates = kf.smooth(observations)[0] return states, observations, filtered_state_estimates
def kalman_filter(self, points): x, y, z = Point.points_2_xyz(points) kf = KalmanFilter(transition_matrices=np.array([[1, 1], [0, 1]]), transition_covariance=0.01 * np.eye(2)) x_filtered = kf.filter(x)[0] y_filtered = kf.filter(y)[0] z_filtered = kf.filter(z)[0] x = x_filtered[:, 0] y = y_filtered[:, 0] z = z_filtered[:, 0] # dx = x_filtered[:, 1] # dy = y_filtered[:, 1] # dz = z_filtered[:, 1] return Point.xyz_2_points(x, y, z)
def test_kalman_filter(self): kf = KalmanFilter(transition_matrices = [[1, 1], [0, 1]], observation_matrices = [[0.1, 0.5], [-0.3, 0.0]]) measurements = np.asarray([[1,0], [0,0], [0,1]]) # 3 observations kf = kf.em(measurements, n_iter=5) (filtered_state_means, filtered_state_covariances) = kf.filter(measurements) (smoothed_state_means, smoothed_state_covariances) = kf.smooth(measurements) return filtered_state_means
def analysis(self, asset1, asset2, visual=False): # Kalman Filter delta = 1e-5 trans_cov = delta / (1 - delta) * np.eye(2) obs_mat = np.vstack([self.data[asset1], \ np.ones(self.data[asset1].shape)]).T[:, np.newaxis] # set parameters kf = KalmanFilter(n_dim_obs=1, n_dim_state=2, initial_state_mean=np.zeros(2), initial_state_covariance=np.ones((2, 2)), transition_matrices=np.eye(2), observation_matrices=obs_mat, observation_covariance=1.0, transition_covariance=trans_cov) # calculate rolling beta and intercept state_means, state_covs = kf.filter(self.data[asset2].values) beta_slope = pd.DataFrame(dict(slope=state_means[:, 0], \ intercept=state_means[:, 1]), index = self.data.index) if visual == True: # visualization for correlation cm = plt.cm.get_cmap('jet') colors = np.linspace(0.1, 1, len(self.data)) sc = plt.scatter(self.data[asset1], self.data[asset2], \ s=30, c=colors, cmap=cm, edgecolor='k', alpha=0.7) cb = plt.colorbar(sc) cb.ax.set_yticklabels([str(p.date()) for p in \ self.data[::len(self.data)//9].index]) plt.xlabel(asset1) plt.ylabel(asset2) plt.show() # plot beta and slope beta_slope.plot(subplots=True) plt.show() # visualize the correlation between assest prices over time cm = plt.cm.get_cmap('jet') colors = np.linspace(0.1, 1, len(self.data)) sc = plt.scatter(self.data[asset1], self.data[asset2], \ s=50, c=colors, cmap=cm, edgecolor='k', alpha=0.7) cb = plt.colorbar(sc) cb.ax.set_yticklabels([ str(p.date()) for p in self.data[::len(self.data) // 9].index ]) plt.xlabel(asset1) plt.ylabel(asset2) # add regression lines step = 5 xi = np.linspace(self.data[asset1].min(), self.data[asset1].max(), 2) colors_l = np.linspace(0.1, 1, len(state_means[::step])) for i, beta in enumerate(state_means[::step]): plt.plot(xi, beta[0] * xi + beta[1], alpha=.2, lw=1, c=cm(colors_l[i])) return beta_slope
def test_kalman_filter_update(): kf = KalmanFilter( data.transition_matrix, data.observation_matrix, data.transition_covariance, data.observation_covariance, data.transition_offsets, data.observation_offset, data.initial_state_mean, data.initial_state_covariance) # use Kalman Filter (x_filt, V_filt) = kf.filter(X=data.observations) # use online Kalman Filter n_timesteps = data.observations.shape[0] n_dim_state = data.transition_matrix.shape[0] x_filt2 = np.zeros((n_timesteps, n_dim_state)) V_filt2 = np.zeros((n_timesteps, n_dim_state, n_dim_state)) for t in range(n_timesteps - 1): if t == 0: x_filt2[0] = data.initial_state_mean V_filt2[0] = data.initial_state_covariance (x_filt2[t + 1], V_filt2[t + 1]) = kf.filter_update( x_filt2[t], V_filt2[t], data.observations[t + 1], transition_offset=data.transition_offsets[t] ) assert_array_almost_equal(x_filt, x_filt2) assert_array_almost_equal(V_filt, V_filt2)
def filter_proc(self, sensor, axes, result, init_st_mean=[0, 0, 0]): init = 0 time_index = np.array(sensor[:, 2], dtype=float) for axe in axes: # Construct a Kalman filter kf = KalmanFilter(transition_matrices=[1], observation_matrices=[1], initial_state_mean=init_st_mean[init], initial_state_covariance=1.1, observation_covariance=0.8, transition_covariance=.07) sensor_axe = pd.Series(sensor[:, axe], index=time_index, dtype=float) # Use the observed values of the price to get a rolling mean state_means, _ = kf.filter(sensor_axe.values) state_means = pd.Series(state_means.flatten(), index=sensor_axe.index) sensor[:, axe] = state_means.values init = init + 1 result.extend(sensor.tolist())
def on_update(self, data): result = {} result["timestamp"] = data["timestamp"] if self.input.size() >= self.length: independent_var = self.input.get_by_idx_range(key=None, start_idx=0, end_idx=-1) symbol_set = set(self.input.keys) depend_symbol = symbol_set.difference(self.input.default_key) depend_var = self.input.get_by_idx_range(key=depend_symbol, start_idx=0, end_idx=-1) obs_mat = np.vstack([independent_var.values, np.ones(independent_var.values.shape)]).T[:, np.newaxis] model = KalmanFilter( n_dim_obs=1, n_dim_state=2, initial_state_mean=np.zeros(2), initial_state_covariance=np.ones((2, 2)), transition_matrices=np.eye(2), observation_matrices=obs_mat, observation_covariance=1.0, transition_covariance=self.trans_cov, ) state_means, state_covs = model.filter(depend_var.values) slope = state_means[:, 0][-1] result[Indicator.VALUE] = slope result[KalmanFilteringPairRegression.SLOPE] = slope result[KalmanFilteringPairRegression.SLOPE] = state_means[:, 1][-1] self.add(result) else: result[Indicator.VALUE] = np.nan result[KalmanFilteringPairRegression.SLOPE] = np.nan result[KalmanFilteringPairRegression.SLOPE] = np.nan self.add(result)
def FilteringStage(data, clase=1): if clase == 1: try: initial_state_mean = filter(lambda x: (x > 0) and (x <= 60), data)[0] except: initial_state_mean = 0 sensor_mask = np.ma.asarray(data) for i in range(0, len(data)): if data[i] <= 0.1 or data[i] >= 61: sensor_mask[i] = np.ma.masked elif clase == 2: try: initial_state_mean = filter(lambda x: (x > 0) and (x < 100), data)[0] except: initial_state_mean = 0 sensor_mask = np.ma.asarray(data) for i in range(0, len(data)): if data[i] <= 4 or data[i] >= 61: sensor_mask[i] = np.ma.masked transition_matrix = [1] observation_matrix = [1] kf = KalmanFilter(transition_matrices=transition_matrix, observation_matrices=observation_matrix, initial_state_mean=initial_state_mean, transition_covariance=0.05, observation_covariance=0.15) state_means, state_covs = kf.filter( sensor_mask) #Ejecuta el filtro de kalman return state_means[:, 0]
def kalman_filter_one_factor_trained(x_series, y_series, train_period): x_train = x_series[:train_period] y_train = y_series[:train_period] x_train = np.vstack([x_train]).T[:, np.newaxis] # shape = (N,1,1) y_train = np.vstack([y_train]) # shape = (1, N) kf = KalmanFilter(transition_matrices=[1], observation_matrices=x_train, initial_state_mean=[1]) kf.em(y_train) state_mean, state_cov = kf.filter(y_train) x_predict = x_series[train_period:] y_predict = y_series[train_period:] predict = {} i = 0 state_mean_cache, state_cov_cache = state_mean[-1], state_cov[-1] for ind, x, y in zip(x_predict.index, x_predict, y_predict): res = kf.filter_update([state_mean_cache], state_cov_cache, observation=y, observation_matrix=np.array([[x]])) state_mean_cache = res[0][0] state_cov_cache = res[1] predict[i] = { x_series.index.name: ind, "state_mean": float(state_mean_cache), "state_cov": float(state_cov_cache) } i = i + 1 return kf, pd.DataFrame(predict).T.set_index(x_series.index.name)
def _KF(y, x, delta=1e-5): delta = delta trans_cov = delta / (1 - delta) * np.eye(2) obs_mat = np.vstack([x, np.ones(x.shape)]).T[:, np.newaxis] kf = KalmanFilter(n_dim_obs=1, n_dim_state=2, initial_state_mean=np.zeros(2), initial_state_covariance=np.ones((2, 2)), transition_matrices=np.eye(2), observation_matrices=obs_mat, observation_covariance=1.0, transition_covariance=trans_cov) state_means, covs = kf.filter(y) beta = state_means[:, 0] intercept = state_means[:, 1] spread = y - x * beta - intercept beta = pd.DataFrame(data=beta, index=spread.index, columns=['Beta']) intercept = pd.DataFrame(data=intercept, index=spread.index, columns=['Intercept']) return spread, beta, intercept
def WLS(series): series = copy.deepcopy(series) close = series.as_matrix() obs_mat = np.vstack([close, np.ones(close.shape)]).T[:, np.newaxis] delta = 1e-5 trans_cov = delta / (delta - np.eye(2)) kf = KalmanFilter(n_dim_obs=1, n_dim_state=2, initial_state_mean=np.zeros(2), initial_state_covariance=np.ones((2, 2)), transition_matrices=np.eye(2), observation_matrices=obs_mat, observation_covariance=1, transition_covariance=trans_cov) state_means, state_covs = kf.filter(close) slope = state_means[:, 0] intercept = state_means[:, 1] estimate = slope * close + intercept df_kalman = pd.DataFrame(index=series.index) # Output feature - error error = close - estimate df_kalman['error'] = error return df_kalman['error']
def calc_slope_intercept_kalman(etfs, prices): """ Utilise the Kalman Filter from the PyKalman package to calculate the slope and intercept of the regressed ETF prices. """ delta = 1e-5 trans_cov = delta / (1 - delta) * np.eye(2) obs_mat = np.vstack( [prices[etfs[0]], np.ones(prices[etfs[0]].shape)] ).T[:, np.newaxis] kf = KalmanFilter( n_dim_obs=1, n_dim_state=2, initial_state_mean=np.zeros(2), initial_state_covariance=np.ones((2, 2)), transition_matrices=np.eye(2), observation_matrices=obs_mat, observation_covariance=1.0, transition_covariance=trans_cov ) state_means, state_covs = kf.filter(prices[etfs[1]].values) return state_means, state_covs
def calc_slope_intercept_using_kalman_filter(etfs, prices): """ Utilise the Kalman Filter from the pyKalman package to calculate the slope and intercept of the regressed ETF prices. """ delta = 1e-5 trans_cov = delta / (1 - delta) * np.eye(2) obs_mat = np.vstack( [prices[etfs[0]], np.ones(prices[etfs[0]].shape)] ).T[:, np.newaxis] kf = KalmanFilter( n_dim_obs=1, n_dim_state=2, initial_state_mean=np.zeros(2), initial_state_covariance=np.ones((2, 2)), transition_matrices=np.eye(2), observation_matrices=obs_mat, observation_covariance=1.0, transition_covariance=trans_cov ) state_means, state_covs = kf.filter(prices[etfs[1]].values) return state_means, state_covs
def KFilt(sample,fs=25): """Input (sample) is fill_in_data output. Outputs two lists of color data """ #kalman filter inputs # Dimensions of parameters: # 'transition_matrices': 2, # 'transition_offsets': 1, # 'observation_matrices': 2, # 'observation_offsets': 1, # 'transition_covariance': 2, # 'observation_covariance': 2, # 'initial_state_mean': 1, # 'initial_state_covariance': 2, n_timesteps = len(sample) trans_mat = [] #mask missing values observations = np.ma.array(sample,mask=np.zeros(sample.shape)) missing_loc = np.where(np.isnan(sample)) observations[missing_loc[0][:],missing_loc[1][:]] = np.ma.masked #Import Kalman filter, inerpolate missing points and get 2nd, 3rd orde kinematics dt = 1./25 #Length of each frame (should be iether 1/25 or 1/30) n_timesteps = len(sample) observation_matrix = np.array([[1,0,0,0], [0,1,0,0]])#np.eye(4) t = np.linspace(0,len(observations)*dt,len(observations)) q = np.cov(observations.T[:2,:400]) qdot = np.cov(np.diff(observations.T[:2,:400]))#np.cov(observations[:1,:400]) h=(t[-1]-t[0])/t.shape[0] A=np.array([[1,0,h,.5*h**2], [0,1,0,h], [0,0,1,0], [0,0,0,1]]) init_mean = [sample[0],0,0] #initial mean should be close to the first point, esp if first point is human-picked and tracking starts at the beginning of a video observation_covariance = q*500 #ADJUST THIS TO CHANGE SMOOTHNESS OF FILTER init_cov = np.eye(4)*.001#*0.0026 transition_matrix = A transition_covariance = np.array([[q[0,0],q[0,1],0,0], [q[1,0],q[1,1],0,0], [0,0,qdot[0,0],qdot[0,1]], [0,0,qdot[1,0],qdot[1,1]]]) kf = KalmanFilter(transition_matrix, observation_matrix,transition_covariance,observation_covariance,n_dim_obs=2) kf = kf.em(observations,n_iter=1,em_vars=['transition_covariance','transition_matrix','observation_covariance']) #pdb.set_trace() global trans_mat, trans_cov, init_cond x_filt = kf.filter(observations[0])[0]#observations.T[0])[0] kf_means = kf.smooth(observations[0])[0] return kf_means,x_filt #np.column_stack((color_x[:,0],color_y[:,0],color_x[:,1],color_y[:,1])),frames
def kalman2D(observations): if len(observations) < 2: return observations kf = KalmanFilter(initial_state_mean=observations[0], transition_matrices=[[1.2, 1], [0, 1]], n_dim_obs=2) pred_state, state_cov = kf.filter(observations) return pred_state.tolist()
def moving_average(values, transition_covariance=.01): kf = KalmanFilter(transition_matrices=[1], observation_matrices=[1], initial_state_mean=0, initial_state_covariance=1, observation_covariance=1, transition_covariance=transition_covariance) means, covs = kf.filter(values) return means, covs
def kalman_regression(x, y): delta = 1e-3 # How much random walk wiggles trans_cov = delta / (1 - delta) * eye(2) obs_mat = expand_dims(vstack([[x], [ones(len(x))]]).T, axis=1) kf = KalmanFilter(n_dim_obs=1, n_dim_state=2, initial_state_mean=[0,0], initial_state_covariance=ones((2, 2)), transition_matrices=eye(2), observation_matrices=obs_mat, observation_covariance=2, transition_covariance=trans_cov) state_means, state_covs = kf.filter(y.values) return state_means
def avg(x): filter = KalmanFilter(transition_matrices=[1], observation_matrices=[1], initial_state_mean=0, initial_state_covariance=1, observation_covariance=1, transition_covariance=.01) spread, _ = filter.filter(x.values) spread = pd.Series(spread.flatten(), index=x.index) return spread
def kalmanfileter(): # Read read_variable var_a = acc_read_variable[["ax"]].values kf = KalmanFilter(initial_state_mean=0, n_dim_obs=1) kf = kf.em(var_a, n_iter=5) (filtered_state_means, filtered_state_covariances) = kf.filter(acc_read_variable["ax"]) return kf
def kalman_filter(value_list: list): measurements = np.asarray(value_list) kf = KalmanFilter(transition_matrices=[1], observation_matrices=[1], initial_state_mean=measurements[0], initial_state_covariance=1, observation_covariance=8, transition_covariance=9) # 0.01) state_means, state_covariances = kf.filter(measurements) state_std = np.sqrt(state_covariances[:, 0])
def Kalman_F(self, data): m = np.mean(sorted(data[0][:6])[2:-2]) kf = KalmanFilter(transition_matrices=np.array([[1, 0], [0, 1]]), observation_matrices=np.array([[1, 0], [0, 1]]), transition_covariance=0.03 * np.eye(2), initial_state_mean=np.array([m, data[1][0]])) x = np.vstack((data[0], data[1])) y = kf.filter(x.T)[0] return [y[:, 0], y[:, 1]]
def test_online_update(self): kf = KalmanFilter(transition_matrices = [[1, 1], [0, 1]], observation_matrices = [[0.1, 0.5], [-0.3, 0.0]]) measurements = np.asarray([[1,0], [0,0], [0,1]]) # 3 observations measurements = np.ma.asarray(measurements) measurements[1] = np.ma.masked # measurement at timestep 1 is unobserved kf = kf.em(measurements, n_iter=5) (filtered_state_means, filtered_state_covariances) = kf.filter(measurements) for t in range(1, 3): filtered_state_means[t], filtered_state_covariances[t] = \ kf.filter_update(filtered_state_means[t-1], filtered_state_covariances[t-1], measurements[t]) return filtered_state_means
def kalmanLib(): """ Try KalmanFilter library :return: """ transition_matrices=[],observation_matrices=[] kf = KalmanFilter(transition_matrices , observation_matrices ) measurements = pos_ned #measurement array kf = kf.em(measurements, n_iter=5) (filtered_state_means, filtered_state_covariances) = kf.filter(measurements) (smoothed_state_means, smoothed_state_covariances) = kf.smooth(measurements)
def your_function_name(x): kf = KalmanFilter(transition_matrices=[1], observation_matrices=[1], observation_covariance=1, transition_covariance=.01, initial_state_mean=0, initial_state_covariance=1) state_means, _ = kf.filter(x.values) state_means = pd.Series(state_means.flatten(), index=x.index) return state_means
def kalman_filter_average(x): kf = KalmanFilter(transition_matrices=[1], observation_matrices=[1], initial_state_mean=0, initial_state_covariance=1, observation_covariance=1, transition_covariance=.01) # Use the observed values of the price to get a rolling mean state_means, _ = kf.filter(x.values) state_means = pd.Series(state_means.flatten(), index=x.index) return state_means
def Kalman(data): kf = KalmanFilter(n_dim_obs=1, n_dim_state=1, initial_state_mean=data[0], initial_state_covariance=20, transition_matrices=[1], transition_covariance=np.eye(1), transition_offsets=None, observation_matrices=[1], observation_covariance=10) mean, cov = kf.filter(data) return mean
def KalmanFilterAverage(V): # Construct a Kalman filter kf = KalmanFilter(transition_matrices = [1], observation_matrices = [1], initial_state_mean = 0, initial_state_covariance = 1, observation_covariance=1, transition_covariance=.01) # Get the rolling mean state_means, _ = kf.filter(V.values) state_means = pd.Series(state_means.flatten(), index=V.index) return state_means
def kfilter(df): kf = KalmanFilter(transition_matrices=[1], observation_matrices=[1], initial_state_mean=0, initial_state_covariance=1, observation_covariance=1, transition_covariance=.01) state_means, _ = kf.filter(df) state_means = pd.Series(state_means.flatten(), index=df.index) return state_means
def kalman_filter_average(x): """ Kalman noise filtering on single series to extract hidden state.""" kf = KalmanFilter(transition_matrices=[1], observation_matrices=[1], initial_state_mean=0, initial_state_covariance=2, observation_covariance=0.01, transition_covariance=0.01) state_means, _ = kf.filter(x.values) state_means = pd.Series(state_means.flatten(), index=x.index) return state_means
def update_smoo_hists(self): """ update self.smoo_X_hist, self.smoo_Y_hist, self.smoo_R3_hist according to filter_type """ if self.dps_settings.filter_type == 'none': self.smoo_X_hist = self.X_hist self.smoo_Y_hist = self.Y_hist self.smoo_R3_hist = self.R3_hist self.smoo_velX_hist = self.velX_hist self.smoo_velY_hist = self.velY_hist self.smoo_velR3_hist = self.velR3_hist elif self.dps_settings.filter_type == 'kalman': self.update_sampled_hists() kf = KalmanFilter( initial_state_mean=0, n_dim_obs=1, ) self.smoo_X_hist, _ = kf.filter(self.sampled_X_hist) self.smoo_Y_hist, _ = kf.filter(self.sampled_Y_hist) self.smoo_R3_hist, _ = kf.filter(self.sampled_R3_hist)
def kalman(self, ds): ds = ds.fillna(method='ffill') # fill in NaN's measurements = ds.to_numpy() kf = KalmanFilter(transition_matrices=[1], observation_matrices=[1], initial_state_mean=measurements[0], initial_state_covariance=1, observation_covariance=10, transition_covariance=1) state_means, state_covariances = kf.filter(measurements) state_std = np.sqrt(state_covariances[:, 0]) # in case I needed them return state_means
class KalmanRegression(object): """ Uses a Kalman Filter to estimate regression parameters in an online fashion. Estimated model: y ~ beta * x + alpha """ def __init__(self, initial_y, initial_x, delta=1e-5, maxlen=3000): self._x = initial_x.name self._y = initial_y.name self.maxlen = maxlen trans_cov = delta / (1 - delta) * np.eye(2) obs_mat = np.expand_dims( np.vstack([[initial_x], [np.ones(initial_x.shape[0])]]).T, axis=1) self.kf = KalmanFilter(n_dim_obs=1, n_dim_state=2, initial_state_mean=np.zeros(2), initial_state_covariance=np.ones((2, 2)), transition_matrices=np.eye(2), observation_matrices=obs_mat, observation_covariance=1.0, transition_covariance=trans_cov) state_means, state_covs = self.kf.filter(initial_y.values) self.means = pd.DataFrame(state_means, index=initial_y.index, columns=['beta', 'alpha']) self.state_cov = state_covs[-1] def update(self, observations): x = observations[self._x] y = observations[self._y] mu, self.state_cov = self.kf.filter_update( self.state_mean, self.state_cov, y, observation_matrix=np.array([[x, 1.0]])) mu = pd.Series(mu, index=['beta', 'alpha'], name=observations.name) self.means = self.means.append(mu) if self.means.shape[0] > self.maxlen: self.means = self.means.iloc[-self.maxlen:] def get_spread(self, observations): x = observations[self._x] y = observations[self._y] return y - (self.means.beta[-1] * x + self.means.alpha[-1]) @property def state_mean(self): return self.means.iloc[-1]
def _KalmanFilterRegression( self ): """ Use Kalman Filter to obtain first-order auto-regression parameters r_t = beta_0 + beta_1 * r_(t-1) """ returns = self.returns; _trans_cov_delta = 1e-3; # Transition matrix and covariance trans_mat = np.eye(2); # Assume beta is not to change from t-1 to t _delta = _trans_cov_delta; trans_cov = _delta / (1 - _delta) * np.eye(2); # This _delta and trans_cov seem to have great impact on the result # form Observation Matrix data = returns.values[:-1,:]; _, num_stocks = data.shape; data = np.expand_dims( data, axis = 2 ); # T-by-2-by-1 array obs_mat = np.insert( data, 1, 1, axis = 2 ); # Insert column of ones T-2-2 array obs_cov = np.eye( num_stocks ); # assume zero correlation among noises in observed stock returns #print "Shape of observation matrix is ", obs_mat.shape; #print "Example of obs_mat is ", obs_mat[:2,:,:]; # Observed stock returns: r_t index = returns.index[1:]; # index for beta_1(t) observations = returns.values[1:,:] # matrix of r_t # Form Kalman Filter and then filter! kf = KalmanFilter( n_dim_obs = num_stocks, n_dim_state = 2, # 2 regression parameters initial_state_mean = np.zeros(2), initial_state_covariance = np.ones((2,2)), transition_matrices = trans_mat, transition_covariance = trans_cov, observation_matrices = obs_mat, observation_covariance = obs_cov, ); state_means, state_covs = kf.filter( observations ); slope = pd.Series( state_means[:,0], index ); intercept = pd.Series( state_means[:,1], index ); kf_coefficients_df = pd.DataFrame( [ intercept, slope ], index = [ "coeff_0", "coeff_1" ] ); self.kf_coefficients_df = kf_coefficients_df.transpose(); return (intercept, slope);
def getPredictedValuesVer2(measurements, standard_deviation = None): #this method use PyKalman module #Apply this method only for Gains. if standard_deviation == None: #if standard deviation is not specified, then get it only from measurements list standard_deviation = getStandardDeviation(measurements) #adapt the transition_covariance. Temporary if standard_deviation < 1e-9: transition_covariance = 1e-21 else: transition_covariance = 5e-17 kf = KalmanFilter(transition_matrices = [1], observation_matrices = [1], transition_covariance = transition_covariance, observation_covariance = math.pow(standard_deviation, 2)) tmp= kf.filter(measurements)[0].tolist() for i in range(0, len(tmp)): tmp[i] = tmp[i][0] return tmp
class Regression(object): """ y = beta * x + alpha """ def __init__(self, initial_x, initial_y, date, delta=1e-5): trans_cov = delta / (1 - delta) * np.eye(2) xList = [] for x in initial_x: xList.append([[x,1.]]) obs_mat = np.vstack([xList]) self.kf = KalmanFilter(n_dim_obs=1, n_dim_state=2, initial_state_mean=np.zeros(2), initial_state_covariance=np.ones((2, 2)), transition_matrices=np.eye(2), observation_matrices=obs_mat, observation_covariance=1.0, transition_covariance=trans_cov) state_means, state_covs = self.kf.filter(np.array(initial_y)) self.means = pd.DataFrame(state_means, columns=['beta', 'alpha']) self.state_cov = state_covs[-1] def update(self, observations, date): x = observations[0] y = observations[1] mu, self.state_cov = self.kf.filter_update(self.state_mean, self.state_cov, y, observation_matrix=np.array([[x, 1.0]])) mu = pd.Series(mu, index=['beta', 'alpha'], name=date) self.means = self.means.append(mu) def get_spread(self, observations): x = observations[0] y = observations[1] return y - (self.means.beta * x + self.means.alpha) @property def state_mean(self): return self.means.iloc[-1]
def kalman_ma(df, transition_covariance=.01): df_new = pd.DataFrame() for c in df.columns: # Construct a Kalman filter kf = KalmanFilter(transition_matrices = [1], observation_matrices = [1], initial_state_mean = df.ix[0,c], initial_state_covariance = 1, observation_covariance=1, transition_covariance=transition_covariance) # Use the observed values of the price to get a rolling mean state_means, _ = kf.filter(df[c].values) df_new[c] = state_means[:,0] df_new.index = df.index return df_new
def test_kalman_filter(): kf = KalmanFilter( data.transition_matrix, data.observation_matrix, data.transition_covariance, data.observation_covariance, data.transition_offsets, data.observation_offset, data.initial_state_mean, data.initial_state_covariance) (x_filt, V_filt) = kf.filter(X=data.observations) assert_array_almost_equal( x_filt[:500], data.filtered_state_means[:500], decimal=7 ) assert_array_almost_equal( V_filt[:500], data.filtered_state_covariances[:500], decimal=7 )
def kalman_filtering(price_sequence): h = 1 #time step A = np.array([[1,h,.5*h**2], [0,1,h], [0,0,1]]) Q = np.eye(A.shape[0]) #2) Apply the filter kf = KalmanFilter(transition_matrices = A , transition_covariance = Q) means, covariances = kf.filter([price_sequence[0]]) filtered_price_sequence = [means[0,0]] for i in range(1,len(price_sequence)): #to track it (streaming) new_mean, new_covariance = kf.filter_update(means[-1], covariances[-1], price_sequence[i]) means = np.vstack([means,new_mean]) covariances = np.vstack([covariances,new_covariance.reshape((1,3,3))]) filtered_price_sequence.append(means[i,0]) return filtered_price_sequence
# n_dim_state = data.transition_matrix.shape[0] # n_timesteps = data.observations.shape[0] # blind_state_estimates = np.zeros((n_timesteps, n_dim_state)) # for t in range(n_timesteps - 1): # if t == 0: # blind_state_estimates[t] = kf.initial_state_mean # blind_state_estimates[t + 1] = ( # np.dot(kf.transition_matrices, blind_state_estimates[t]) # + kf.transition_offsets[t] # ) # Estimate the hidden states using observations up to and including # time t for t in [0...n_timesteps-1]. This method outputs the mean and # covariance characterizing the Multivariate Normal distribution for # P(x_t | z_{1:t}) filtered_state_estimates = kf.filter(data.observations)[0] # Estimate the hidden states using all observations. These estimates # will be 'smoother' (and are to be preferred) to those produced by # simply filtering as they are made with later observations in mind. # Probabilistically, this method produces the mean and covariance # characterizing, # P(x_t | z_{1:n_timesteps}) smoothed_state_estimates = kf.smooth(data.observations)[0] # Draw the true, blind,e filtered, and smoothed state estimates for all 5 # dimensions. pl.figure(figsize=(16, 6)) lines_true = pl.plot(data.states, linestyle='-', color='b') # lines_blind = pl.plot(blind_state_estimates, linestyle=':', color='m') lines_observations = pl.plot(data.observations, linestyle=':', color='m')
state_cov_multiplier = np.power(0.01, 2) # 0.1: spread_std=2.2, cov=16 ==> 0.01: 0.22, 0.16 observation_cov = 0.001 # observation matrix F is 2-dimensional, containing sym_a price and 1 # there are data.shape[0] observations obs_mat_F = np.transpose(np.vstack([data[sym_a].values, np.ones(data.shape[0])])).reshape(-1, 1, 2) kf = KalmanFilter(n_dim_obs=1, # y is 1-dimensional n_dim_state=2, # states (alpha, beta) is 2-dimensinal initial_state_mean=np.ones(2), # initial value of intercept and slope theta0|0 initial_state_covariance=np.ones((2, 2)), # initial cov matrix between intercept and slope P0|0 transition_matrices=np.eye(2), # G, constant observation_matrices=obs_mat_F, # F, depends on x observation_covariance=observation_cov, # v_t, constant transition_covariance= np.eye(2)*state_cov_multiplier) # w_t, constant state_means, state_covs = kf.filter(data[sym_b]) # observes sym_b price beta_kf = pd.DataFrame({'Slope': state_means[:, 0], 'Intercept': state_means[:, 1]}, index=data.index) beta_kf.plot(subplots=True) plt.show() # ------------------------------------This should be equivalent to above --------------------------------------------# means_trace = [] covs_trace = [] step = 0 x = data[sym_a][step] y = data[sym_b][step] observation_matrix_stepwise = np.array([[x, 1]]) observation_stepwise = y kf = KalmanFilter(n_dim_obs=1, n_dim_state=2, initial_state_mean=np.ones(2), # initial value initial_state_covariance=np.ones((2, 2)), # initial value
observation_offset = 0.0 # d transition_covariance = 0.05 # Q observation_covariance = 0.5 # R initial_state_mean = 0.0 initial_state_covariance = 1.0 # sample from model kf = KalmanFilter( transition_matrix, observation_matrix, transition_covariance, observation_covariance, transition_offset, observation_offset, initial_state_mean, initial_state_covariance, ) #kf = kf.em(y, n_iter=5) # Expectation Maximization y_filtered['pykalman-zero-order'] = kf.filter(y)[0].flatten() y_filtered['pykalman_smoothed-zero-order'] = kf.smooth(y)[0][:,0] #-------------------------------------------------------------------------------------------- # Kalman filter 2nd order (mass-spring-damper) (PyKalman) #-------------------------------------------------------------------------------------------- m = 1000.0 # mass k_s = 0.05 # spring (the larger, the harder) k_d = 0.5 # damper (the larger, the harder) d = k_d / k_s print("d: " + str(d) + ", Damped: " + str(d > 1)) q = 0.1 # Variance of process noise, i.e. state uncertainty
def main(): #Paramètres du modèle #Indiquer à quoi correspond chaque variable osigma=0.1; transition_matrix = np.array([[1., 0.,0.],[1., 1.,0.],[0.,0,0.9]]) transition_covariance = np.zeros((3,3)); observation_matrix = np.array([[0., 1.,0.],[0., 0.,1.]]) observation_covariance = np.eye(2)*osigma initial_state_mean = np.array([1,0,10]) initial_state_covariance = np.eye(3); #Observations observations=np.array([ [1.1,9.2], [1.9,8.1], [2.8,7.2], [4.2,6.6], [5.0,5.9], [6.1,5.32], [7.2,4.7], [8.1,4.3], [9.0,3.9]]) # Filtrage à la main # Quels sont les paramètres du constructeur ? kf = KalmanFilter( transition_matrix, observation_matrix, transition_covariance, observation_covariance, ) # Que conserverons les variables suivantes ? hand_state_estimates=[initial_state_mean] hand_state_cov_estimates=[initial_state_covariance] for anObs in observations: # Quelles étapes du filtrage sont réalisées par la ligne suivante (aMean,aCov) = kf.filter_update(hand_state_estimates[-1],hand_state_cov_estimates[-1],anObs) hand_state_estimates.append(aMean) hand_state_cov_estimates.append(aCov) hand_state_estimates=np.array(hand_state_estimates) # A quoi sert la ligne suivante ? hand_positions=np.dot(hand_state_estimates,observation_matrix.T ) plt.figure(1) plt.plot(observations[:,0],observations[:,1], 'r+') plt.plot(hand_positions[:,0],hand_positions[:,1], 'b') plt.savefig('illustration_filtrage_main.pdf') plt.close() # Filtrage complet # Que fait cette section ? # Quels sont les paramètres supplémentaires donnés au constructeur ? # Quels sont les autres paramètres possibles ? kf = KalmanFilter( transition_matrix, observation_matrix, transition_covariance, observation_covariance, initial_state_mean=initial_state_mean, initial_state_covariance=initial_state_covariance, ) (filtered_state_estimates,filtered_state_cov_estimates) = kf.filter(observations) filtered_positions=np.dot(filtered_state_estimates,observation_matrix.T ) plt.figure(1) plt.plot(observations[:,0],observations[:,1], 'r+') plt.plot(filtered_positions[:,0],filtered_positions[:,1], 'b') plt.savefig('illustration_filtrage.pdf') plt.close() # Lissage # Que fait cette section ? # Quel est la différence avec le filtrage ? # Puis-je faire un lissage "à la main" observation par observation ? kf = KalmanFilter( transition_matrix, observation_matrix, transition_covariance, observation_covariance, initial_state_mean=initial_state_mean, initial_state_covariance=initial_state_covariance, ) (smoothed_state_estimates,smoothed_state_cov_estimates) = kf.smooth(observations) smoothed_positions=np.dot(smoothed_state_estimates,observation_matrix.T ) plt.figure(2) plt.plot(observations[:,0],observations[:,1], 'r+') plt.plot(smoothed_positions[:,0],smoothed_positions[:,1], 'b') plt.savefig('illustration_lissage.pdf') plt.close()
from scipy import poly1d tau = 0.1 # Set up the filter kf = KalmanFilter(n_dim_obs=1, n_dim_state=2, # position is 1-dimensional, (x,v) is 2-dimensional initial_state_mean=[30,10], initial_state_covariance=np.eye(2), transition_matrices=[[1,tau], [0,1]], observation_matrices=[[1,0]], observation_covariance=3, transition_covariance=np.zeros((2,2)), transition_offsets=[-4.9*tau**2, -9.8*tau]) # Create a simulation of a ball falling for 40 units of time (each of length tau) times = np.arange(40) actual = -4.9*tau**2*times**2 # Simulate the noisy camera data sim = actual + 3*np.random.randn(40) # Run filter on camera data state_means, state_covs = kf.filter(sim) plt.plot(times, state_means[:,0]) plt.plot(times, sim) plt.plot(times, actual) plt.legend(['Filter estimate', 'Camera data', 'Actual']) plt.xlabel('Time') plt.ylabel('Height') plt.show()
obs_mat = np.vstack([measurements, np.ones(measurements.shape)]).T[:, np.newaxis] delta = 1e-5 trans_cov = delta / (1 - delta) * np.eye(2) kf = KalmanFilter(transition_matrices = [[1, 1], [0, 1]], observation_matrices = [[0.1, 0.5], [-0.3, 0.0]], initial_state_mean=np.zeros(2)) """ kf = KalmanFilter(initial_state_mean=np.zeros(2), initial_state_covariance=np.ones((2, 2)), transition_matrices=np.eye(2), #observation_matrices = [[0.1, 0.5], [-0.3, 0.0]], observation_matrices = [[1., 1.], [3., 1.]], transition_covariance=trans_cov) """ #kf = kf.em(measurements, n_iter=10) state_means1, state_covs1 = kf.filter(measurements) #smoothed_state_means, smoothed_state_covariances = kf.smooth(measurements) print(state_covs1) pl.scatter(times, state_covs1[:, 0], marker='*', color='b') pl.show() """ plt.figure(1) times = range(measurements.shape[0]) obs_scatter = pl.scatter(times, measurements[:, 0], marker='*', color='b', label='Measurement of Regfuge 1') position_line = pl.plot(times, measurements[:, 1], marker='8', color='r', label='Measurement of Regfuge 2') velocity_line = pl.plot(times, smoothed_state_means[:, 0], linestyle='-', marker='_', color='g', label='Smoothing mean value') #velocity_line2 = pl.plot(times, smoothed_state_means[:, 2], linestyle='-', marker='_', color='m', label='Smoothing mean value 2') pl.legend(loc='lower right')
def analysis_Kalman(countries_num, type): with open("D3-data-file-refugee-1.csv") as csvfile: reader = csvfile.readlines() skipline = 0 line_ = 0 country_name = "" date_limit = 0 c = 0 delta = 1e-5 trans_cov = delta / (1 - delta) * np.eye(2) for row in reader: if skipline > 1: if date_limit < 20: "" x_dat.append(row.split("\t")[0]) t_s__.append(c) tmp_kal_val__.append(int(row.split("\t")[index_number])) y_dat.append(int(row.split("\t")[index_number])) c += 1 """ if date_limit > 20 and date_limit <= 40: x_dat_1.append(row.split("\t")[0]) tmp_kal_val__.append(int(row.split("\t")[index_number])) y_dat_1.append(int(row.split("\t")[index_number])) """ else: try: country_name = row.split("\t")[1].split(",")[index_number - 1] except: "" skipline += 1 date_limit += 1 kf = KalmanFilter(transition_matrices=[1], observation_matrices=[1], initial_state_mean=y_dat[0], initial_state_covariance=1, observation_covariance=1, random_state=0) TOOLS = "pan,wheel_zoom,box_zoom,reset,save" times = range(y_dat[index_number]) p = figure(plot_width=1000, toolbar_location="right", y_axis_label = "Refuges", x_axis_label = "Weekly") #state_means, _ = kf.em(np.asarray(y_dat)).smooth(np.asarray(y_dat)) #.filter(y_dat) #state_means = state_means.flatten() state_means1, _ = kf.filter(np.asarray(y_dat)) state_means1 = state_means1.flatten() #MSE = stats.linregress(t_s__, state_means) #MSE1 = stats.linregress(t_s__, state_means1) #print(MSE, MSE1) #print(state_means) #np.savetxt("trained_dataset_Syria_15.txt", state_means, delimiter=',') #print(len(x_dat), len(state_means)) data_1 = [] data_2 = [] tmp_cov = [] tmp_cov_1 = [] with open("trained_dataset_Syria_15.txt")as train_data: data__ex = train_data.readlines() for j in data__ex: tmp_cov.append(j) with open("trained_dataset_Syria_14.txt")as train_data: data__ex = train_data.readlines() for j in data__ex: tmp_cov_1.append(j) index_ = 0 for kk in tmp_cov: data_1.append(kk) data_2.append(tmp_cov_1[index_]) index_ += 1 #slope, intercept, rvalue, pvalue, stderr = stats.linregress(np.array(data_1).astype(np.float), np.array(data_2).astype(np.float)) #print (stderr) p = figure(x_range = x_dat, plot_width=1000, toolbar_location="right",y_axis_label = "Refuges moving through "+country_type +" country("+ country_name +")", x_axis_label = "Weeks") #p.line(x_dat, state_means, line_width=1, line_color = 'blue', legend="Kalman filter 0") p.line(x_dat, y_dat, line_width=1, line_color = 'green', legend="Refuge from Syria to destination country ") p.line(x_dat, state_means1, line_width=1, line_color = 'red', legend="Kalman filter 1") #slope, intercept, rvalue, pvalue, stderr = stats.linregress(np.array(y_dat).astype(np.float), np.array(state_means1).astype(np.float)) slope, intercept, rvalue, pvalue, stderr = stats.linregress(t_s__, np.array(state_means1).astype(np.float)) print(stderr) print statistics.pvariance(state_means1) #show(p) """
def main(): #Paramètres du modèle #Indiquer à quoi correspond chaque variable osigma=0.1; transition_matrix = np.array([[1., 0.,0.,0.,0.,0.],[1., 1.,0.,0.,0.,0.],[0.,0,1,0.,0.,0.]]) transition_covariance = np.zeros((6,6)); observation_matrix = np.array([[0., 1.,0.],[0., 0.,1.]]) observation_covariance = np.eye(2)*osigma initial_state_mean = np.array([1,0,10]) # m(0) initial_state_covariance = np.eye(3);# p(0) #Observations observations = csv.reader(open('voitureObservations.csv','r'),delimiter=' ') # Filtrage à la main # Quels sont les paramètres du constructeur ? kf = KalmanFilter( transition_matrix, observation_matrix, transition_covariance, observation_covariance, ) # Que conserverons les variables suivantes ? hand_state_estimates=[initial_state_mean] hand_state_cov_estimates=[initial_state_covariance] for anObs in observations: # Quelles étapes du filtrage sont réalisées par la ligne suivante (aMean,aCov) = kf.filter_update(hand_state_estimates[-1],hand_state_cov_estimates[-1],anObs) hand_state_estimates.append(aMean) hand_state_cov_estimates.append(aCov) hand_state_estimates=np.array(hand_state_estimates) # A quoi sert la ligne suivante ? hand_positions=np.dot(hand_state_estimates,observation_matrix.T ) # obtenir les observation (H*Mk-1 dans le cours, rouge) projet mon espace etat ver mon espace observation plt.figure(1) plt.plot(observations[:,0],observations[:,1], 'r+') plt.plot(hand_positions[:,0],hand_positions[:,1], 'b') plt.savefig('illustration_filtrage_main_voiture.pdf') plt.close() # Filtrage complet # Que fait cette section ? # Quels sont les paramètres supplémentaires donnés au constructeur ? # Quels sont les autres paramètres possibles ? kf = KalmanFilter( transition_matrix, observation_matrix, transition_covariance, observation_covariance, initial_state_mean=initial_state_mean, initial_state_covariance=initial_state_covariance, ) (filtered_state_estimates,filtered_state_cov_estimates) = kf.filter(observations) filtered_positions=np.dot(filtered_state_estimates,observation_matrix.T ) plt.figure(1) plt.plot(observations[:,0],observations[:,1], 'r+') plt.plot(filtered_positions[:,0],filtered_positions[:,1], 'b') plt.savefig('illustration_filtrage_voiture.pdf') plt.close() # Lissage # Que fait cette section ? # Quel est la différence avec le filtrage ? # Puis-je faire un lissage "à la main" observation par observation ? kf = KalmanFilter( transition_matrix, observation_matrix, transition_covariance, observation_covariance, initial_state_mean=initial_state_mean, initial_state_covariance=initial_state_covariance, ) (smoothed_state_estimates,smoothed_state_cov_estimates) = kf.smooth(observations) # au lieu de filtre, on appel smooth smoothed_positions=np.dot(smoothed_state_estimates,observation_matrix.T ) plt.figure(2) plt.plot(observations[:,0],observations[:,1], 'r+') plt.plot(smoothed_positions[:,0],smoothed_positions[:,1], 'b') plt.savefig('illustration_lissage_voiture.pdf') plt.close()
class KalmanFilterPairsTradingStrategy(StrategyBase): """ MeanReversionSpreadStrategy """ def __init__( self, events_engine, data_board ): super(KalmanFilterPairsTradingStrategy, self).__init__(events_engine, data_board) self.symbols = ['EWA US Equity', 'EWC US Equity'] self.bollinger_scaler = 1.0 self.state_cov_multiplier = np.power(0.01, 2) # 0.1: spread_std=2.2, cov=16 ==> 0.01: 0.22, 0.16 self.observation_cov = 0.001 self.current_ewa_size = 0 self.current_ewc_size = 0 self._kf = None # Kalman Filter self._current_state_means = None self._current_state_covs = None def on_bar(self, bar_event): print(bar_event.bar_start_time) A = self._data_board.get_hist_price(self.symbols[0], bar_event.bar_start_time) B = self._data_board.get_hist_price(self.symbols[1], bar_event.bar_start_time) x = A['Price'].iloc[-1] y = B['Price'].iloc[-1] observation_matrix_stepwise = np.array([[x, 1]]) observation_stepwise = y spread = None spread_std = None if self._kf is None: self._kf = KalmanFilter(n_dim_obs=1, n_dim_state=2, initial_state_mean=np.ones(2), # initial value initial_state_covariance=np.ones((2, 2)), # initial value transition_matrices=np.eye(2), # constant observation_matrices=observation_matrix_stepwise, # depend on x observation_covariance=self.observation_cov, # constant transition_covariance=np.eye(2) * self.state_cov_multiplier) # constant P = np.ones((2, 2)) + np.eye(2)*self.state_cov_multiplier spread = y - observation_matrix_stepwise.dot(np.ones(2))[0] spread_std = np.sqrt(observation_matrix_stepwise.dot(P).dot(observation_matrix_stepwise.transpose())[0][0] + self.observation_cov) state_means_stepwise, state_covs_stepwise = self._kf.filter(observation_stepwise) # depend on y self._current_state_means = state_means_stepwise[0] self._current_state_covs = state_covs_stepwise[0] else: state_means_stepwise, state_covs_stepwise = self._kf.filter_update( self._current_state_means, self._current_state_covs, observation=observation_stepwise, observation_matrix=observation_matrix_stepwise) P = self._current_state_covs + np.eye(2)*self.state_cov_multiplier # This has to be small enough spread = y - observation_matrix_stepwise.dot(self._current_state_means)[0] spread_std = np.sqrt(observation_matrix_stepwise.dot(P).dot(observation_matrix_stepwise.transpose())[0][0] + self.observation_cov) self._current_state_means = state_means_stepwise self._current_state_covs = state_covs_stepwise # residual is assumed to be N(0, spread_std) bollinger_ub = self.bollinger_scaler * spread_std bollinger_lb = - self.bollinger_scaler * spread_std coeff = self._current_state_means[0] print(spread, spread_std) if (spread > bollinger_ub) and (self.current_ewa_size >= 0): print ('Hit upper band, short spread.') o = OrderEvent() o.full_symbol = self.symbols[0] o.order_type = OrderType.MARKET o.order_size = int(-1000*coeff) - self.current_ewa_size self.place_order(o) self.current_ewa_size = int(-1000*coeff) o = OrderEvent() o.full_symbol = self.symbols[1] o.order_type = OrderType.MARKET o.order_size = 1000 - self.current_ewc_size self.place_order(o) self.current_ewc_size = 1000 elif (spread < bollinger_lb) and (self.current_ewa_size <= 0): print('Hit lower band, long spread.') o = OrderEvent() o.full_symbol = self.symbols[0] o.order_type = OrderType.MARKET o.order_size = int(1000 * coeff) - self.current_ewa_size self.place_order(o) self.current_ewa_size = int(1000 * coeff) o = OrderEvent() o.full_symbol = self.symbols[1] o.order_type = OrderType.MARKET o.order_size = -1000 - self.current_ewc_size self.place_order(o) self.current_ewc_size = -1000 elif (spread < 0) and (spread > bollinger_lb) and (self.current_ewa_size < 0): print('spread crosses below average.cover short spread') o = OrderEvent() o.full_symbol = self.symbols[0] o.order_type = OrderType.MARKET o.order_size = - self.current_ewa_size self.place_order(o) self.current_ewa_size = 0 o = OrderEvent() o.full_symbol = self.symbols[1] o.order_type = OrderType.MARKET o.order_size = - self.current_ewc_size self.place_order(o) self.current_ewc_size = 0 elif (spread > 0) and (spread < bollinger_ub) and (self.current_ewa_size > 0): print('spread crosses above average.cover long spread') o = OrderEvent() o.full_symbol = self.symbols[0] o.order_type = OrderType.MARKET o.order_size = - self.current_ewa_size self.place_order(o) self.current_ewa_size = 0 o = OrderEvent() o.full_symbol = self.symbols[1] o.order_type = OrderType.MARKET o.order_size = - self.current_ewc_size self.place_order(o) self.current_ewc_size = 0
from pykalman import KalmanFilter import numpy as np kf = KalmanFilter(transition_matrices = [[1, 1], [0, 1]], observation_matrices = [[0.1, 0.5], [-0.3, 0.0]]) measurements = np.asarray([[1,0], [0,0], [0,1]]) # 3 observations kf = kf.em(measurements, n_iter=5) (filtered_state_means, filtered_state_covariances) = kf.filter(measurements) print filtered_state_means
# sample from model kf = KalmanFilter( transition_matrix, observation_matrix, transition_covariance, observation_covariance, transition_offset, observation_offset, initial_state_mean, initial_state_covariance, ) # kf = kf.em(y, n_iter=5) taxels = tsframes3D[y, x, :] tsframes3D_kalman[y, x, :] = np.round(kf.filter(taxels)[0].flatten()) kalman = tsframes3D_kalman[:, :, frameID].astype(np.float32) ############################ # Spatio-Temporal Filtering ############################ # --------------- # Kalman + Median filter # --------------- kalman_median = cv2.medianBlur(kalman, d) masked = ma.masked_greater(kalman_median, 0) # Masked where result of filter > 0 kalman_median_masked = np.copy(tsframe) kalman_median_masked[~masked.mask] = 0
def speakerTracker(self): # Clean up cv2.destroyAllWindows() if self.inputPath is not None: # If a path to a file was given, assume it is a single video file if os.path.isfile(self.inputPath): cap = cv2.VideoCapture(self.inputPath) clip = VideoFileClip(self.inputPath,audio=False) fps = cap.get(cv2.cv.CV_CAP_PROP_FPS) self.numFrames = cap.get(cv2.cv.CV_CAP_PROP_FRAME_COUNT) print "[speakerTracker] Number of frames" , self.numFrames pathBase = os.path.basename(self.inputPath) pathDirectory = os.path.dirname(self.inputPath) baseName = pathDirectory + '/' + os.path.splitext(pathBase)[0] + '_' #Skip first frames if required if self.skip is not None: cap.set(cv2.cv.CV_CAP_PROP_POS_FRAMES, self.skip) # Otherwise assume it is a format string for reading images else: cap = cmtutil.FileVideoCapture(self.inputPath) #Skip first frames if required if self.skip is not None: cap.frame = 1 + self.skip else: # If no input path was specified, open camera device sys.exit("[speakerTracker] Error: no input path was specified") # The number of pixels from the center of object till the border of cropped image marginPixels = 300 # Read first frame status, im0 = cap.read() imGray0 = cv2.cvtColor(im0, cv2.COLOR_BGR2GRAY) imDraw = np.copy(im0) (tl0, br0) = cmtutil.get_rect(imDraw) print '[speakerTracker] Using', tl0, br0, 'as initial bounding box for the speaker' # First initialization to get the center self.CMT.initialise(imGray0, tl0, br0) # The first x and y coordinates of the object of interest self.inity = tl0[1] - self.CMT.center_to_tl[1] self.initx = tl0[0] - self.CMT.center_to_tl[0] # Crop the first frame imGray0_initial = imGray0[self.inity - marginPixels : self.inity + marginPixels, self.initx - marginPixels : self.initx + marginPixels] # Calculate the translation vector from main image to the cropped frame self.originFromMainImageY = self.inity - marginPixels self.originFromMainImageX = self.initx - marginPixels # Calculate the position of the selected rectangle in the cropped frame tl = (tl0[0] - self.originFromMainImageX , tl0[1] - self.originFromMainImageY) br = (br0[0] - self.originFromMainImageX , br0[1] - self.originFromMainImageY) #print '[speakerTracker] Using', tl, br, 'as initial bounding box for the speaker' # initialization and keypoint calculation self.CMT.initialise(imGray0_initial, tl, br) # Center of object in cropped frame self.currentY = tl[1] - self.CMT.center_to_tl[1] self.currentX = tl[0] - self.CMT.center_to_tl[0] # Center of object in main frame self.currentYMainImage = self.currentY + self.originFromMainImageY self.currentXMainImage = self.currentX + self.originFromMainImageX measuredTrack=np.zeros((self.numFrames+10,2))-1 count =0 # loop to read all frames, # crop them with the center of last frame, # calculate keypoints and center of the object for frame in clip.iter_frames(): # Read the frame and convert it to gray scale im_gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) # Corner correction (Height) if (self.currentYMainImage + marginPixels >= im_gray.shape[0]): self.currentYMainImage = im_gray.shape[0] - marginPixels -1 else: self.currentYMainImage = self.currentYMainImage if (self.currentXMainImage + marginPixels >= im_gray.shape[1]): self.currentXMainImage = im_gray.shape[1] - marginPixels -1 else: self.currentXMainImage = self.currentXMainImage if (self.currentYMainImage - marginPixels <= 0): self.currentYMainImage = 0 + marginPixels +1 else: self.currentYMainImage = self.currentYMainImage if (self.currentXMainImage - marginPixels <= 0): self.currentXMainImage = 0 + marginPixels +1 else: self.currentXMainImage = self.currentXMainImage # Crop it by previous coordinates im_gray_crop = im_gray[self.currentYMainImage - marginPixels : self.currentYMainImage + marginPixels, self.currentXMainImage - marginPixels : self.currentXMainImage + marginPixels] #plt.imshow(im_gray_crop, cmap = cm.Greys_r) #plt.show() #print "self.currentYMainImage:", self.currentYMainImage #print "self.currentXMainImage:", self.currentXMainImage #print im_gray_crop.shape # Compute all keypoints in the cropped frame self.CMT.process_frame(im_gray_crop) #print 'frame: {2:4d}, Center: {0:.2f},{1:.2f}'.format(self.CMT.center[0], self.CMT.center[1] , count) if not (math.isnan(self.CMT.center[0]) or math.isnan(self.CMT.center[1]) or (self.CMT.center[0] <= 0) or (self.CMT.center[1] <= 0)): # Compute the center of the object with respect to the main image self.diffY = self.CMT.center[0] - self.currentY self.diffX = self.CMT.center[1] - self.currentX self.currentYMainImage = self.diffY + self.currentYMainImage self.currentXMainImage = self.diffX + self.currentXMainImage self.currentY = self.CMT.center[0] self.currentX = self.CMT.center[1] # Save the center of frames in an array for further process measuredTrack[count,0] = self.currentYMainImage measuredTrack[count,1] = self.currentXMainImage else: self.currentYMainImage = self.currentYMainImage self.currentXMainImage = self.currentXMainImage print 'frame: {2:4d}, Center: {0:.2f},{1:.2f}'.format(self.currentYMainImage, self.currentXMainImage , count) count += 1 numMeas=measuredTrack.shape[0] markedMeasure=np.ma.masked_less(measuredTrack,0) # Kalman Filter Parameters deltaT = 1.0/clip.fps transitionMatrix=[[1,0,deltaT,0],[0,1,0,deltaT],[0,0,1,0],[0,0,0,1]] #A observationMatrix=[[1,0,0,0],[0,1,0,0]] #C xinit = markedMeasure[0,0] yinit = markedMeasure[0,1] vxinit = markedMeasure[1,0]-markedMeasure[0,0] vyinit = markedMeasure[1,1]-markedMeasure[0,1] initState = [xinit,yinit,vxinit,vyinit] #mu0 initCovariance = 1.0e-3*np.eye(4) #sigma0 transistionCov = 1.0e-4*np.eye(4) #Q observationCov = 1.0e-1*np.eye(2) #R # Kalman Filter bias kf = KalmanFilter(transition_matrices = transitionMatrix, observation_matrices = observationMatrix, initial_state_mean = initState, initial_state_covariance = initCovariance, transition_covariance = transistionCov, observation_covariance = observationCov) self.measuredTrack = measuredTrack # Kalman Filter (self.filteredStateMeans, self.filteredStateCovariances) = kf.filter(markedMeasure) # Kalman Smoother (self.filterStateMeanSmooth, self.filterStateCovariancesSmooth) = kf.smooth(markedMeasure) newClip = clip.fl_image( self.crop ) return newClip
class KalmanRegression(object): def __init__(self, df, dependent=None, independent=None, delta=None, trans_cov=None, obs_cov=None): if not dependent: dependent = df.columns[1] if not independent: independent = df.columns[2] self.x = df[independent] self.x.index = df.index self.y = df[dependent] self.y.index = df.index self.dependent = dependent self.independent = independent self.delta = delta or 1e-5 self.trans_cov = trans_cov or self.delta / (1 - self.delta) * np.eye(2) self.obs_mat = np.expand_dims( np.vstack([[self.x.values], [np.ones(len(self.x))]]).T, axis=1 ) self.obs_cov = obs_cov or 1 self.kf = KalmanFilter(n_dim_obs=1, n_dim_state=2, initial_state_mean=np.zeros(2), initial_state_covariance=np.ones((2, 2)), transition_matrices=np.eye(2), observation_matrices=self.obs_mat, observation_covariance=self.obs_cov, transition_covariance=self.trans_cov) self.state_means, self.state_covs = self.kf.filter(self.y.values) def slope(self): state_means = self.state_means return pd.Series(state_means[:, 0], index=self.x.index) def plot_params(self): state_means = self.state_means x = self.x _, axarr = plt.subplots(2, sharex=True) axarr[0].plot(x.index, state_means[:, 0], label='slope') axarr[0].legend() axarr[1].plot(x.index, state_means[:, 1], label='intercept') axarr[1].legend() plt.tight_layout() plt.show() return state_means[:, 0] def plot2D(self): x = self.x y = self.y state_means = self.state_means cm = plt.get_cmap('jet') colors = np.linspace(0.1, 1, len(x)) # Plot data points using colormap sc = plt.scatter(x, y, s=30, c=colors, cmap=cm, edgecolor='k', alpha=0.7) cb = plt.colorbar(sc) cb.ax.set_yticklabels([str(p.date()) for p in x[::len(x) // 9].index]) # Plot every fifth line step = 100 xi = np.linspace(x.min() - 5, x.max() + 5, 2) colors_l = np.linspace(0.1, 1, len(state_means[::step])) for i, beta in enumerate(state_means[::step]): plt.plot(xi, beta[0] * xi + beta[1], alpha=.2, lw=1, c=cm(colors_l[i])) # Plot the OLS regression line plt.plot(xi, poly1d(np.polyfit(x, y, 1))(xi), '0.4') plt.title(self.dependent + ' vs. ' + self.independent) plt.show() def run_all(self): self.plot_params() self.plot2D()
# create a Kalman Filter by hinting at the size of the state and observation space. # If you already have good guesses for the initial parameters, put them in here. # The Kalman filter will try to learn the values of all variables # The Kalman Filter is parameterized by 3 arrays for state transitions, 3 for measurements, and 2 more for initial conditions. ## This worked kf = KalmanFilter(transition_matrices=transition_matrix, observation_matrices=observation_matix, transition_covariance=0.01 * np.eye(3) ) kf = kf.em(X, n_iter=5) (filtered_state_means, filtered_state_covariances) = kf.filter(X) (smoothed_state_means, smoothed_state_covariances) = kf.smooth(X) # kf.em(X).smooth([X]) ## this gives the shape error kf = kf.em(X).smooth(X) # This returns large tuple # print len(kf_test) # Plot lines for the observations without noise, the estimated position of the # target before fitting, and the estimated position after fitting. states_pred = kf n_timesteps = X.shape[0] x = np.linspace(0, 3 * np.pi, n_timesteps) xobservations = 20 * (np.sin(x) + 0.5 * rnd.randn(n_timesteps))
#Kalman Filter tau = 0.1 kf = KalmanFilter(n_dim_obs=1,n_dim_state=2, initial_state_mean=starting_point, initial_state_covariance=np.eye(2), transition_matrices=[[1,tau],[0,1]], observation_matrices=[[1,0]], observation_covariance=3, transition_covariance=np.zeros((2,2)), transition_offsets=[0,0]) np_data = tst_data["10y Close"].values state_means, state_covs = kf.filter(np_data) #tst_data['kf_predict']=pd.Series(state_means[:,0]) #print(state_means.shape) times = np.arange(tst_data.shape[0]) plt.plot(times, state_means[:,0]) plt.plot(times, tst_data["10y Close"]) #plt.show() tst_data['KF_Value']=state_means[:,0] print("tst data",tst_data) WRITE_PATH = "L:\Trade_Output.xlsx"
from pylab import * from pykalman import KalmanFilter kf = KalmanFilter(); data = loadtxt('data_gps1.txt'); lat = radians(data[0,:]); lon = radians(data[1,:]); dlat = lat[1:] - lat[0:-1]; dlon = lon[1:] - lon[0:-1]; R = 6373; #Radius of earth in kilometers a = (sin(dlat/2))**2 + cos(lat[0:-1])*cos(lat[1:])*((sin(dlon/2))**2); c = 2*arctan2(sqrt(a),sqrt(1-a)); d = R * c; kf = kf.em(d,n_iter = 5); (smoothed_means,smoothed_cov) = kf.smooth(d); (filtered_means,filtered_cov) = kf.filter(d); plot(d,label = 'Original Data'); plot(smoothed_means,label = 'Kalman Smoothed'); plot(filtered_means,label = 'Kalman Filtered'); legend(); show();
def speakerTracker(self): # Clean up cv2.destroyAllWindows() if self.inputPath is not None: # If a path to a file was given, assume it is a single video file if os.path.isfile(self.inputPath): cap = cv2.VideoCapture(self.inputPath) clip = VideoFileClip(self.inputPath,audio=False) fps = cap.get(cv2.cv.CV_CAP_PROP_FPS) self.numFrames = cap.get(cv2.cv.CV_CAP_PROP_FRAME_COUNT) print "[speakerTracker] Number of frames" , self.numFrames pathBase = os.path.basename(self.inputPath) pathDirectory = os.path.dirname(self.inputPath) baseName = pathDirectory + '/' + os.path.splitext(pathBase)[0] + '_' # Otherwise assume it is a format string for reading images else: cap = cmtutil.FileVideoCapture(self.inputPath) else: # If no input path was specified, open camera device sys.exit("[speakerTracker] Error: no input path was specified") # Read first frame status, im0 = cap.read() imGray0 = cv2.cvtColor(im0, cv2.COLOR_BGR2GRAY) imDraw = np.copy(im0) (tl, br) = cmtutil.get_rect(imDraw) print '[speakerTrackering] Using', tl, br, 'as initial bounding box for the speaker' self.CMT.initialise(imGray0, tl, br) #self.inity = tl[1] - self.CMT.center_to_tl[1] measuredTrack=np.zeros((self.numFrames+10,2))-1 count =0 for frame in clip.iter_frames(): im_gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) self.CMT.process_frame(im_gray) print 'frame: {2:4d}, Center: {0:.2f},{1:.2f}'.format(self.CMT.center[0], self.CMT.center[1] , count) #print self.inity if not (math.isnan(self.CMT.center[0]) or (self.CMT.center[0] <= 0)): measuredTrack[count,0] = self.CMT.center[0] measuredTrack[count,1] = self.CMT.center[1] count += 1 numMeas=measuredTrack.shape[0] markedMeasure=np.ma.masked_less(measuredTrack,0) # Kalman Filter Parameters deltaT = 1.0/clip.fps transitionMatrix=[[1,0,deltaT,0],[0,1,0,deltaT],[0,0,1,0],[0,0,0,1]] #A observationMatrix=[[1,0,0,0],[0,1,0,0]] #C xinit = markedMeasure[0,0] yinit = markedMeasure[0,1] vxinit = markedMeasure[1,0]-markedMeasure[0,0] vyinit = markedMeasure[1,1]-markedMeasure[0,1] initState = [xinit,yinit,vxinit,vyinit] #mu0 initCovariance = 1.0e-3*np.eye(4) #sigma0 transistionCov = 1.0e-4*np.eye(4) #Q observationCov = 1.0e-1*np.eye(2) #R kf = KalmanFilter(transition_matrices = transitionMatrix, observation_matrices = observationMatrix, initial_state_mean = initState, initial_state_covariance = initCovariance, transition_covariance = transistionCov, observation_covariance = observationCov) self.measuredTrack = measuredTrack (self.filteredStateMeans, self.filteredStateCovariances) = kf.filter(markedMeasure) (self.filterStateMeanSmooth, self.filterStateCovariancesSmooth) = kf.smooth(markedMeasure) self.inity = np.mean(self.filterStateMeanSmooth[:][1], axis=0) newClip = clip.fl_image( self.crop ) return newClip
def pykalman_test(): kf = KalmanFilter(transition_matrices = [[1, 1], [0, 1]], observation_matrices = [[0.1, 0.5], [-0.3, 0.0]]) measurements = numpy.asarray([[1,0], [0,0], [0,1]]) # 3 observations kf = kf.em(measurements, n_iter=5) print "pykalman test >>>", kf.filter(measurements)
# Initialize filter kf = KalmanFilter(transition_matrices = [[1,0],[0,1]], observation_matrices = [[1,0],[0,1]]) image = numpy.ones([480, 640, 3]) * 128 # First two (2) measurements (initializes variables) measurements = numpy.ma.asarray(UV) # masking enabled! measurements = numpy.ma.concatenate((numpy.ma.asarray([[0,0],[0,0]]), measurements)) measurements[0:2] = numpy.ma.masked measurements[::2] = numpy.ma.masked measurements[100:110] = numpy.ma.masked measurements[200:210] = numpy.ma.masked measurements[300:310] = numpy.ma.masked measurements[400:410] = numpy.ma.masked mu, sig = kf.filter(measurements[0:2]) # initialize on bogus data print measurements print mu print sig print '' #t = 0 mu = mu[-1] sig = sig[-1] for i in range(2, len(measurements)): # starting with third measurement... if rospy.is_shutdown(): # easy interruption break t = i * 1/10. #t += 1/10.