def _process_kalman(df):
kf1 = KalmanFilter(transition_matrices=[1],
observation_matrices=[1],
initial_state_mean=0,
initial_state_covariance=1,
observation_covariance=1,
transition_covariance=.01)
x_state_means, _ = kf1.filter(df.x.values)
y_state_means, _ = kf1.filter(df.y.values)
df['smooth_x'] = x_state_means
df['smooth_y'] = y_state_means
obs_mat = np.vstack([df.smooth_x,
np.ones(df.smooth_x.shape)]).T[:, np.newaxis]
delta = 1e-5
trans_cov = delta / (1 - delta) * np.eye(2)
kf2 = KalmanFilter(n_dim_obs=1,
n_dim_state=2,
initial_state_mean=np.zeros(2),
initial_state_covariance=np.ones((2, 2)),
transition_matrices=np.eye(2),
observation_matrices=obs_mat,
observation_covariance=1.0,
transition_covariance=trans_cov)
reg_state, _ = kf2.filter(df.smooth_y.values)
df['beta'], df['alpha'] = reg_state[:, 0], reg_state[:, 1]
df['new_x'] = df['smooth_x'] * df['beta'] + df['alpha']
df['spread'] = df['smooth_y'] - df['new_x']
df['stdev'] = df.expanding().spread.std()
df['z'] = df['spread'] / df['stdev']
def rebalance(context, data):
x = np.asarray([data.current(context.ewa, 'price'), 1.0])
y = data.current(context.ewc, 'price')
delta = 0.0001
trans_cov = delta / (1 - delta) * np.eye(2)
obs_mat = np.expand_dims(np.vstack([[x], [np.ones(x.shape[0])]]).T, axis=1)
kf = KalmanFilter(n_dim_obs=1,
n_dim_state=2,
initial_state_mean=[0, 0],
initial_state_covariance=np.ones((2, 2)),
transition_matrices=np.eye(2),
observation_matrices=obs_mat,
observation_covariance=2,
transition_covariance=trans_cov)
state_means = kf.filter(y)[0]
curx = data.current(context.ewa, 'price')
cury = data.current(context.ewc, 'price')
sm = state_means[0, -1]
est = (cury - (sm * curx))
threshold = est - sm
if threshold <= 0.095:
order_target_percent(context.ewc, est)
order_target_percent(context.ewa, -1 * est)
else:
order_target_percent(context.ewa, est)
order_target_percent(context.ewc, -1 * est)
class KalmanRegression(object):
def __init__(self, x_state_mean, y_state_mean, delta=1e-5):
trans_cov = delta / (1 - delta) * np.eye(2)
obs_mat = np.expand_dims(np.vstack([[x_state_mean],
[np.ones(len(x_state_mean))]]).T,
axis=1)
self.kf = KalmanFilter(n_dim_obs=1,
n_dim_state=2,
initial_state_mean=np.zeros(2),
initial_state_covariance=np.ones((2, 2)),
transition_matrices=np.eye(2),
observation_matrices=obs_mat,
observation_covariance=1.0,
transition_covariance=trans_cov)
means, cov = self.kf.filter(y_state_mean)
self.mean = means[-1]
self.cov = cov[-1]
def update(self, x, y):
self.mean, self.cov = self.kf.filter_update(
self.mean, self.cov, y, observation_matrix=np.array([[x, 1.0]]))
@property
def state_mean(self):
return self.mean
def KalmanSequence(size, a):
# a = transition matrix's (0,0) and (1,1) coordinate
##########
# specify parameters
random_state = np.random.RandomState(0)
transition_matrix = [[a, 0], [0, a]]
transition_offset = [0, 0]
observation_matrix = np.eye(2) #+ random_state.randn(2, 2) * 0.1
observation_offset = [0, 0]
transition_covariance = np.eye(2)
observation_covariance = np.eye(2) #+ random_state.randn(2, 2) * 0.1
initial_state_mean = [5, -5]
initial_state_covariance = [[1, 0], [0, 1]]
# sample from model
kf = KalmanFilter(
transition_matrix, observation_matrix, transition_covariance,
observation_covariance, transition_offset, observation_offset,
initial_state_mean, initial_state_covariance,
random_state=random_state
)
states, observations = kf.sample(n_timesteps=size, initial_state=initial_state_mean)
# estimate state with filtering and smoothing
filtered_state_estimates = kf.filter(observations)[0]
# smoothed_state_estimates = kf.smooth(observations)[0]
return states, observations, filtered_state_estimates
def kalman_filter(self, points):
x, y, z = Point.points_2_xyz(points)
kf = KalmanFilter(transition_matrices=np.array([[1, 1], [0, 1]]),
transition_covariance=0.01 * np.eye(2))
x_filtered = kf.filter(x)[0]
y_filtered = kf.filter(y)[0]
z_filtered = kf.filter(z)[0]
x = x_filtered[:, 0]
y = y_filtered[:, 0]
z = z_filtered[:, 0]
# dx = x_filtered[:, 1]
# dy = y_filtered[:, 1]
# dz = z_filtered[:, 1]
return Point.xyz_2_points(x, y, z)
def test_kalman_filter(self):
kf = KalmanFilter(transition_matrices = [[1, 1], [0, 1]], observation_matrices = [[0.1, 0.5], [-0.3, 0.0]])
measurements = np.asarray([[1,0], [0,0], [0,1]]) # 3 observations
kf = kf.em(measurements, n_iter=5)
(filtered_state_means, filtered_state_covariances) = kf.filter(measurements)
(smoothed_state_means, smoothed_state_covariances) = kf.smooth(measurements)
return filtered_state_means
def analysis(self, asset1, asset2, visual=False):
# Kalman Filter
delta = 1e-5
trans_cov = delta / (1 - delta) * np.eye(2)
obs_mat = np.vstack([self.data[asset1], \
np.ones(self.data[asset1].shape)]).T[:, np.newaxis]
# set parameters
kf = KalmanFilter(n_dim_obs=1,
n_dim_state=2,
initial_state_mean=np.zeros(2),
initial_state_covariance=np.ones((2, 2)),
transition_matrices=np.eye(2),
observation_matrices=obs_mat,
observation_covariance=1.0,
transition_covariance=trans_cov)
# calculate rolling beta and intercept
state_means, state_covs = kf.filter(self.data[asset2].values)
beta_slope = pd.DataFrame(dict(slope=state_means[:, 0], \
intercept=state_means[:, 1]), index = self.data.index)
if visual == True:
# visualization for correlation
cm = plt.cm.get_cmap('jet')
colors = np.linspace(0.1, 1, len(self.data))
sc = plt.scatter(self.data[asset1], self.data[asset2], \
s=30, c=colors, cmap=cm, edgecolor='k', alpha=0.7)
cb = plt.colorbar(sc)
cb.ax.set_yticklabels([str(p.date()) for p in \
self.data[::len(self.data)//9].index])
plt.xlabel(asset1)
plt.ylabel(asset2)
plt.show()
# plot beta and slope
beta_slope.plot(subplots=True)
plt.show()
# visualize the correlation between assest prices over time
cm = plt.cm.get_cmap('jet')
colors = np.linspace(0.1, 1, len(self.data))
sc = plt.scatter(self.data[asset1], self.data[asset2], \
s=50, c=colors, cmap=cm, edgecolor='k', alpha=0.7)
cb = plt.colorbar(sc)
cb.ax.set_yticklabels([
str(p.date()) for p in self.data[::len(self.data) // 9].index
])
plt.xlabel(asset1)
plt.ylabel(asset2)
# add regression lines
step = 5
xi = np.linspace(self.data[asset1].min(), self.data[asset1].max(),
2)
colors_l = np.linspace(0.1, 1, len(state_means[::step]))
for i, beta in enumerate(state_means[::step]):
plt.plot(xi,
beta[0] * xi + beta[1],
alpha=.2,
lw=1,
c=cm(colors_l[i]))
return beta_slope
def test_kalman_filter_update():
kf = KalmanFilter(
data.transition_matrix,
data.observation_matrix,
data.transition_covariance,
data.observation_covariance,
data.transition_offsets,
data.observation_offset,
data.initial_state_mean,
data.initial_state_covariance)
# use Kalman Filter
(x_filt, V_filt) = kf.filter(X=data.observations)
# use online Kalman Filter
n_timesteps = data.observations.shape[0]
n_dim_state = data.transition_matrix.shape[0]
x_filt2 = np.zeros((n_timesteps, n_dim_state))
V_filt2 = np.zeros((n_timesteps, n_dim_state, n_dim_state))
for t in range(n_timesteps - 1):
if t == 0:
x_filt2[0] = data.initial_state_mean
V_filt2[0] = data.initial_state_covariance
(x_filt2[t + 1], V_filt2[t + 1]) = kf.filter_update(
x_filt2[t], V_filt2[t], data.observations[t + 1],
transition_offset=data.transition_offsets[t]
)
assert_array_almost_equal(x_filt, x_filt2)
assert_array_almost_equal(V_filt, V_filt2)
def filter_proc(self, sensor, axes, result, init_st_mean=[0, 0, 0]):
init = 0
time_index = np.array(sensor[:, 2], dtype=float)
for axe in axes:
# Construct a Kalman filter
kf = KalmanFilter(transition_matrices=[1],
observation_matrices=[1],
initial_state_mean=init_st_mean[init],
initial_state_covariance=1.1,
observation_covariance=0.8,
transition_covariance=.07)
sensor_axe = pd.Series(sensor[:, axe],
index=time_index,
dtype=float)
# Use the observed values of the price to get a rolling mean
state_means, _ = kf.filter(sensor_axe.values)
state_means = pd.Series(state_means.flatten(),
index=sensor_axe.index)
sensor[:, axe] = state_means.values
init = init + 1
result.extend(sensor.tolist())
def on_update(self, data):
result = {}
result["timestamp"] = data["timestamp"]
if self.input.size() >= self.length:
independent_var = self.input.get_by_idx_range(key=None, start_idx=0, end_idx=-1)
symbol_set = set(self.input.keys)
depend_symbol = symbol_set.difference(self.input.default_key)
depend_var = self.input.get_by_idx_range(key=depend_symbol, start_idx=0, end_idx=-1)
obs_mat = np.vstack([independent_var.values, np.ones(independent_var.values.shape)]).T[:, np.newaxis]
model = KalmanFilter(
n_dim_obs=1,
n_dim_state=2,
initial_state_mean=np.zeros(2),
initial_state_covariance=np.ones((2, 2)),
transition_matrices=np.eye(2),
observation_matrices=obs_mat,
observation_covariance=1.0,
transition_covariance=self.trans_cov,
)
state_means, state_covs = model.filter(depend_var.values)
slope = state_means[:, 0][-1]
result[Indicator.VALUE] = slope
result[KalmanFilteringPairRegression.SLOPE] = slope
result[KalmanFilteringPairRegression.SLOPE] = state_means[:, 1][-1]
self.add(result)
else:
result[Indicator.VALUE] = np.nan
result[KalmanFilteringPairRegression.SLOPE] = np.nan
result[KalmanFilteringPairRegression.SLOPE] = np.nan
self.add(result)
def FilteringStage(data, clase=1):
if clase == 1:
try:
initial_state_mean = filter(lambda x: (x > 0) and (x <= 60),
data)[0]
except:
initial_state_mean = 0
sensor_mask = np.ma.asarray(data)
for i in range(0, len(data)):
if data[i] <= 0.1 or data[i] >= 61:
sensor_mask[i] = np.ma.masked
elif clase == 2:
try:
initial_state_mean = filter(lambda x: (x > 0) and (x < 100),
data)[0]
except:
initial_state_mean = 0
sensor_mask = np.ma.asarray(data)
for i in range(0, len(data)):
if data[i] <= 4 or data[i] >= 61:
sensor_mask[i] = np.ma.masked
transition_matrix = [1]
observation_matrix = [1]
kf = KalmanFilter(transition_matrices=transition_matrix,
observation_matrices=observation_matrix,
initial_state_mean=initial_state_mean,
transition_covariance=0.05,
observation_covariance=0.15)
state_means, state_covs = kf.filter(
sensor_mask) #Ejecuta el filtro de kalman
return state_means[:, 0]
def kalman_filter_one_factor_trained(x_series, y_series, train_period):
x_train = x_series[:train_period]
y_train = y_series[:train_period]
x_train = np.vstack([x_train]).T[:, np.newaxis] # shape = (N,1,1)
y_train = np.vstack([y_train]) # shape = (1, N)
kf = KalmanFilter(transition_matrices=[1],
observation_matrices=x_train,
initial_state_mean=[1])
kf.em(y_train)
state_mean, state_cov = kf.filter(y_train)
x_predict = x_series[train_period:]
y_predict = y_series[train_period:]
predict = {}
i = 0
state_mean_cache, state_cov_cache = state_mean[-1], state_cov[-1]
for ind, x, y in zip(x_predict.index, x_predict, y_predict):
res = kf.filter_update([state_mean_cache],
state_cov_cache,
observation=y,
observation_matrix=np.array([[x]]))
state_mean_cache = res[0][0]
state_cov_cache = res[1]
predict[i] = {
x_series.index.name: ind,
"state_mean": float(state_mean_cache),
"state_cov": float(state_cov_cache)
}
i = i + 1
return kf, pd.DataFrame(predict).T.set_index(x_series.index.name)
def _KF(y, x, delta=1e-5):
delta = delta
trans_cov = delta / (1 - delta) * np.eye(2)
obs_mat = np.vstack([x, np.ones(x.shape)]).T[:, np.newaxis]
kf = KalmanFilter(n_dim_obs=1,
n_dim_state=2,
initial_state_mean=np.zeros(2),
initial_state_covariance=np.ones((2, 2)),
transition_matrices=np.eye(2),
observation_matrices=obs_mat,
observation_covariance=1.0,
transition_covariance=trans_cov)
state_means, covs = kf.filter(y)
beta = state_means[:, 0]
intercept = state_means[:, 1]
spread = y - x * beta - intercept
beta = pd.DataFrame(data=beta, index=spread.index, columns=['Beta'])
intercept = pd.DataFrame(data=intercept,
index=spread.index,
columns=['Intercept'])
return spread, beta, intercept
def WLS(series):
series = copy.deepcopy(series)
close = series.as_matrix()
obs_mat = np.vstack([close, np.ones(close.shape)]).T[:, np.newaxis]
delta = 1e-5
trans_cov = delta / (delta - np.eye(2))
kf = KalmanFilter(n_dim_obs=1,
n_dim_state=2,
initial_state_mean=np.zeros(2),
initial_state_covariance=np.ones((2, 2)),
transition_matrices=np.eye(2),
observation_matrices=obs_mat,
observation_covariance=1,
transition_covariance=trans_cov)
state_means, state_covs = kf.filter(close)
slope = state_means[:, 0]
intercept = state_means[:, 1]
estimate = slope * close + intercept
df_kalman = pd.DataFrame(index=series.index)
# Output feature - error
error = close - estimate
df_kalman['error'] = error
return df_kalman['error']
def calc_slope_intercept_kalman(etfs, prices):
"""
Utilise the Kalman Filter from the PyKalman package
to calculate the slope and intercept of the regressed
ETF prices.
"""
delta = 1e-5
trans_cov = delta / (1 - delta) * np.eye(2)
obs_mat = np.vstack(
[prices[etfs[0]], np.ones(prices[etfs[0]].shape)]
).T[:, np.newaxis]
kf = KalmanFilter(
n_dim_obs=1,
n_dim_state=2,
initial_state_mean=np.zeros(2),
initial_state_covariance=np.ones((2, 2)),
transition_matrices=np.eye(2),
observation_matrices=obs_mat,
observation_covariance=1.0,
transition_covariance=trans_cov
)
state_means, state_covs = kf.filter(prices[etfs[1]].values)
return state_means, state_covs
def calc_slope_intercept_using_kalman_filter(etfs, prices):
"""
Utilise the Kalman Filter from the pyKalman package
to calculate the slope and intercept of the regressed
ETF prices.
"""
delta = 1e-5
trans_cov = delta / (1 - delta) * np.eye(2)
obs_mat = np.vstack(
[prices[etfs[0]], np.ones(prices[etfs[0]].shape)]
).T[:, np.newaxis]
kf = KalmanFilter(
n_dim_obs=1,
n_dim_state=2,
initial_state_mean=np.zeros(2),
initial_state_covariance=np.ones((2, 2)),
transition_matrices=np.eye(2),
observation_matrices=obs_mat,
observation_covariance=1.0,
transition_covariance=trans_cov
)
state_means, state_covs = kf.filter(prices[etfs[1]].values)
return state_means, state_covs
def KFilt(sample,fs=25):
"""Input (sample) is fill_in_data output. Outputs two lists of color data
"""
#kalman filter inputs
# Dimensions of parameters:
# 'transition_matrices': 2,
# 'transition_offsets': 1,
# 'observation_matrices': 2,
# 'observation_offsets': 1,
# 'transition_covariance': 2,
# 'observation_covariance': 2,
# 'initial_state_mean': 1,
# 'initial_state_covariance': 2,
n_timesteps = len(sample)
trans_mat = []
#mask missing values
observations = np.ma.array(sample,mask=np.zeros(sample.shape))
missing_loc = np.where(np.isnan(sample))
observations[missing_loc[0][:],missing_loc[1][:]] = np.ma.masked
#Import Kalman filter, inerpolate missing points and get 2nd, 3rd orde kinematics
dt = 1./25 #Length of each frame (should be iether 1/25 or 1/30)
n_timesteps = len(sample)
observation_matrix = np.array([[1,0,0,0],
[0,1,0,0]])#np.eye(4)
t = np.linspace(0,len(observations)*dt,len(observations))
q = np.cov(observations.T[:2,:400])
qdot = np.cov(np.diff(observations.T[:2,:400]))#np.cov(observations[:1,:400])
h=(t[-1]-t[0])/t.shape[0]
A=np.array([[1,0,h,.5*h**2],
[0,1,0,h],
[0,0,1,0],
[0,0,0,1]])
init_mean = [sample[0],0,0] #initial mean should be close to the first point, esp if first point is human-picked and tracking starts at the beginning of a video
observation_covariance = q*500 #ADJUST THIS TO CHANGE SMOOTHNESS OF FILTER
init_cov = np.eye(4)*.001#*0.0026
transition_matrix = A
transition_covariance = np.array([[q[0,0],q[0,1],0,0],
[q[1,0],q[1,1],0,0],
[0,0,qdot[0,0],qdot[0,1]],
[0,0,qdot[1,0],qdot[1,1]]])
kf = KalmanFilter(transition_matrix, observation_matrix,transition_covariance,observation_covariance,n_dim_obs=2)
kf = kf.em(observations,n_iter=1,em_vars=['transition_covariance','transition_matrix','observation_covariance'])
#pdb.set_trace()
global trans_mat, trans_cov, init_cond
x_filt = kf.filter(observations[0])[0]#observations.T[0])[0]
kf_means = kf.smooth(observations[0])[0]
return kf_means,x_filt #np.column_stack((color_x[:,0],color_y[:,0],color_x[:,1],color_y[:,1])),frames
def kalman2D(observations):
if len(observations) < 2:
return observations
kf = KalmanFilter(initial_state_mean=observations[0],
transition_matrices=[[1.2, 1], [0, 1]],
n_dim_obs=2)
pred_state, state_cov = kf.filter(observations)
return pred_state.tolist()
def moving_average(values, transition_covariance=.01):
kf = KalmanFilter(transition_matrices=[1],
observation_matrices=[1],
initial_state_mean=0,
initial_state_covariance=1,
observation_covariance=1,
transition_covariance=transition_covariance)
means, covs = kf.filter(values)
return means, covs
def kalman_regression(x, y):
delta = 1e-3
# How much random walk wiggles
trans_cov = delta / (1 - delta) * eye(2)
obs_mat = expand_dims(vstack([[x], [ones(len(x))]]).T, axis=1)
kf = KalmanFilter(n_dim_obs=1, n_dim_state=2, initial_state_mean=[0,0], initial_state_covariance=ones((2, 2)), transition_matrices=eye(2),
observation_matrices=obs_mat, observation_covariance=2, transition_covariance=trans_cov)
state_means, state_covs = kf.filter(y.values)
return state_means
def avg(x):
filter = KalmanFilter(transition_matrices=[1],
observation_matrices=[1],
initial_state_mean=0,
initial_state_covariance=1,
observation_covariance=1,
transition_covariance=.01)
spread, _ = filter.filter(x.values)
spread = pd.Series(spread.flatten(), index=x.index)
return spread
def kalmanfileter():
# Read read_variable
var_a = acc_read_variable[["ax"]].values
kf = KalmanFilter(initial_state_mean=0, n_dim_obs=1)
kf = kf.em(var_a, n_iter=5)
(filtered_state_means,
filtered_state_covariances) = kf.filter(acc_read_variable["ax"])
return kf
def kalman_filter(value_list: list):
measurements = np.asarray(value_list)
kf = KalmanFilter(transition_matrices=[1],
observation_matrices=[1],
initial_state_mean=measurements[0],
initial_state_covariance=1,
observation_covariance=8,
transition_covariance=9) # 0.01)
state_means, state_covariances = kf.filter(measurements)
state_std = np.sqrt(state_covariances[:, 0])
def Kalman_F(self, data):
m = np.mean(sorted(data[0][:6])[2:-2])
kf = KalmanFilter(transition_matrices=np.array([[1, 0], [0, 1]]),
observation_matrices=np.array([[1, 0], [0, 1]]),
transition_covariance=0.03 * np.eye(2),
initial_state_mean=np.array([m, data[1][0]]))
x = np.vstack((data[0], data[1]))
y = kf.filter(x.T)[0]
return [y[:, 0], y[:, 1]]
def test_online_update(self):
kf = KalmanFilter(transition_matrices = [[1, 1], [0, 1]], observation_matrices = [[0.1, 0.5], [-0.3, 0.0]])
measurements = np.asarray([[1,0], [0,0], [0,1]]) # 3 observations
measurements = np.ma.asarray(measurements)
measurements[1] = np.ma.masked # measurement at timestep 1 is unobserved
kf = kf.em(measurements, n_iter=5)
(filtered_state_means, filtered_state_covariances) = kf.filter(measurements)
for t in range(1, 3):
filtered_state_means[t], filtered_state_covariances[t] = \
kf.filter_update(filtered_state_means[t-1], filtered_state_covariances[t-1], measurements[t])
return filtered_state_means
def kalmanLib():
"""
Try KalmanFilter library
:return:
"""
transition_matrices=[],observation_matrices=[]
kf = KalmanFilter(transition_matrices , observation_matrices )
measurements = pos_ned #measurement array
kf = kf.em(measurements, n_iter=5)
(filtered_state_means, filtered_state_covariances) = kf.filter(measurements)
(smoothed_state_means, smoothed_state_covariances) = kf.smooth(measurements)
def your_function_name(x):
kf = KalmanFilter(transition_matrices=[1],
observation_matrices=[1],
observation_covariance=1,
transition_covariance=.01,
initial_state_mean=0,
initial_state_covariance=1)
state_means, _ = kf.filter(x.values)
state_means = pd.Series(state_means.flatten(), index=x.index)
return state_means
def kalman_filter_average(x):
kf = KalmanFilter(transition_matrices=[1],
observation_matrices=[1],
initial_state_mean=0,
initial_state_covariance=1,
observation_covariance=1,
transition_covariance=.01)
# Use the observed values of the price to get a rolling mean
state_means, _ = kf.filter(x.values)
state_means = pd.Series(state_means.flatten(), index=x.index)
return state_means
def Kalman(data):
kf = KalmanFilter(n_dim_obs=1,
n_dim_state=1,
initial_state_mean=data[0],
initial_state_covariance=20,
transition_matrices=[1],
transition_covariance=np.eye(1),
transition_offsets=None,
observation_matrices=[1],
observation_covariance=10)
mean, cov = kf.filter(data)
return mean
def KalmanFilterAverage(V):
# Construct a Kalman filter
kf = KalmanFilter(transition_matrices = [1],
observation_matrices = [1],
initial_state_mean = 0,
initial_state_covariance = 1,
observation_covariance=1,
transition_covariance=.01)
# Get the rolling mean
state_means, _ = kf.filter(V.values)
state_means = pd.Series(state_means.flatten(), index=V.index)
return state_means
def kfilter(df):
kf = KalmanFilter(transition_matrices=[1],
observation_matrices=[1],
initial_state_mean=0,
initial_state_covariance=1,
observation_covariance=1,
transition_covariance=.01)
state_means, _ = kf.filter(df)
state_means = pd.Series(state_means.flatten(), index=df.index)
return state_means
def kalman_filter_average(x):
""" Kalman noise filtering on single series to extract hidden state."""
kf = KalmanFilter(transition_matrices=[1],
observation_matrices=[1],
initial_state_mean=0,
initial_state_covariance=2,
observation_covariance=0.01,
transition_covariance=0.01)
state_means, _ = kf.filter(x.values)
state_means = pd.Series(state_means.flatten(), index=x.index)
return state_means
def update_smoo_hists(self):
"""
update self.smoo_X_hist, self.smoo_Y_hist, self.smoo_R3_hist according to filter_type
"""
if self.dps_settings.filter_type == 'none':
self.smoo_X_hist = self.X_hist
self.smoo_Y_hist = self.Y_hist
self.smoo_R3_hist = self.R3_hist
self.smoo_velX_hist = self.velX_hist
self.smoo_velY_hist = self.velY_hist
self.smoo_velR3_hist = self.velR3_hist
elif self.dps_settings.filter_type == 'kalman':
self.update_sampled_hists()
kf = KalmanFilter(
initial_state_mean=0,
n_dim_obs=1,
)
self.smoo_X_hist, _ = kf.filter(self.sampled_X_hist)
self.smoo_Y_hist, _ = kf.filter(self.sampled_Y_hist)
self.smoo_R3_hist, _ = kf.filter(self.sampled_R3_hist)
def kalman(self, ds):
ds = ds.fillna(method='ffill') # fill in NaN's
measurements = ds.to_numpy()
kf = KalmanFilter(transition_matrices=[1],
observation_matrices=[1],
initial_state_mean=measurements[0],
initial_state_covariance=1,
observation_covariance=10,
transition_covariance=1)
state_means, state_covariances = kf.filter(measurements)
state_std = np.sqrt(state_covariances[:, 0]) # in case I needed them
return state_means
class KalmanRegression(object):
"""
Uses a Kalman Filter to estimate regression parameters
in an online fashion.
Estimated model: y ~ beta * x + alpha
"""
def __init__(self, initial_y, initial_x, delta=1e-5, maxlen=3000):
self._x = initial_x.name
self._y = initial_y.name
self.maxlen = maxlen
trans_cov = delta / (1 - delta) * np.eye(2)
obs_mat = np.expand_dims(
np.vstack([[initial_x], [np.ones(initial_x.shape[0])]]).T, axis=1)
self.kf = KalmanFilter(n_dim_obs=1, n_dim_state=2,
initial_state_mean=np.zeros(2),
initial_state_covariance=np.ones((2, 2)),
transition_matrices=np.eye(2),
observation_matrices=obs_mat,
observation_covariance=1.0,
transition_covariance=trans_cov)
state_means, state_covs = self.kf.filter(initial_y.values)
self.means = pd.DataFrame(state_means,
index=initial_y.index,
columns=['beta', 'alpha'])
self.state_cov = state_covs[-1]
def update(self, observations):
x = observations[self._x]
y = observations[self._y]
mu, self.state_cov = self.kf.filter_update(
self.state_mean, self.state_cov, y,
observation_matrix=np.array([[x, 1.0]]))
mu = pd.Series(mu, index=['beta', 'alpha'],
name=observations.name)
self.means = self.means.append(mu)
if self.means.shape[0] > self.maxlen:
self.means = self.means.iloc[-self.maxlen:]
def get_spread(self, observations):
x = observations[self._x]
y = observations[self._y]
return y - (self.means.beta[-1] * x + self.means.alpha[-1])
@property
def state_mean(self):
return self.means.iloc[-1]
def _KalmanFilterRegression( self ): """ Use Kalman Filter to obtain first-order auto-regression parameters r_t = beta_0 + beta_1 * r_(t-1) """ returns = self.returns; _trans_cov_delta = 1e-3; # Transition matrix and covariance trans_mat = np.eye(2); # Assume beta is not to change from t-1 to t _delta = _trans_cov_delta; trans_cov = _delta / (1 - _delta) * np.eye(2); # This _delta and trans_cov seem to have great impact on the result # form Observation Matrix data = returns.values[:-1,:]; _, num_stocks = data.shape; data = np.expand_dims( data, axis = 2 ); # T-by-2-by-1 array obs_mat = np.insert( data, 1, 1, axis = 2 ); # Insert column of ones T-2-2 array obs_cov = np.eye( num_stocks ); # assume zero correlation among noises in observed stock returns #print "Shape of observation matrix is ", obs_mat.shape; #print "Example of obs_mat is ", obs_mat[:2,:,:]; # Observed stock returns: r_t index = returns.index[1:]; # index for beta_1(t) observations = returns.values[1:,:] # matrix of r_t # Form Kalman Filter and then filter! kf = KalmanFilter( n_dim_obs = num_stocks, n_dim_state = 2, # 2 regression parameters initial_state_mean = np.zeros(2), initial_state_covariance = np.ones((2,2)), transition_matrices = trans_mat, transition_covariance = trans_cov, observation_matrices = obs_mat, observation_covariance = obs_cov, ); state_means, state_covs = kf.filter( observations ); slope = pd.Series( state_means[:,0], index ); intercept = pd.Series( state_means[:,1], index ); kf_coefficients_df = pd.DataFrame( [ intercept, slope ], index = [ "coeff_0", "coeff_1" ] ); self.kf_coefficients_df = kf_coefficients_df.transpose(); return (intercept, slope);
def getPredictedValuesVer2(measurements, standard_deviation = None):
#this method use PyKalman module
#Apply this method only for Gains.
if standard_deviation == None:
#if standard deviation is not specified, then get it only from measurements list
standard_deviation = getStandardDeviation(measurements)
#adapt the transition_covariance. Temporary
if standard_deviation < 1e-9:
transition_covariance = 1e-21
else:
transition_covariance = 5e-17
kf = KalmanFilter(transition_matrices = [1], observation_matrices = [1], transition_covariance = transition_covariance, observation_covariance = math.pow(standard_deviation, 2))
tmp= kf.filter(measurements)[0].tolist()
for i in range(0, len(tmp)):
tmp[i] = tmp[i][0]
return tmp
class Regression(object):
"""
y = beta * x + alpha
"""
def __init__(self, initial_x, initial_y, date, delta=1e-5):
trans_cov = delta / (1 - delta) * np.eye(2)
xList = []
for x in initial_x:
xList.append([[x,1.]])
obs_mat = np.vstack([xList])
self.kf = KalmanFilter(n_dim_obs=1, n_dim_state=2,
initial_state_mean=np.zeros(2),
initial_state_covariance=np.ones((2, 2)),
transition_matrices=np.eye(2),
observation_matrices=obs_mat,
observation_covariance=1.0,
transition_covariance=trans_cov)
state_means, state_covs = self.kf.filter(np.array(initial_y))
self.means = pd.DataFrame(state_means,
columns=['beta', 'alpha'])
self.state_cov = state_covs[-1]
def update(self, observations, date):
x = observations[0]
y = observations[1]
mu, self.state_cov = self.kf.filter_update(self.state_mean, self.state_cov, y,
observation_matrix=np.array([[x, 1.0]]))
mu = pd.Series(mu, index=['beta', 'alpha'],
name=date)
self.means = self.means.append(mu)
def get_spread(self, observations):
x = observations[0]
y = observations[1]
return y - (self.means.beta * x + self.means.alpha)
@property
def state_mean(self):
return self.means.iloc[-1]
def kalman_ma(df, transition_covariance=.01):
df_new = pd.DataFrame()
for c in df.columns:
# Construct a Kalman filter
kf = KalmanFilter(transition_matrices = [1],
observation_matrices = [1],
initial_state_mean = df.ix[0,c],
initial_state_covariance = 1,
observation_covariance=1,
transition_covariance=transition_covariance)
# Use the observed values of the price to get a rolling mean
state_means, _ = kf.filter(df[c].values)
df_new[c] = state_means[:,0]
df_new.index = df.index
return df_new
def test_kalman_filter():
kf = KalmanFilter(
data.transition_matrix,
data.observation_matrix,
data.transition_covariance,
data.observation_covariance,
data.transition_offsets,
data.observation_offset,
data.initial_state_mean,
data.initial_state_covariance)
(x_filt, V_filt) = kf.filter(X=data.observations)
assert_array_almost_equal(
x_filt[:500],
data.filtered_state_means[:500],
decimal=7
)
assert_array_almost_equal(
V_filt[:500],
data.filtered_state_covariances[:500],
decimal=7
)
def kalman_filtering(price_sequence):
h = 1 #time step
A = np.array([[1,h,.5*h**2],
[0,1,h],
[0,0,1]])
Q = np.eye(A.shape[0])
#2) Apply the filter
kf = KalmanFilter(transition_matrices = A , transition_covariance = Q)
means, covariances = kf.filter([price_sequence[0]])
filtered_price_sequence = [means[0,0]]
for i in range(1,len(price_sequence)):
#to track it (streaming)
new_mean, new_covariance = kf.filter_update(means[-1], covariances[-1], price_sequence[i])
means = np.vstack([means,new_mean])
covariances = np.vstack([covariances,new_covariance.reshape((1,3,3))])
filtered_price_sequence.append(means[i,0])
return filtered_price_sequence
# n_dim_state = data.transition_matrix.shape[0]
# n_timesteps = data.observations.shape[0]
# blind_state_estimates = np.zeros((n_timesteps, n_dim_state))
# for t in range(n_timesteps - 1):
# if t == 0:
# blind_state_estimates[t] = kf.initial_state_mean
# blind_state_estimates[t + 1] = (
# np.dot(kf.transition_matrices, blind_state_estimates[t])
# + kf.transition_offsets[t]
# )
# Estimate the hidden states using observations up to and including
# time t for t in [0...n_timesteps-1]. This method outputs the mean and
# covariance characterizing the Multivariate Normal distribution for
# P(x_t | z_{1:t})
filtered_state_estimates = kf.filter(data.observations)[0]
# Estimate the hidden states using all observations. These estimates
# will be 'smoother' (and are to be preferred) to those produced by
# simply filtering as they are made with later observations in mind.
# Probabilistically, this method produces the mean and covariance
# characterizing,
# P(x_t | z_{1:n_timesteps})
smoothed_state_estimates = kf.smooth(data.observations)[0]
# Draw the true, blind,e filtered, and smoothed state estimates for all 5
# dimensions.
pl.figure(figsize=(16, 6))
lines_true = pl.plot(data.states, linestyle='-', color='b')
# lines_blind = pl.plot(blind_state_estimates, linestyle=':', color='m')
lines_observations = pl.plot(data.observations, linestyle=':', color='m')
state_cov_multiplier = np.power(0.01, 2) # 0.1: spread_std=2.2, cov=16 ==> 0.01: 0.22, 0.16
observation_cov = 0.001
# observation matrix F is 2-dimensional, containing sym_a price and 1
# there are data.shape[0] observations
obs_mat_F = np.transpose(np.vstack([data[sym_a].values, np.ones(data.shape[0])])).reshape(-1, 1, 2)
kf = KalmanFilter(n_dim_obs=1, # y is 1-dimensional
n_dim_state=2, # states (alpha, beta) is 2-dimensinal
initial_state_mean=np.ones(2), # initial value of intercept and slope theta0|0
initial_state_covariance=np.ones((2, 2)), # initial cov matrix between intercept and slope P0|0
transition_matrices=np.eye(2), # G, constant
observation_matrices=obs_mat_F, # F, depends on x
observation_covariance=observation_cov, # v_t, constant
transition_covariance= np.eye(2)*state_cov_multiplier) # w_t, constant
state_means, state_covs = kf.filter(data[sym_b]) # observes sym_b price
beta_kf = pd.DataFrame({'Slope': state_means[:, 0], 'Intercept': state_means[:, 1]}, index=data.index)
beta_kf.plot(subplots=True)
plt.show()
# ------------------------------------This should be equivalent to above --------------------------------------------#
means_trace = []
covs_trace = []
step = 0
x = data[sym_a][step]
y = data[sym_b][step]
observation_matrix_stepwise = np.array([[x, 1]])
observation_stepwise = y
kf = KalmanFilter(n_dim_obs=1, n_dim_state=2,
initial_state_mean=np.ones(2), # initial value
initial_state_covariance=np.ones((2, 2)), # initial value
observation_offset = 0.0 # d
transition_covariance = 0.05 # Q
observation_covariance = 0.5 # R
initial_state_mean = 0.0
initial_state_covariance = 1.0
# sample from model
kf = KalmanFilter(
transition_matrix, observation_matrix, transition_covariance,
observation_covariance, transition_offset, observation_offset,
initial_state_mean, initial_state_covariance,
)
#kf = kf.em(y, n_iter=5) # Expectation Maximization
y_filtered['pykalman-zero-order'] = kf.filter(y)[0].flatten()
y_filtered['pykalman_smoothed-zero-order'] = kf.smooth(y)[0][:,0]
#--------------------------------------------------------------------------------------------
# Kalman filter 2nd order (mass-spring-damper) (PyKalman)
#--------------------------------------------------------------------------------------------
m = 1000.0 # mass
k_s = 0.05 # spring (the larger, the harder)
k_d = 0.5 # damper (the larger, the harder)
d = k_d / k_s
print("d: " + str(d) + ", Damped: " + str(d > 1))
q = 0.1 # Variance of process noise, i.e. state uncertainty
def main():
#Paramètres du modèle
#Indiquer à quoi correspond chaque variable
osigma=0.1;
transition_matrix = np.array([[1., 0.,0.],[1., 1.,0.],[0.,0,0.9]])
transition_covariance = np.zeros((3,3));
observation_matrix = np.array([[0., 1.,0.],[0., 0.,1.]])
observation_covariance = np.eye(2)*osigma
initial_state_mean = np.array([1,0,10])
initial_state_covariance = np.eye(3);
#Observations
observations=np.array([ [1.1,9.2],
[1.9,8.1],
[2.8,7.2],
[4.2,6.6],
[5.0,5.9],
[6.1,5.32],
[7.2,4.7],
[8.1,4.3],
[9.0,3.9]])
# Filtrage à la main
# Quels sont les paramètres du constructeur ?
kf = KalmanFilter(
transition_matrix, observation_matrix,
transition_covariance, observation_covariance,
)
# Que conserverons les variables suivantes ?
hand_state_estimates=[initial_state_mean]
hand_state_cov_estimates=[initial_state_covariance]
for anObs in observations:
# Quelles étapes du filtrage sont réalisées par la ligne suivante
(aMean,aCov) = kf.filter_update(hand_state_estimates[-1],hand_state_cov_estimates[-1],anObs)
hand_state_estimates.append(aMean)
hand_state_cov_estimates.append(aCov)
hand_state_estimates=np.array(hand_state_estimates)
# A quoi sert la ligne suivante ?
hand_positions=np.dot(hand_state_estimates,observation_matrix.T )
plt.figure(1)
plt.plot(observations[:,0],observations[:,1], 'r+')
plt.plot(hand_positions[:,0],hand_positions[:,1], 'b')
plt.savefig('illustration_filtrage_main.pdf')
plt.close()
# Filtrage complet
# Que fait cette section ?
# Quels sont les paramètres supplémentaires donnés au constructeur ?
# Quels sont les autres paramètres possibles ?
kf = KalmanFilter(
transition_matrix, observation_matrix,
transition_covariance, observation_covariance,
initial_state_mean=initial_state_mean, initial_state_covariance=initial_state_covariance,
)
(filtered_state_estimates,filtered_state_cov_estimates) = kf.filter(observations)
filtered_positions=np.dot(filtered_state_estimates,observation_matrix.T )
plt.figure(1)
plt.plot(observations[:,0],observations[:,1], 'r+')
plt.plot(filtered_positions[:,0],filtered_positions[:,1], 'b')
plt.savefig('illustration_filtrage.pdf')
plt.close()
# Lissage
# Que fait cette section ?
# Quel est la différence avec le filtrage ?
# Puis-je faire un lissage "à la main" observation par observation ?
kf = KalmanFilter(
transition_matrix, observation_matrix,
transition_covariance, observation_covariance,
initial_state_mean=initial_state_mean, initial_state_covariance=initial_state_covariance,
)
(smoothed_state_estimates,smoothed_state_cov_estimates) = kf.smooth(observations)
smoothed_positions=np.dot(smoothed_state_estimates,observation_matrix.T )
plt.figure(2)
plt.plot(observations[:,0],observations[:,1], 'r+')
plt.plot(smoothed_positions[:,0],smoothed_positions[:,1], 'b')
plt.savefig('illustration_lissage.pdf')
plt.close()
from scipy import poly1d
tau = 0.1
# Set up the filter
kf = KalmanFilter(n_dim_obs=1, n_dim_state=2, # position is 1-dimensional, (x,v) is 2-dimensional
initial_state_mean=[30,10],
initial_state_covariance=np.eye(2),
transition_matrices=[[1,tau], [0,1]],
observation_matrices=[[1,0]],
observation_covariance=3,
transition_covariance=np.zeros((2,2)),
transition_offsets=[-4.9*tau**2, -9.8*tau])
# Create a simulation of a ball falling for 40 units of time (each of length tau)
times = np.arange(40)
actual = -4.9*tau**2*times**2
# Simulate the noisy camera data
sim = actual + 3*np.random.randn(40)
# Run filter on camera data
state_means, state_covs = kf.filter(sim)
plt.plot(times, state_means[:,0])
plt.plot(times, sim)
plt.plot(times, actual)
plt.legend(['Filter estimate', 'Camera data', 'Actual'])
plt.xlabel('Time')
plt.ylabel('Height')
plt.show()
obs_mat = np.vstack([measurements, np.ones(measurements.shape)]).T[:, np.newaxis]
delta = 1e-5
trans_cov = delta / (1 - delta) * np.eye(2)
kf = KalmanFilter(transition_matrices = [[1, 1], [0, 1]], observation_matrices = [[0.1, 0.5], [-0.3, 0.0]], initial_state_mean=np.zeros(2))
"""
kf = KalmanFilter(initial_state_mean=np.zeros(2),
initial_state_covariance=np.ones((2, 2)),
transition_matrices=np.eye(2),
#observation_matrices = [[0.1, 0.5], [-0.3, 0.0]],
observation_matrices = [[1., 1.], [3., 1.]],
transition_covariance=trans_cov)
"""
#kf = kf.em(measurements, n_iter=10)
state_means1, state_covs1 = kf.filter(measurements)
#smoothed_state_means, smoothed_state_covariances = kf.smooth(measurements)
print(state_covs1)
pl.scatter(times, state_covs1[:, 0], marker='*', color='b')
pl.show()
"""
plt.figure(1)
times = range(measurements.shape[0])
obs_scatter = pl.scatter(times, measurements[:, 0], marker='*', color='b', label='Measurement of Regfuge 1')
position_line = pl.plot(times, measurements[:, 1], marker='8', color='r', label='Measurement of Regfuge 2')
velocity_line = pl.plot(times, smoothed_state_means[:, 0], linestyle='-', marker='_', color='g', label='Smoothing mean value')
#velocity_line2 = pl.plot(times, smoothed_state_means[:, 2], linestyle='-', marker='_', color='m', label='Smoothing mean value 2')
pl.legend(loc='lower right')
def analysis_Kalman(countries_num, type):
with open("D3-data-file-refugee-1.csv") as csvfile:
reader = csvfile.readlines()
skipline = 0
line_ = 0
country_name = ""
date_limit = 0
c = 0
delta = 1e-5
trans_cov = delta / (1 - delta) * np.eye(2)
for row in reader:
if skipline > 1:
if date_limit < 20:
""
x_dat.append(row.split("\t")[0])
t_s__.append(c)
tmp_kal_val__.append(int(row.split("\t")[index_number]))
y_dat.append(int(row.split("\t")[index_number]))
c += 1
"""
if date_limit > 20 and date_limit <= 40:
x_dat_1.append(row.split("\t")[0])
tmp_kal_val__.append(int(row.split("\t")[index_number]))
y_dat_1.append(int(row.split("\t")[index_number]))
"""
else:
try:
country_name = row.split("\t")[1].split(",")[index_number - 1]
except:
""
skipline += 1
date_limit += 1
kf = KalmanFilter(transition_matrices=[1],
observation_matrices=[1],
initial_state_mean=y_dat[0],
initial_state_covariance=1,
observation_covariance=1,
random_state=0)
TOOLS = "pan,wheel_zoom,box_zoom,reset,save"
times = range(y_dat[index_number])
p = figure(plot_width=1000, toolbar_location="right", y_axis_label = "Refuges",
x_axis_label = "Weekly")
#state_means, _ = kf.em(np.asarray(y_dat)).smooth(np.asarray(y_dat)) #.filter(y_dat)
#state_means = state_means.flatten()
state_means1, _ = kf.filter(np.asarray(y_dat))
state_means1 = state_means1.flatten()
#MSE = stats.linregress(t_s__, state_means)
#MSE1 = stats.linregress(t_s__, state_means1)
#print(MSE, MSE1)
#print(state_means)
#np.savetxt("trained_dataset_Syria_15.txt", state_means, delimiter=',')
#print(len(x_dat), len(state_means))
data_1 = []
data_2 = []
tmp_cov = []
tmp_cov_1 = []
with open("trained_dataset_Syria_15.txt")as train_data:
data__ex = train_data.readlines()
for j in data__ex:
tmp_cov.append(j)
with open("trained_dataset_Syria_14.txt")as train_data:
data__ex = train_data.readlines()
for j in data__ex:
tmp_cov_1.append(j)
index_ = 0
for kk in tmp_cov:
data_1.append(kk)
data_2.append(tmp_cov_1[index_])
index_ += 1
#slope, intercept, rvalue, pvalue, stderr = stats.linregress(np.array(data_1).astype(np.float), np.array(data_2).astype(np.float))
#print (stderr)
p = figure(x_range = x_dat, plot_width=1000, toolbar_location="right",y_axis_label = "Refuges moving through "+country_type +" country("+ country_name +")",
x_axis_label = "Weeks")
#p.line(x_dat, state_means, line_width=1, line_color = 'blue', legend="Kalman filter 0")
p.line(x_dat, y_dat, line_width=1, line_color = 'green', legend="Refuge from Syria to destination country ")
p.line(x_dat, state_means1, line_width=1, line_color = 'red', legend="Kalman filter 1")
#slope, intercept, rvalue, pvalue, stderr = stats.linregress(np.array(y_dat).astype(np.float), np.array(state_means1).astype(np.float))
slope, intercept, rvalue, pvalue, stderr = stats.linregress(t_s__, np.array(state_means1).astype(np.float))
print(stderr)
print statistics.pvariance(state_means1)
#show(p)
"""
def main():
#Paramètres du modèle
#Indiquer à quoi correspond chaque variable
osigma=0.1;
transition_matrix = np.array([[1., 0.,0.,0.,0.,0.],[1., 1.,0.,0.,0.,0.],[0.,0,1,0.,0.,0.]])
transition_covariance = np.zeros((6,6));
observation_matrix = np.array([[0., 1.,0.],[0., 0.,1.]])
observation_covariance = np.eye(2)*osigma
initial_state_mean = np.array([1,0,10]) # m(0)
initial_state_covariance = np.eye(3);# p(0)
#Observations
observations = csv.reader(open('voitureObservations.csv','r'),delimiter=' ')
# Filtrage à la main
# Quels sont les paramètres du constructeur ?
kf = KalmanFilter(
transition_matrix, observation_matrix,
transition_covariance, observation_covariance,
)
# Que conserverons les variables suivantes ?
hand_state_estimates=[initial_state_mean]
hand_state_cov_estimates=[initial_state_covariance]
for anObs in observations:
# Quelles étapes du filtrage sont réalisées par la ligne suivante
(aMean,aCov) = kf.filter_update(hand_state_estimates[-1],hand_state_cov_estimates[-1],anObs)
hand_state_estimates.append(aMean)
hand_state_cov_estimates.append(aCov)
hand_state_estimates=np.array(hand_state_estimates)
# A quoi sert la ligne suivante ?
hand_positions=np.dot(hand_state_estimates,observation_matrix.T ) # obtenir les observation (H*Mk-1 dans le cours, rouge) projet mon espace etat ver mon espace observation
plt.figure(1)
plt.plot(observations[:,0],observations[:,1], 'r+')
plt.plot(hand_positions[:,0],hand_positions[:,1], 'b')
plt.savefig('illustration_filtrage_main_voiture.pdf')
plt.close()
# Filtrage complet
# Que fait cette section ?
# Quels sont les paramètres supplémentaires donnés au constructeur ?
# Quels sont les autres paramètres possibles ?
kf = KalmanFilter(
transition_matrix, observation_matrix,
transition_covariance, observation_covariance,
initial_state_mean=initial_state_mean, initial_state_covariance=initial_state_covariance,
)
(filtered_state_estimates,filtered_state_cov_estimates) = kf.filter(observations)
filtered_positions=np.dot(filtered_state_estimates,observation_matrix.T )
plt.figure(1)
plt.plot(observations[:,0],observations[:,1], 'r+')
plt.plot(filtered_positions[:,0],filtered_positions[:,1], 'b')
plt.savefig('illustration_filtrage_voiture.pdf')
plt.close()
# Lissage
# Que fait cette section ?
# Quel est la différence avec le filtrage ?
# Puis-je faire un lissage "à la main" observation par observation ?
kf = KalmanFilter(
transition_matrix, observation_matrix,
transition_covariance, observation_covariance,
initial_state_mean=initial_state_mean, initial_state_covariance=initial_state_covariance,
)
(smoothed_state_estimates,smoothed_state_cov_estimates) = kf.smooth(observations) # au lieu de filtre, on appel smooth
smoothed_positions=np.dot(smoothed_state_estimates,observation_matrix.T )
plt.figure(2)
plt.plot(observations[:,0],observations[:,1], 'r+')
plt.plot(smoothed_positions[:,0],smoothed_positions[:,1], 'b')
plt.savefig('illustration_lissage_voiture.pdf')
plt.close()
class KalmanFilterPairsTradingStrategy(StrategyBase):
"""
MeanReversionSpreadStrategy
"""
def __init__(
self, events_engine, data_board
):
super(KalmanFilterPairsTradingStrategy, self).__init__(events_engine, data_board)
self.symbols = ['EWA US Equity', 'EWC US Equity']
self.bollinger_scaler = 1.0
self.state_cov_multiplier = np.power(0.01, 2) # 0.1: spread_std=2.2, cov=16 ==> 0.01: 0.22, 0.16
self.observation_cov = 0.001
self.current_ewa_size = 0
self.current_ewc_size = 0
self._kf = None # Kalman Filter
self._current_state_means = None
self._current_state_covs = None
def on_bar(self, bar_event):
print(bar_event.bar_start_time)
A = self._data_board.get_hist_price(self.symbols[0], bar_event.bar_start_time)
B = self._data_board.get_hist_price(self.symbols[1], bar_event.bar_start_time)
x = A['Price'].iloc[-1]
y = B['Price'].iloc[-1]
observation_matrix_stepwise = np.array([[x, 1]])
observation_stepwise = y
spread = None
spread_std = None
if self._kf is None:
self._kf = KalmanFilter(n_dim_obs=1, n_dim_state=2,
initial_state_mean=np.ones(2), # initial value
initial_state_covariance=np.ones((2, 2)), # initial value
transition_matrices=np.eye(2), # constant
observation_matrices=observation_matrix_stepwise, # depend on x
observation_covariance=self.observation_cov, # constant
transition_covariance=np.eye(2) * self.state_cov_multiplier) # constant
P = np.ones((2, 2)) + np.eye(2)*self.state_cov_multiplier
spread = y - observation_matrix_stepwise.dot(np.ones(2))[0]
spread_std = np.sqrt(observation_matrix_stepwise.dot(P).dot(observation_matrix_stepwise.transpose())[0][0] + self.observation_cov)
state_means_stepwise, state_covs_stepwise = self._kf.filter(observation_stepwise) # depend on y
self._current_state_means = state_means_stepwise[0]
self._current_state_covs = state_covs_stepwise[0]
else:
state_means_stepwise, state_covs_stepwise = self._kf.filter_update(
self._current_state_means, self._current_state_covs,
observation=observation_stepwise,
observation_matrix=observation_matrix_stepwise)
P = self._current_state_covs + np.eye(2)*self.state_cov_multiplier # This has to be small enough
spread = y - observation_matrix_stepwise.dot(self._current_state_means)[0]
spread_std = np.sqrt(observation_matrix_stepwise.dot(P).dot(observation_matrix_stepwise.transpose())[0][0] + self.observation_cov)
self._current_state_means = state_means_stepwise
self._current_state_covs = state_covs_stepwise
# residual is assumed to be N(0, spread_std)
bollinger_ub = self.bollinger_scaler * spread_std
bollinger_lb = - self.bollinger_scaler * spread_std
coeff = self._current_state_means[0]
print(spread, spread_std)
if (spread > bollinger_ub) and (self.current_ewa_size >= 0):
print ('Hit upper band, short spread.')
o = OrderEvent()
o.full_symbol = self.symbols[0]
o.order_type = OrderType.MARKET
o.order_size = int(-1000*coeff) - self.current_ewa_size
self.place_order(o)
self.current_ewa_size = int(-1000*coeff)
o = OrderEvent()
o.full_symbol = self.symbols[1]
o.order_type = OrderType.MARKET
o.order_size = 1000 - self.current_ewc_size
self.place_order(o)
self.current_ewc_size = 1000
elif (spread < bollinger_lb) and (self.current_ewa_size <= 0):
print('Hit lower band, long spread.')
o = OrderEvent()
o.full_symbol = self.symbols[0]
o.order_type = OrderType.MARKET
o.order_size = int(1000 * coeff) - self.current_ewa_size
self.place_order(o)
self.current_ewa_size = int(1000 * coeff)
o = OrderEvent()
o.full_symbol = self.symbols[1]
o.order_type = OrderType.MARKET
o.order_size = -1000 - self.current_ewc_size
self.place_order(o)
self.current_ewc_size = -1000
elif (spread < 0) and (spread > bollinger_lb) and (self.current_ewa_size < 0):
print('spread crosses below average.cover short spread')
o = OrderEvent()
o.full_symbol = self.symbols[0]
o.order_type = OrderType.MARKET
o.order_size = - self.current_ewa_size
self.place_order(o)
self.current_ewa_size = 0
o = OrderEvent()
o.full_symbol = self.symbols[1]
o.order_type = OrderType.MARKET
o.order_size = - self.current_ewc_size
self.place_order(o)
self.current_ewc_size = 0
elif (spread > 0) and (spread < bollinger_ub) and (self.current_ewa_size > 0):
print('spread crosses above average.cover long spread')
o = OrderEvent()
o.full_symbol = self.symbols[0]
o.order_type = OrderType.MARKET
o.order_size = - self.current_ewa_size
self.place_order(o)
self.current_ewa_size = 0
o = OrderEvent()
o.full_symbol = self.symbols[1]
o.order_type = OrderType.MARKET
o.order_size = - self.current_ewc_size
self.place_order(o)
self.current_ewc_size = 0
from pykalman import KalmanFilter import numpy as np kf = KalmanFilter(transition_matrices = [[1, 1], [0, 1]], observation_matrices = [[0.1, 0.5], [-0.3, 0.0]]) measurements = np.asarray([[1,0], [0,0], [0,1]]) # 3 observations kf = kf.em(measurements, n_iter=5) (filtered_state_means, filtered_state_covariances) = kf.filter(measurements) print filtered_state_means
# sample from model
kf = KalmanFilter(
transition_matrix,
observation_matrix,
transition_covariance,
observation_covariance,
transition_offset,
observation_offset,
initial_state_mean,
initial_state_covariance,
)
# kf = kf.em(y, n_iter=5)
taxels = tsframes3D[y, x, :]
tsframes3D_kalman[y, x, :] = np.round(kf.filter(taxels)[0].flatten())
kalman = tsframes3D_kalman[:, :, frameID].astype(np.float32)
############################
# Spatio-Temporal Filtering
############################
# ---------------
# Kalman + Median filter
# ---------------
kalman_median = cv2.medianBlur(kalman, d)
masked = ma.masked_greater(kalman_median, 0) # Masked where result of filter > 0
kalman_median_masked = np.copy(tsframe)
kalman_median_masked[~masked.mask] = 0
def speakerTracker(self):
# Clean up
cv2.destroyAllWindows()
if self.inputPath is not None:
# If a path to a file was given, assume it is a single video file
if os.path.isfile(self.inputPath):
cap = cv2.VideoCapture(self.inputPath)
clip = VideoFileClip(self.inputPath,audio=False)
fps = cap.get(cv2.cv.CV_CAP_PROP_FPS)
self.numFrames = cap.get(cv2.cv.CV_CAP_PROP_FRAME_COUNT)
print "[speakerTracker] Number of frames" , self.numFrames
pathBase = os.path.basename(self.inputPath)
pathDirectory = os.path.dirname(self.inputPath)
baseName = pathDirectory + '/' + os.path.splitext(pathBase)[0] + '_'
#Skip first frames if required
if self.skip is not None:
cap.set(cv2.cv.CV_CAP_PROP_POS_FRAMES, self.skip)
# Otherwise assume it is a format string for reading images
else:
cap = cmtutil.FileVideoCapture(self.inputPath)
#Skip first frames if required
if self.skip is not None:
cap.frame = 1 + self.skip
else:
# If no input path was specified, open camera device
sys.exit("[speakerTracker] Error: no input path was specified")
# The number of pixels from the center of object till the border of cropped image
marginPixels = 300
# Read first frame
status, im0 = cap.read()
imGray0 = cv2.cvtColor(im0, cv2.COLOR_BGR2GRAY)
imDraw = np.copy(im0)
(tl0, br0) = cmtutil.get_rect(imDraw)
print '[speakerTracker] Using', tl0, br0, 'as initial bounding box for the speaker'
# First initialization to get the center
self.CMT.initialise(imGray0, tl0, br0)
# The first x and y coordinates of the object of interest
self.inity = tl0[1] - self.CMT.center_to_tl[1]
self.initx = tl0[0] - self.CMT.center_to_tl[0]
# Crop the first frame
imGray0_initial = imGray0[self.inity - marginPixels : self.inity + marginPixels,
self.initx - marginPixels : self.initx + marginPixels]
# Calculate the translation vector from main image to the cropped frame
self.originFromMainImageY = self.inity - marginPixels
self.originFromMainImageX = self.initx - marginPixels
# Calculate the position of the selected rectangle in the cropped frame
tl = (tl0[0] - self.originFromMainImageX , tl0[1] - self.originFromMainImageY)
br = (br0[0] - self.originFromMainImageX , br0[1] - self.originFromMainImageY)
#print '[speakerTracker] Using', tl, br, 'as initial bounding box for the speaker'
# initialization and keypoint calculation
self.CMT.initialise(imGray0_initial, tl, br)
# Center of object in cropped frame
self.currentY = tl[1] - self.CMT.center_to_tl[1]
self.currentX = tl[0] - self.CMT.center_to_tl[0]
# Center of object in main frame
self.currentYMainImage = self.currentY + self.originFromMainImageY
self.currentXMainImage = self.currentX + self.originFromMainImageX
measuredTrack=np.zeros((self.numFrames+10,2))-1
count =0
# loop to read all frames,
# crop them with the center of last frame,
# calculate keypoints and center of the object
for frame in clip.iter_frames():
# Read the frame and convert it to gray scale
im_gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
# Corner correction (Height)
if (self.currentYMainImage + marginPixels >= im_gray.shape[0]):
self.currentYMainImage = im_gray.shape[0] - marginPixels -1
else:
self.currentYMainImage = self.currentYMainImage
if (self.currentXMainImage + marginPixels >= im_gray.shape[1]):
self.currentXMainImage = im_gray.shape[1] - marginPixels -1
else:
self.currentXMainImage = self.currentXMainImage
if (self.currentYMainImage - marginPixels <= 0):
self.currentYMainImage = 0 + marginPixels +1
else:
self.currentYMainImage = self.currentYMainImage
if (self.currentXMainImage - marginPixels <= 0):
self.currentXMainImage = 0 + marginPixels +1
else:
self.currentXMainImage = self.currentXMainImage
# Crop it by previous coordinates
im_gray_crop = im_gray[self.currentYMainImage - marginPixels : self.currentYMainImage + marginPixels,
self.currentXMainImage - marginPixels : self.currentXMainImage + marginPixels]
#plt.imshow(im_gray_crop, cmap = cm.Greys_r)
#plt.show()
#print "self.currentYMainImage:", self.currentYMainImage
#print "self.currentXMainImage:", self.currentXMainImage
#print im_gray_crop.shape
# Compute all keypoints in the cropped frame
self.CMT.process_frame(im_gray_crop)
#print 'frame: {2:4d}, Center: {0:.2f},{1:.2f}'.format(self.CMT.center[0], self.CMT.center[1] , count)
if not (math.isnan(self.CMT.center[0]) or math.isnan(self.CMT.center[1])
or (self.CMT.center[0] <= 0) or (self.CMT.center[1] <= 0)):
# Compute the center of the object with respect to the main image
self.diffY = self.CMT.center[0] - self.currentY
self.diffX = self.CMT.center[1] - self.currentX
self.currentYMainImage = self.diffY + self.currentYMainImage
self.currentXMainImage = self.diffX + self.currentXMainImage
self.currentY = self.CMT.center[0]
self.currentX = self.CMT.center[1]
# Save the center of frames in an array for further process
measuredTrack[count,0] = self.currentYMainImage
measuredTrack[count,1] = self.currentXMainImage
else:
self.currentYMainImage = self.currentYMainImage
self.currentXMainImage = self.currentXMainImage
print 'frame: {2:4d}, Center: {0:.2f},{1:.2f}'.format(self.currentYMainImage, self.currentXMainImage , count)
count += 1
numMeas=measuredTrack.shape[0]
markedMeasure=np.ma.masked_less(measuredTrack,0)
# Kalman Filter Parameters
deltaT = 1.0/clip.fps
transitionMatrix=[[1,0,deltaT,0],[0,1,0,deltaT],[0,0,1,0],[0,0,0,1]] #A
observationMatrix=[[1,0,0,0],[0,1,0,0]] #C
xinit = markedMeasure[0,0]
yinit = markedMeasure[0,1]
vxinit = markedMeasure[1,0]-markedMeasure[0,0]
vyinit = markedMeasure[1,1]-markedMeasure[0,1]
initState = [xinit,yinit,vxinit,vyinit] #mu0
initCovariance = 1.0e-3*np.eye(4) #sigma0
transistionCov = 1.0e-4*np.eye(4) #Q
observationCov = 1.0e-1*np.eye(2) #R
# Kalman Filter bias
kf = KalmanFilter(transition_matrices = transitionMatrix,
observation_matrices = observationMatrix,
initial_state_mean = initState,
initial_state_covariance = initCovariance,
transition_covariance = transistionCov,
observation_covariance = observationCov)
self.measuredTrack = measuredTrack
# Kalman Filter
(self.filteredStateMeans, self.filteredStateCovariances) = kf.filter(markedMeasure)
# Kalman Smoother
(self.filterStateMeanSmooth, self.filterStateCovariancesSmooth) = kf.smooth(markedMeasure)
newClip = clip.fl_image( self.crop )
return newClip
class KalmanRegression(object):
def __init__(self, df, dependent=None, independent=None, delta=None, trans_cov=None, obs_cov=None):
if not dependent:
dependent = df.columns[1]
if not independent:
independent = df.columns[2]
self.x = df[independent]
self.x.index = df.index
self.y = df[dependent]
self.y.index = df.index
self.dependent = dependent
self.independent = independent
self.delta = delta or 1e-5
self.trans_cov = trans_cov or self.delta / (1 - self.delta) * np.eye(2)
self.obs_mat = np.expand_dims(
np.vstack([[self.x.values], [np.ones(len(self.x))]]).T,
axis=1
)
self.obs_cov = obs_cov or 1
self.kf = KalmanFilter(n_dim_obs=1, n_dim_state=2,
initial_state_mean=np.zeros(2),
initial_state_covariance=np.ones((2, 2)),
transition_matrices=np.eye(2),
observation_matrices=self.obs_mat,
observation_covariance=self.obs_cov,
transition_covariance=self.trans_cov)
self.state_means, self.state_covs = self.kf.filter(self.y.values)
def slope(self):
state_means = self.state_means
return pd.Series(state_means[:, 0], index=self.x.index)
def plot_params(self):
state_means = self.state_means
x = self.x
_, axarr = plt.subplots(2, sharex=True)
axarr[0].plot(x.index, state_means[:, 0], label='slope')
axarr[0].legend()
axarr[1].plot(x.index, state_means[:, 1], label='intercept')
axarr[1].legend()
plt.tight_layout()
plt.show()
return state_means[:, 0]
def plot2D(self):
x = self.x
y = self.y
state_means = self.state_means
cm = plt.get_cmap('jet')
colors = np.linspace(0.1, 1, len(x))
# Plot data points using colormap
sc = plt.scatter(x, y, s=30, c=colors, cmap=cm, edgecolor='k', alpha=0.7)
cb = plt.colorbar(sc)
cb.ax.set_yticklabels([str(p.date()) for p in x[::len(x) // 9].index])
# Plot every fifth line
step = 100
xi = np.linspace(x.min() - 5, x.max() + 5, 2)
colors_l = np.linspace(0.1, 1, len(state_means[::step]))
for i, beta in enumerate(state_means[::step]):
plt.plot(xi, beta[0] * xi + beta[1], alpha=.2, lw=1, c=cm(colors_l[i]))
# Plot the OLS regression line
plt.plot(xi, poly1d(np.polyfit(x, y, 1))(xi), '0.4')
plt.title(self.dependent + ' vs. ' + self.independent)
plt.show()
def run_all(self):
self.plot_params()
self.plot2D()
# create a Kalman Filter by hinting at the size of the state and observation space.
# If you already have good guesses for the initial parameters, put them in here.
# The Kalman filter will try to learn the values of all variables
# The Kalman Filter is parameterized by 3 arrays for state transitions, 3 for measurements, and 2 more for initial conditions.
## This worked
kf = KalmanFilter(transition_matrices=transition_matrix,
observation_matrices=observation_matix,
transition_covariance=0.01 * np.eye(3)
)
kf = kf.em(X, n_iter=5)
(filtered_state_means, filtered_state_covariances) = kf.filter(X)
(smoothed_state_means, smoothed_state_covariances) = kf.smooth(X)
# kf.em(X).smooth([X]) ## this gives the shape error
kf = kf.em(X).smooth(X) # This returns large tuple
# print len(kf_test)
# Plot lines for the observations without noise, the estimated position of the
# target before fitting, and the estimated position after fitting.
states_pred = kf
n_timesteps = X.shape[0]
x = np.linspace(0, 3 * np.pi, n_timesteps)
xobservations = 20 * (np.sin(x) + 0.5 * rnd.randn(n_timesteps))
#Kalman Filter
tau = 0.1
kf = KalmanFilter(n_dim_obs=1,n_dim_state=2,
initial_state_mean=starting_point,
initial_state_covariance=np.eye(2),
transition_matrices=[[1,tau],[0,1]],
observation_matrices=[[1,0]],
observation_covariance=3,
transition_covariance=np.zeros((2,2)),
transition_offsets=[0,0])
np_data = tst_data["10y Close"].values
state_means, state_covs = kf.filter(np_data)
#tst_data['kf_predict']=pd.Series(state_means[:,0])
#print(state_means.shape)
times = np.arange(tst_data.shape[0])
plt.plot(times, state_means[:,0])
plt.plot(times, tst_data["10y Close"])
#plt.show()
tst_data['KF_Value']=state_means[:,0]
print("tst data",tst_data)
WRITE_PATH = "L:\Trade_Output.xlsx"
from pylab import *
from pykalman import KalmanFilter
kf = KalmanFilter();
data = loadtxt('data_gps1.txt');
lat = radians(data[0,:]);
lon = radians(data[1,:]);
dlat = lat[1:] - lat[0:-1];
dlon = lon[1:] - lon[0:-1];
R = 6373; #Radius of earth in kilometers
a = (sin(dlat/2))**2 + cos(lat[0:-1])*cos(lat[1:])*((sin(dlon/2))**2);
c = 2*arctan2(sqrt(a),sqrt(1-a));
d = R * c;
kf = kf.em(d,n_iter = 5);
(smoothed_means,smoothed_cov) = kf.smooth(d);
(filtered_means,filtered_cov) = kf.filter(d);
plot(d,label = 'Original Data');
plot(smoothed_means,label = 'Kalman Smoothed');
plot(filtered_means,label = 'Kalman Filtered');
legend();
show();
def speakerTracker(self):
# Clean up
cv2.destroyAllWindows()
if self.inputPath is not None:
# If a path to a file was given, assume it is a single video file
if os.path.isfile(self.inputPath):
cap = cv2.VideoCapture(self.inputPath)
clip = VideoFileClip(self.inputPath,audio=False)
fps = cap.get(cv2.cv.CV_CAP_PROP_FPS)
self.numFrames = cap.get(cv2.cv.CV_CAP_PROP_FRAME_COUNT)
print "[speakerTracker] Number of frames" , self.numFrames
pathBase = os.path.basename(self.inputPath)
pathDirectory = os.path.dirname(self.inputPath)
baseName = pathDirectory + '/' + os.path.splitext(pathBase)[0] + '_'
# Otherwise assume it is a format string for reading images
else:
cap = cmtutil.FileVideoCapture(self.inputPath)
else:
# If no input path was specified, open camera device
sys.exit("[speakerTracker] Error: no input path was specified")
# Read first frame
status, im0 = cap.read()
imGray0 = cv2.cvtColor(im0, cv2.COLOR_BGR2GRAY)
imDraw = np.copy(im0)
(tl, br) = cmtutil.get_rect(imDraw)
print '[speakerTrackering] Using', tl, br, 'as initial bounding box for the speaker'
self.CMT.initialise(imGray0, tl, br)
#self.inity = tl[1] - self.CMT.center_to_tl[1]
measuredTrack=np.zeros((self.numFrames+10,2))-1
count =0
for frame in clip.iter_frames():
im_gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
self.CMT.process_frame(im_gray)
print 'frame: {2:4d}, Center: {0:.2f},{1:.2f}'.format(self.CMT.center[0], self.CMT.center[1] , count)
#print self.inity
if not (math.isnan(self.CMT.center[0]) or (self.CMT.center[0] <= 0)):
measuredTrack[count,0] = self.CMT.center[0]
measuredTrack[count,1] = self.CMT.center[1]
count += 1
numMeas=measuredTrack.shape[0]
markedMeasure=np.ma.masked_less(measuredTrack,0)
# Kalman Filter Parameters
deltaT = 1.0/clip.fps
transitionMatrix=[[1,0,deltaT,0],[0,1,0,deltaT],[0,0,1,0],[0,0,0,1]] #A
observationMatrix=[[1,0,0,0],[0,1,0,0]] #C
xinit = markedMeasure[0,0]
yinit = markedMeasure[0,1]
vxinit = markedMeasure[1,0]-markedMeasure[0,0]
vyinit = markedMeasure[1,1]-markedMeasure[0,1]
initState = [xinit,yinit,vxinit,vyinit] #mu0
initCovariance = 1.0e-3*np.eye(4) #sigma0
transistionCov = 1.0e-4*np.eye(4) #Q
observationCov = 1.0e-1*np.eye(2) #R
kf = KalmanFilter(transition_matrices = transitionMatrix,
observation_matrices = observationMatrix,
initial_state_mean = initState,
initial_state_covariance = initCovariance,
transition_covariance = transistionCov,
observation_covariance = observationCov)
self.measuredTrack = measuredTrack
(self.filteredStateMeans, self.filteredStateCovariances) = kf.filter(markedMeasure)
(self.filterStateMeanSmooth, self.filterStateCovariancesSmooth) = kf.smooth(markedMeasure)
self.inity = np.mean(self.filterStateMeanSmooth[:][1], axis=0)
newClip = clip.fl_image( self.crop )
return newClip
def pykalman_test():
kf = KalmanFilter(transition_matrices = [[1, 1], [0, 1]], observation_matrices = [[0.1, 0.5], [-0.3, 0.0]])
measurements = numpy.asarray([[1,0], [0,0], [0,1]]) # 3 observations
kf = kf.em(measurements, n_iter=5)
print "pykalman test >>>", kf.filter(measurements)
# Initialize filter
kf = KalmanFilter(transition_matrices = [[1,0],[0,1]],
observation_matrices = [[1,0],[0,1]])
image = numpy.ones([480, 640, 3]) * 128
# First two (2) measurements (initializes variables)
measurements = numpy.ma.asarray(UV) # masking enabled!
measurements = numpy.ma.concatenate((numpy.ma.asarray([[0,0],[0,0]]),
measurements))
measurements[0:2] = numpy.ma.masked
measurements[::2] = numpy.ma.masked
measurements[100:110] = numpy.ma.masked
measurements[200:210] = numpy.ma.masked
measurements[300:310] = numpy.ma.masked
measurements[400:410] = numpy.ma.masked
mu, sig = kf.filter(measurements[0:2]) # initialize on bogus data
print measurements
print mu
print sig
print ''
#t = 0
mu = mu[-1]
sig = sig[-1]
for i in range(2, len(measurements)): # starting with third measurement...
if rospy.is_shutdown(): # easy interruption
break
t = i * 1/10.
#t += 1/10.
