def kalman(train):
# define a Kalman filter model with a weekly cycle, parametrized by a simple "smoothing factor", s
smoothing_factor = 2
n_seasons = 7
# --- define state transition matrix A
state_transition = np.zeros((n_seasons+1, n_seasons+1))
# hidden level
state_transition[0,0] = 1
# season cycle
state_transition[1,1:-1] = [-1.0] * (n_seasons-1)
state_transition[2:,1:-1] = np.eye(n_seasons-1)
# --- observation model H
# observation is hidden level + weekly seasonal compoenent
observation_model = [[1,1] + [0]*(n_seasons-1)]
# --- noise models, parametrized by the smoothing factor
level_noise = 0.2 / smoothing_factor
observation_noise = 0.2
season_noise = 1e-3
process_noise_cov = np.diag([level_noise, season_noise] + [0]*(n_seasons-1))**2
observation_noise_cov = observation_noise**2
kf = simdkalman.KalmanFilter(
state_transition,
process_noise_cov,
observation_model,
observation_noise_cov)
result = kf.compute(train, 10)
return result.smoothed.states.mean[:,:,0]
def test_one_dimensional(self):
training_matrix = range(10)
kf = simdkalman.KalmanFilter(
state_transition = np.eye(2),
process_noise = 0.1,
observation_model = np.array([[1,1]]),
observation_noise = 0.1)
r = kf.compute(
training_matrix,
n_test = 4,
initial_value = [0,0],
initial_covariance = 1.0,
smoothed = True,
gains = True,
log_likelihood = True)
self.assertSequenceEqual(r.predicted.observations.mean.shape, (4,))
self.assertSequenceEqual(r.smoothed.observations.mean.shape, (10,))
self.assertSequenceEqual(r.predicted.states.mean.shape, (4,2))
self.assertSequenceEqual(r.smoothed.states.mean.shape, (10,2))
self.assertSequenceEqual(r.predicted.states.cov.shape, (4,2,2))
self.assertSequenceEqual(r.smoothed.states.cov.shape, (10,2,2))
kf_e = kf.em(training_matrix, n_iter=5, verbose=False)
self.assertSequenceEqual(kf_e.process_noise.shape, (1,2,2))
def test_4_dimensional_observations(self):
training_matrix = np.ones((5, 10, 4))
training_matrix[1, 1, 1] = np.nan
kf = simdkalman.KalmanFilter(state_transition=np.eye(3),
process_noise=0.1,
observation_model=np.array([[1, 1, 0],
[0, 0, 1],
[0, 1, 1],
[1, 1, 0]]),
observation_noise=np.eye(4))
r = kf.compute(training_matrix,
15,
gains=True,
filtered=True,
smoothed=True,
log_likelihood=True)
self.assertSequenceEqual(r.smoothed.observations.mean.shape,
training_matrix.shape)
self.assertSequenceEqual(r.smoothed.states.mean.shape, (5, 10, 3))
self.assertSequenceEqual(r.smoothed.states.cov.shape, (5, 10, 3, 3))
self.assertSequenceEqual(r.smoothed.gains.shape, (5, 10, 3, 3))
self.assertSequenceEqual(r.filtered.gains.shape, (5, 10, 3, 4))
self.assertSequenceEqual(r.predicted.observations.mean.shape,
(5, 15, 4))
self.assertSequenceEqual(r.predicted.observations.cov.shape,
(5, 15, 4, 4))
def test_train_and_predict_vectorized_kalman_filter_2_states(self):
training_matrix = np.ones((5, 10))
kf = simdkalman.KalmanFilter(state_transition=np.eye(2),
process_noise=0.1,
observation_model=np.array([[1, 1]]),
observation_noise=0.1)
r = kf.compute(training_matrix,
n_test=4,
initial_covariance=1.0,
smoothed=True,
gains=True,
log_likelihood=True)
self.assertSequenceEqual(r.predicted.observations.mean.shape, (5, 4))
self.assertSequenceEqual(r.smoothed.observations.mean.shape,
training_matrix.shape)
self.assertSequenceEqual(r.predicted.states.mean.shape, (5, 4, 2))
self.assertSequenceEqual(r.smoothed.states.mean.shape, (5, 10, 2))
self.assertSequenceEqual(r.predicted.states.cov.shape, (5, 4, 2, 2))
self.assertSequenceEqual(r.smoothed.states.cov.shape, (5, 10, 2, 2))
self.assertMatrixEqual(r.log_likelihood, np.array([3.792] * 5), 1e-2)
A = kf.em_process_noise(r)
self.assertSequenceEqual(A.shape, (5, 2, 2))
A0 = A[0, ...]
self.assertMatrixEqual(A0, A0.T)
self.assertTrue(min(np.linalg.eig(A0)[0]) > 0)
B = kf.em_observation_noise(r, training_matrix)
self.assertSequenceEqual(B.shape, (5, 1, 1))
self.assertTrue(min(list(B)) > 0)
def compute_simd():
kf = simdkalman.KalmanFilter(state_transition, process_noise,
observation_model, observation_noise)
r = kf.smooth(data,
initial_value=initial_values[..., np.newaxis],
initial_covariance=initial_covariance,
observations=False)
return (r.states.mean, r.states.cov)
def test_predict_helper_ema(self):
training_matrix = np.ones((5, 10))
kf = simdkalman.KalmanFilter(state_transition=1,
process_noise=0.1,
observation_model=1,
observation_noise=0.1)
r = kf.predict(training_matrix, n_test=4)
self.assertSequenceEqual(r.observations.mean.shape, (5, 4))
self.assertSequenceEqual(r.states.mean.shape, (5, 4, 1))
self.assertSequenceEqual(r.states.cov.shape, (5, 4, 1, 1))
def create_linear_kalman_filter_1d(dt: float,
discrete_white_noise_var: float,
r: float) -> simdkalman.KalmanFilter:
"""Create a simdkalman filter."""
# dynamic (process) model - 1D equation of motion
f = np.array([[1, dt, 0.5 * dt * dt], [0, 1, dt], [0, 0, 1]])
# assume piecewise discrete white noise for the process (dynamic) model
q = Q_discrete_white_noise(dim=3, dt=dt, var=discrete_white_noise_var)
# measurement model - we only know position
h = np.array([[1.0, 0.0, 0.0]])
return simdkalman.KalmanFilter(state_transition=f,
process_noise=q,
observation_model=h,
observation_noise=r)
def test_smooth_helper_kalman_filter_2_states(self):
training_matrix = np.ones((5,10))
training_matrix[1,1] = np.nan
kf = simdkalman.KalmanFilter(
state_transition = np.eye(2),
process_noise = 0.1,
observation_model = np.array([[1,1]]),
observation_noise = 0.1)
r = kf.smooth(training_matrix)
self.assertSequenceEqual(r.observations.mean.shape, training_matrix.shape)
self.assertSequenceEqual(r.states.mean.shape, (5,10,2))
self.assertSequenceEqual(r.states.cov.shape, (5,10,2,2))
def test_train_and_predict_vectorized_kalman_filter_ema(self):
training_matrix = np.ones((5, 10))
kf = simdkalman.KalmanFilter(state_transition=1,
process_noise=0.1,
observation_model=1,
observation_noise=0.1)
r = kf.compute(training_matrix, n_test=4, initial_covariance=1.0)
self.assertSequenceEqual(r.predicted.observations.mean.shape, (5, 4))
self.assertSequenceEqual(r.smoothed.observations.mean.shape,
training_matrix.shape)
self.assertSequenceEqual(r.predicted.states.mean.shape, (5, 4, 1))
self.assertSequenceEqual(r.smoothed.states.mean.shape, (5, 10, 1))
self.assertSequenceEqual(r.predicted.states.cov.shape, (5, 4, 1, 1))
self.assertSequenceEqual(r.smoothed.states.cov.shape, (5, 10, 1, 1))
def test_em_algorithm(self):
training_matrix = np.ones((5, 10))
kf = simdkalman.KalmanFilter(state_transition=np.eye(2),
process_noise=0.1,
observation_model=np.array([[1, 1]]),
observation_noise=0.1)
r = kf.em(training_matrix, n_iter=5, verbose=False)
self.assertSequenceEqual(r.process_noise.shape, (5, 2, 2))
A0 = r.process_noise[0, ...]
self.assertMatrixEqual(A0, A0.T)
self.assertTrue(min(np.linalg.eig(A0)[0]) > 0)
B = r.observation_noise
self.assertSequenceEqual(B.shape, (5, 1, 1))
self.assertTrue(min(list(B)) > 0)
def compute_non_simd():
mean = np.empty((N_SERIES, N_MEAS, 2))
cov = np.empty((N_SERIES, N_MEAS, 2, 2))
for j in range(N_SERIES):
kf = simdkalman.KalmanFilter(state_transition, process_noise,
observation_model, observation_noise)
r = kf.smooth(data[j, :],
initial_value=initial_values[j, :],
initial_covariance=initial_covariance,
observations=False)
mean[j, :, :] = r.states.mean
cov[j, :, :, :] = r.states.cov
return mean, cov
def test_em_algorithm_multi_dimensional_observations(self):
training_matrix = np.ones((5,10,2))
training_matrix[1,1,1] = np.nan
kf = simdkalman.KalmanFilter(
state_transition = np.eye(2),
process_noise = 0.1,
observation_model = np.array([[1,1], [0,1]]),
observation_noise = 0.1)
r = kf.em(training_matrix, n_iter=5, verbose=False)
self.assertSequenceEqual(r.process_noise.shape, (5,2,2))
A0 = r.process_noise[0,...]
self.assertMatrixEqual(A0, A0.T, epsilon=1e-8)
self.assertTrue(min(np.linalg.eig(A0)[0]) > 0)
B = r.observation_noise
self.assertSequenceEqual(B.shape, (5,2,2))
self.assertTrue(min(np.linalg.eig(B[0,...])[0]) > 0)
def test_multi_dimensional_observations(self):
training_matrix = np.ones((5,10,2))
training_matrix[1,1,1] = np.nan
kf = simdkalman.KalmanFilter(
state_transition = np.eye(3),
process_noise = 0.1,
observation_model = np.array([[1,1,0],[0,0,1]]),
observation_noise = np.eye(2))
r = kf.compute(training_matrix, 15)
self.assertSequenceEqual(
r.smoothed.observations.mean.shape,
training_matrix.shape)
self.assertSequenceEqual(r.smoothed.states.mean.shape, (5,10,3))
self.assertSequenceEqual(r.smoothed.states.cov.shape, (5,10,3,3))
self.assertSequenceEqual(r.predicted.observations.mean.shape, (5,15,2))
self.assertSequenceEqual(r.predicted.observations.cov.shape, (5,15,2,2))
"""
Smoothing and prediction examples with figures. Needs matplotlib
to display the graphics.
Example output:
https://github.com/oseiskar/simdkalman/blob/master/doc/example.png
"""
import simdkalman
import numpy as np
import numpy.random as random
kf = simdkalman.KalmanFilter(state_transition=np.array([[1, 1], [0, 1]]),
process_noise=np.diag([0.1, 0.01]),
observation_model=np.array([[1, 0]]),
observation_noise=1.0)
# simulate 100 random walk time series
rand = lambda: random.normal(size=(100, 200))
data = np.cumsum(np.cumsum(rand() * 0.02, axis=1) + rand(),
axis=1) + rand() * 3
# introduce 10% of NaNs denoting missing values
data[random.uniform(size=data.shape) < 0.1] = np.nan
# fit noise parameters to data with the EM algorithm (optional)
kf = kf.em(data, n_iter=10)
# smooth and explain existing data
smoothed = kf.smooth(data)
# predict new data
def kalman(file):
print("Kalman filter Preprocessing file: ",file)
if os.path.exists(file):
try:
dataset = pd.read_csv(file)
time = dataset['time']
# dataset['event'] = pd.factorize(dataset['event'])[0]
# dataset['experiment'] = pd.factorize(dataset['experiment'])[0]
# dataset = dataset.drop(columns=['event', 'experiment', 'crew', 'time', 'seat' ])
#drop_these = list(set(list(dataset)) - set(['event', 'experiment', 'crew', 'time', 'seat' ]))
drop_these = list(set(['event', 'experiment', 'crew', 'seat', 'time' ]))
# data_time = dataset['time']
# data_ecg = dataset['ecg']
data_ = dataset.drop(drop_these, axis = 1).to_numpy()
lower_b = 0
upper_b = 0
increment = 1000
kalman_filtered = np.array(([]))
for i in range(0,data_.shape[0]// increment, 1):
lower_b = upper_b
upper_b += increment
data = preprocessing.scale(data_[lower_b:upper_b, 1:21])
#data_ = data_.to_numpy()[0:300, 1::]
# print(data_time.shape)
# print(data_ecg.shape)
# data_comb = np.stack((data_time,data_ecg))
# data = preprocessing.scale(data_)
# plt.plot(data)
# plt.show()
# data = data.reshape((data.shape[0],data.shape[0]))
kf = simdkalman.KalmanFilter(
state_transition = np.array([[1,1],[0,1]]),
process_noise = np.diag([0.1, 0.01]),
observation_model = np.array([[1,0]]),
observation_noise = 1.0)
kf = kf.em(data, n_iter=10)
# smooth and explain existing data
smoothed = kf.smooth(data)
# predict new data
pred = kf.predict(data, 15)
#plot_kalman(smoothed,pred, data)
smoothed_obs = smoothed.observations.mean[0,:]
# plt.plot(smoothed_obs)
# plt.show()
kalman_filtered = np.append(kalman_filtered, smoothed_obs)
k_inter = int_spline(kalman_filtered, np.arange(0, kalman_filtered.shape[0]), time)
return k_inter
except OSError:
print("Could not open/read file:", file)
sys.exit()
else:
print("File: ", file, " Does not exist")
sys.exit()
#!pip install simdkalman
import simdkalman
T = 1.0
state_transition = np.array([[1, 0, T, 0, 0.5 * T**2, 0],
[0, 1, 0, T, 0, 0.5 * T**2], [0, 0, 1, 0, T, 0],
[0, 0, 0, 1, 0, T], [0, 0, 0, 0, 1, 0],
[0, 0, 0, 0, 0, 1]])
process_noise = np.diag([1e-5, 1e-5, 5e-6, 5e-6, 1e-6, 1e-6]) + np.ones(
(6, 6)) * 1e-9
observation_model = np.array([[1, 0, 0, 0, 0, 0], [0, 1, 0, 0, 0, 0]])
observation_noise = np.diag([5e-5, 5e-5]) + np.ones((2, 2)) * 1e-9
kf = simdkalman.KalmanFilter(state_transition=state_transition,
process_noise=process_noise,
observation_model=observation_model,
observation_noise=observation_noise)
def apply_kf_smoothing(df, kf_=kf):
unique_paths = df[['collectionName',
'phoneName']].drop_duplicates().to_numpy()
for collection, phone in unique_paths:
cond = np.logical_and(df['collectionName'] == collection,
df['phoneName'] == phone)
data = df[cond][['latDeg', 'lngDeg']].to_numpy()
data = data.reshape(1, len(data), 2)
smoothed = kf_.smooth(data)
df.loc[cond, 'latDeg'] = smoothed.states.mean[0, :, 0]
df.loc[cond, 'lngDeg'] = smoothed.states.mean[0, :, 1]
return df
"""
Multi-dimensional observations example
"""
import simdkalman
import numpy as np
import numpy.random as random
# In this model, there is a hidden trend and two independent noisy observations
# are made at each step
kf = simdkalman.KalmanFilter(state_transition=np.array([[1, 1], [0, 1]]),
process_noise=np.diag([0.2, 0.01])**2,
observation_model=[[1, 0], [1, 0]],
observation_noise=np.eye(2) * 3**2)
# simulate 100 time series of 200 two-dimensional observations
rand = lambda: random.normal(size=(100, 200))
data = np.cumsum(np.cumsum(rand() * 0.02, axis=1) + rand(), axis=1)
data = np.dstack((data + rand() * 3, data + rand() * 3))
# smooth and explain existing data
smoothed = kf.smooth(data)
# predict new data
pred = kf.predict(data, 15)
import matplotlib.pyplot as plt
# show the first smoothed time series
i = 0
n_obs = data.shape[2]
H1 = np.array([[0.9, 0.1, 0]])
H0.shape = (1, 3)
H1.shape = (1, 3)
H = np.zeros((2, 1, 3))
H[0, :, :] = H0
H[1, :, :] = H1
# Transition, process noise
A = np.array([[0.5, 0.3, 0.2], [1, 0, 0], [0, 1, 0]])
Q = np.zeros((2, 3, 3))
Q[0, :, :] = np.eye(3)
Q[1, :, :] = np.eye(3)
# Alternative 1: primitives module
m1_prior, P1_prior = simdkalman.primitives.predict(m0, P0, A, Q)
m1, P1 = simdkalman.primitives.update(m1_prior, P1_prior, H, R, y_update)
print('\nWith primitives, m =')
print(m1)
# Alternative 2: update & predict_next
kf = simdkalman.KalmanFilter(
state_transition=A, # A
process_noise=Q, # Q
observation_model=H, # H
observation_noise=R) # R
m1_prior, P1_prior = kf.predict_next(m0, P0)
m1, P1, K, ll = kf.update(m1_prior, P1_prior, y_update, log_likelihood=True)
print('\nWith KalmanFilter, m = ')
print(m1)
