def _volume_shift_detection(self,
mean_list=None,
sd_list=None,
probability_threshold=0.5):
"""
This function detects any significant shift in the training data volume using a Bayesian change point detection
technique.
:param list mean_list: The list of means from each training sub-window.
:param list sd_list: The list of standard deviations from each training sub-window.
:param float probability_threshold: Threshold for the probability value to be flagged as a change point.
:return: Indices with significant vdata volume shift.
:rtype: int
"""
import numpy as np
from bayesian_changepoint_detection import offline_changepoint_detection as offcd
from functools import partial
# Volume shift detection over the means of the training window
q, p, pcp = offcd.offline_changepoint_detection(
data=np.array(mean_list),
prior_func=partial(offcd.const_prior, l=(len(mean_list) + 1)),
observation_log_likelihood_function=offcd.
gaussian_obs_log_likelihood,
truncate=-10)
mask_mean = np.append(0, np.exp(pcp).sum(0)) > probability_threshold
# Volume shift detection over the standard deviations of the training window
change_points = np.array(mask_mean).nonzero()
last_mean_cp = change_points[0][-1] if len(
change_points[0]) > 0 else []
q, p, pcp = offcd.offline_changepoint_detection(
data=np.array(sd_list),
prior_func=partial(offcd.const_prior, l=(len(sd_list) + 1)),
observation_log_likelihood_function=offcd.
gaussian_obs_log_likelihood,
truncate=-10)
mask_sd = np.append(0, np.exp(pcp).sum(0)) > probability_threshold
change_points = np.array(mask_sd).nonzero()
last_sd_cp = change_points[0][-1] if len(change_points[0]) > 0 else []
# Change point is the maximum obtained from mean list and the standard deviation list
cdate = max(last_mean_cp, last_sd_cp)
return cdate
def cp_func1(d):
signal = (d[['smooth_speed']]).values
#print(signal)
Q, P, Pcp = offcd.offline_changepoint_detection(
signal,
partial(offcd.const_prior, l=(len(signal) + 1)),
offcd.gaussian_obs_log_likelihood,
truncate=-50)
j2 = (np.exp(Pcp).sum(0))
j2 = np.insert(j2, 0, 0)
series = pd.DataFrame({
'Speed': signal.flatten(),
'Prob': j2,
'time': d.time,
'segment': d.segment,
'date': d.date,
'raw_speed': d.speed,
'Traveltime': d.traveltime,
'score': d.score_30,
'ref_speed': d.ref_speed
})
#series.to_csv('C:/Users/atousaz/Desktop/loop/CP/'+str(segment)+'/'+str(date1)+'.csv')
#complete = time.time()
#print(c, 'done in ', (complete-now), ' s')
return series
def find_changes(series, truncate=-np.inf):
data = series.to_frame()
Q, P, Pcp = offcd.offline_changepoint_detection(
data,
partial(offcd.const_prior, l=(len(data) + 1)),
offcd.gaussian_obs_log_likelihood,
truncate=truncate)
data['Change'] = np.append([0], np.exp(Pcp).sum(0))
return data.Change
def get_cp_prob(self, ts):
l = np.asarray(ts.y)
prior = self.get_prior(self.prior, self.p, self.k, len(l) + 1)
# ALERT: the truncate parameter can affect the performance
q, p, pcp = offcd.offline_changepoint_detection(
l, prior, offcd.gaussian_obs_log_likelihood, truncate=-90)
cp_prob = np.exp(pcp).sum(0)
ts_cp_prob = time_series.dist_ts(ts)
for i in xrange(len(ts.x) - 1):
ts_cp_prob.x.append(ts.x[i])
ts_cp_prob.y.append(cp_prob[i])
return ts_cp_prob
def plot_changepoint(data, col, interval=10, title='', output=''):
"""
data: data frame
col: column in the data frame
interval: interval of years to show in the x-axis
ci = if True, plot confidence intervals
"""
print('Computing change point for', title)
# Changepoint detection
Q, P, Pcp = offcd.offline_changepoint_detection(
data[col],
partial(offcd.const_prior, l=(len(data[col]) + 1)),
offcd.gaussian_obs_log_likelihood,
truncate=-40)
print('Plotting...')
# Getting info for the x-axis
indexes = data.index[data.index % interval == 0].tolist()
labels = list(map(str, indexes))
time_ticks = np.where(data.index.isin(indexes))[0].tolist()
# Plotting
fig, (ax1, ax2) = plt.subplots(2, 1)
line = savgol_filter(data[col], 33, 3)
ax1.plot(range(len(data.index)), line)
ax1.plot(range(len(data.index)), data[col], '.', ms=2)
ax1.set_xticks(time_ticks)
ax1.set_xticklabels(labels)
ax1.set_ylabel(title)
ax2.plot(np.exp(Pcp).sum(0))
ax2.set_ylim([0, 1])
ax2.set_xticks(time_ticks)
ax2.set_xticklabels(labels)
ax2.set_ylabel('Probability')
if output:
filename = output
else:
min_sample_size = min(data.corpus_N)
filename = title.lower() + str(min_sample_size) + '.png'
plt.savefig(filename)
def identify_change_points(self):
#print('estimating change points')
if 'cp_params' in self.extra_inputs.keys():
cp_params = self.extra_inputs['cp_params']
method = cp_params['method']
else: # defaults
method == 'Online'
if method == 'Online': # online method
R, maxes = oncd.online_changepoint_detection(
self.feature_by_time_matrix_reduced,
partial(oncd.constant_hazard, 250),
oncd.MV_Norm(
mu=np.zeros(self.feature_by_time_matrix_reduced.shape[1]),
Sigma=5.0 * np.diag(
np.ones(self.feature_by_time_matrix_reduced.shape[1])),
n=np.array([1.0])))
diff_in_max = np.abs(np.diff(np.argmax(
R,
axis=0))) # looks for differences in most likely run lengths
expected_run_len = np.dot(R.T, np.arange(len(R)))
self.R = R
self.diff_in_max = diff_in_max
self.expected_run_len = expected_run_len
# calculate change points
self.change_points = np.zeros(
self.feature_by_time_matrix_reduced.shape[0])
self.change_points[diff_in_max > 5] = 1.0
elif method == 'Offline':
Q, P, Pcp = offcd.offline_changepoint_detection(
self.feature_by_time_matrix_reduced,
partial(offcd.const_prior,
l=(self.feature_by_time_matrix_reduced.shape[0] + 1)),
offcd.fullcov_obs_log_likelihood,
truncate=-20)
self.cp_prob = np.exp(Pcp).sum(0)
self.change_points = self.cp_prob > 0.7
def update_p_cp(world, use_ros):
P_cp = []
pid = multiprocessing.current_process().pid
for j, joint in enumerate(world.joints):
if use_ros:
q = Record.records[pid]["q_" + str(j)].as_matrix()
af = Record.records[pid]["applied_force_" + str(j)][0:].as_matrix()
v = q[1:] - q[0:-1] # we can't measure the velocity directly
vn = v[:] + af[1:]
d = np.zeros((v.shape[0] + 1,))
d[1:] = abs((vn**2 - v[:]**2)/(0.1 * vn))
else:
v = Record.records[pid]["v_" + str(j)][0:].as_matrix()
af = Record.records[pid]["applied_force_" + str(j)][0:].as_matrix()
vn = v[:-1] + af[:-1]
d = np.zeros(v.shape)
d[1:] = abs((vn**2 - v[1:]**2)/(0.1 * vn))
nans, x = nan_helper(d)
d[nans] = np.interp(x(nans), x(~nans), d[~nans])
Q, P, Pcp = bcd.offline_changepoint_detection(
data=d,
prior_func=partial(bcd.const_prior, l=(len(d)+1)),
observation_log_likelihood_function=
bcd.gaussian_obs_log_likelihood,
truncate=-50)
p_cp, count = get_probability_over_degree(
np.exp(Pcp).sum(0)[:1],
Record.records[pid]['q_' + str(j)][-1:].as_matrix())
P_cp.append(p_cp)
return P_cp
def update_p_cp(world, use_ros):
P_cp = []
pid = multiprocessing.current_process().pid
for j, joint in enumerate(world.joints):
if use_ros:
q = Record.records[pid]["q_" + str(j)].as_matrix()
af = Record.records[pid]["applied_force_" + str(j)][0:].as_matrix()
v = q[1:] - q[0:-1] # we can't measure the velocity directly
vn = v[:] + af[1:]
d = np.zeros((v.shape[0] + 1,))
d[1:] = abs((vn**2 - v[:]**2)/(0.1 * vn))
else:
v = Record.records[pid]["v_" + str(j)][0:].as_matrix()
af = Record.records[pid]["applied_force_" + str(j)][0:].as_matrix()
vn = v[:-1] + af[:-1]
d = np.zeros(v.shape)
d[1:] = abs((vn**2 - v[1:]**2)/(0.1 * vn))
nans, x = nan_helper(d)
d[nans] = np.interp(x(nans), x(~nans), d[~nans])
Q, P, Pcp = bcd.offline_changepoint_detection(
data=d,
prior_func=partial(bcd.const_prior, l=(len(d)+1)),
observation_log_likelihood_function=
bcd.gaussian_obs_log_likelihood,
truncate=-50)
p_cp, count = get_probability_over_degree(
np.exp(Pcp).sum(0)[:1],
Record.records[pid]['q_' + str(j)][-1:].as_matrix())
P_cp.append(p_cp)
return P_cp
import numpy as np
import matplotlib.pyplot as plt
import bayesian_changepoint_detection.offline_changepoint_detection as offcd
import bayesian_changepoint_detection.generate_data as gd
from functools import partial
if __name__ == '__main__':
show_plot = True
partition, data = gd.generate_xuan_motivating_example(50, 200)
changes = np.cumsum(partition)
Q_ifm, P_ifm, Pcp_ifm = offcd.offline_changepoint_detection(
data,
partial(offcd.const_prior, l=(len(data) + 1)),
offcd.ifm_obs_log_likelihood,
truncate=-20)
Q_full, P_full, Pcp_full = offcd.offline_changepoint_detection(
data,
partial(offcd.const_prior, l=(len(data) + 1)),
offcd.fullcov_obs_log_likelihood,
truncate=-20)
if show_plot:
fig, ax = plt.subplots(figsize=[18, 16])
ax = fig.add_subplot(3, 1, 1)
for p in changes:
ax.plot([p, p], [np.min(data), np.max(data)], 'r')
for d in range(2):
ax.plot(data[:, d])
ope = []
clo = []
for p in range(nrows):
temp = table.row_values(p, start_colx=1)
ope.append(float(temp[0]))
clo.append(float(temp[3]))
for q in range(len(temp)):
temp[q] = (float(temp[q]) - minn[q]) / (maxn[q] - minn[q])
x.append(temp)
arr = np.array(x)
#print arr
Q, P, Pcp = offcd.offline_changepoint_detection(
arr,
partial(offcd.const_prior, l=(len(arr) + 1)),
offcd.gaussian_obs_log_likelihood,
truncate=-20)
#Q_ifm, P_ifm, Pcp_ifm = offcd.offline_changepoint_detection(arr,partial(offcd.const_prior, l=(len(arr)+1)),offcd.ifm_obs_log_likelihood,truncate=-20)
#Q_full, P_full, Pcp_full = offcd.offline_changepoint_detection(arr,partial(offcd.const_prior, l=(len(arr)+1)),offcd.fullcov_obs_log_likelihood, truncate=-20)
if show_plot:
fig, ax = plt.subplots(figsize=[18, 16])
#ax = fig.add_subplot(2, 1, 1)
# for p in changes:
# ax.plot([p,p],[np.min(data),np.max(data)],'r')
for d in range(dim):
ax.plot(arr[:, d])
#print arr[:,d]
#print d
''' Example from Xiang Xuan's thesis: Section 3.2'''
from __future__ import division
import numpy as np
import matplotlib.pyplot as plt
import bayesian_changepoint_detection.offline_changepoint_detection as offcd
import bayesian_changepoint_detection.generate_data as gd
from functools import partial
if __name__ == '__main__':
show_plot = True
partition, data = gd.generate_xuan_motivating_example(50,200)
changes = np.cumsum(partition)
Q_ifm, P_ifm, Pcp_ifm = offcd.offline_changepoint_detection(data,partial(offcd.const_prior, l=(len(data)+1)),offcd.ifm_obs_log_likelihood,truncate=-20)
Q_full, P_full, Pcp_full = offcd.offline_changepoint_detection(data,partial(offcd.const_prior, l=(len(data)+1)),offcd.fullcov_obs_log_likelihood, truncate=-20)
if show_plot:
fig, ax = plt.subplots(figsize=[18, 16])
ax = fig.add_subplot(3, 1, 1)
for p in changes:
ax.plot([p,p],[np.min(data),np.max(data)],'r')
for d in range(2):
ax.plot(data[:,d])
plt.legend(['Raw data with Original Changepoints'])
ax1 = fig.add_subplot(3, 1, 2, sharex=ax)
ax1.plot(np.exp(Pcp_ifm).sum(0))
plt.legend(['Independent Factor Model'])
ax2 = fig.add_subplot(3, 1, 3, sharex=ax)
ax2.plot(np.exp(Pcp_full).sum(0))
ope = []
clo = []
for p in range(nrows):
temp = table.row_values(p, start_colx=1)
ope.append(float(temp[0]))
clo.append(float(temp[3]))
for q in range(len(temp)):
temp[q] = (float(temp[q])-minn[q])/(maxn[q]-minn[q])
x.append(temp)
arr = np.array(x)
#print arr
Q, P, Pcp = offcd.offline_changepoint_detection(arr,partial(offcd.const_prior, l=(len(arr)+1)),offcd.gaussian_obs_log_likelihood, truncate=-20)
#Q_ifm, P_ifm, Pcp_ifm = offcd.offline_changepoint_detection(arr,partial(offcd.const_prior, l=(len(arr)+1)),offcd.ifm_obs_log_likelihood,truncate=-20)
#Q_full, P_full, Pcp_full = offcd.offline_changepoint_detection(arr,partial(offcd.const_prior, l=(len(arr)+1)),offcd.fullcov_obs_log_likelihood, truncate=-20)
if show_plot:
fig, ax = plt.subplots(figsize=[18, 16])
#ax = fig.add_subplot(2, 1, 1)
# for p in changes:
# ax.plot([p,p],[np.min(data),np.max(data)],'r')
for d in range(dim):
ax.plot(arr[:,d])
#print arr[:,d]
#print d
#ax = fig.add_subplot(2, 1, 2, sharex=ax)
index = table1.cell_value(i,4)
temp = float(index)
if i !=0:
index = float(float(index)-pre)/pre*100
else:
index = 0
pre = temp
re = table2.cell_value(i,2)
re = float(re)
y = []
y.append(index)
y.append(re)
x.append(y)
arr2 = np.array(x)
Q_1, P_1, Pcp_1 = offcd.offline_changepoint_detection(arr1,partial(offcd.const_prior, l=(len(arr1)+1)),offcd.gaussian_obs_log_likelihood, truncate=-20)
Q_2, P_2, Pcp_2 = offcd.offline_changepoint_detection(arr2,partial(offcd.const_prior, l=(len(arr2)+1)),offcd.gaussian_obs_log_likelihood, truncate=-20)
#Q_ifm, P_ifm, Pcp_ifm = offcd.offline_changepoint_detection(arr,partial(offcd.const_prior, l=(len(arr)+1)),offcd.ifm_obs_log_likelihood,truncate=-20)
#Q_full, P_full, Pcp_full = offcd.offline_changepoint_detection(arr,partial(offcd.const_prior, l=(len(arr)+1)),offcd.fullcov_obs_log_likelihood, truncate=-20)
#if show_plot:
# fig, ax = plt.subplots(figsize=[18, 16])
# ax = fig.add_subplot(2, 1, 1)
# for p in changes:
# ax.plot([p,p],[np.min(data),np.max(data)],'r')
#for d in range(dim):
# ax.plot(arr[:,d])
#print arr[:,d]
#print d
#ax = fig.add_subplot(2, 1, 2, sharex=ax)
if i != 0:
index = float(float(index) - pre) / pre * 100
else:
index = 0
pre = temp
re = table2.cell_value(i, 2)
re = float(re)
y = []
y.append(index)
y.append(re)
x.append(y)
arr2 = np.array(x)
Q_1, P_1, Pcp_1 = offcd.offline_changepoint_detection(
arr1,
partial(offcd.const_prior, l=(len(arr1) + 1)),
offcd.gaussian_obs_log_likelihood,
truncate=-20)
Q_2, P_2, Pcp_2 = offcd.offline_changepoint_detection(
arr2,
partial(offcd.const_prior, l=(len(arr2) + 1)),
offcd.gaussian_obs_log_likelihood,
truncate=-20)
#Q_ifm, P_ifm, Pcp_ifm = offcd.offline_changepoint_detection(arr,partial(offcd.const_prior, l=(len(arr)+1)),offcd.ifm_obs_log_likelihood,truncate=-20)
#Q_full, P_full, Pcp_full = offcd.offline_changepoint_detection(arr,partial(offcd.const_prior, l=(len(arr)+1)),offcd.fullcov_obs_log_likelihood, truncate=-20)
#if show_plot:
# fig, ax = plt.subplots(figsize=[18, 16])
# ax = fig.add_subplot(2, 1, 1)
# for p in changes: