def model_selection(self,
data,
month,
year,
addition_method=True,
max_freqs=30):
spectrum_selection = list()
validate_data = dict()
for region in self.regions:
validate_data.update({
region:
trajectories_full_dates_periodic(data[region], month, year,
self.length_of_periodicity,
self.window_interval,
self.minute_interval)
})
for num_of_freqs in range(1, max_freqs + 1):
mse_region = list()
for region in self.regions:
reconstruction_tof, original_tof = self.reconstruct_tof_from_spectrum(
region, addition_method, num_of_freqs)
mse_recon, _ = self._calculate_mse(reconstruction_tof,
original_tof,
validate_data[region])
mse_region.append(mse_recon)
mse_region = [i for i in mse_region if i != -1]
spectrum_selection.append(sum(mse_region) / float(len(mse_region)))
spectrum_selection = spectrum_selection.index(
min(spectrum_selection)) + 1
rospy.loginfo("Ideal total spectrums: %d" % spectrum_selection)
self.spectrum_selection = spectrum_selection
self.addition_technique = addition_method
def model_selection(self, data, month, year, addition_method=True, max_freqs=30):
spectrum_selection = list()
validate_data = dict()
for region in self.regions:
validate_data.update({
region: trajectories_full_dates_periodic(
data[region], month, year, self.length_of_periodicity,
self.window_interval, self.minute_interval
)
})
for num_of_freqs in range(1, max_freqs+1):
mse_region = list()
for region in self.regions:
reconstruction_tof, original_tof = self.reconstruct_tof_from_spectrum(
region, addition_method, num_of_freqs
)
mse_recon, _ = self._calculate_mse(
reconstruction_tof, original_tof, validate_data[region]
)
mse_region.append(mse_recon)
mse_region = [i for i in mse_region if i != -1]
spectrum_selection.append(sum(mse_region) / float(len(mse_region)))
spectrum_selection = spectrum_selection.index(min(spectrum_selection)) + 1
rospy.loginfo("Ideal total spectrums: %d" % spectrum_selection)
self.spectrum_selection = spectrum_selection
self.addition_technique = addition_method
def prediction_accuracy(self,
region,
data,
month,
year,
percentile=0.1,
plot=False):
if self.spectrum_selection != 0:
reconstruction_tof, original_tof = self.reconstruct_tof_from_spectrum(
region, self.addition_technique, self.spectrum_selection)
else:
reconstruction_tof, original_tof = self.reconstruct_tof_from_spectrum(
region, self.addition_technique
# region, False, 26
)
test_data = trajectories_full_dates_periodic(
data, month, year, self.length_of_periodicity,
self.window_interval, self.minute_interval)
original_predict = self._get_prediction(original_tof, test_data,
percentile)
reconstr_predict = self._get_prediction(reconstruction_tof, test_data,
percentile)
_, clean_ori_pred, clean_recon_pred, indices = self._remove_unobserved_data(
test_data, reconstr_predict, original_predict)
if len(clean_ori_pred) and len(clean_recon_pred):
mse_predict = mse(clean_ori_pred, clean_recon_pred)
else:
mse_predict = -1
rospy.loginfo(
"Calculated MSE for prediction between original and reconstruction for Region %s: %.2f"
% (region, mse_predict))
if plot:
for index in indices:
original_predict[index] = -1
reconstr_predict[index] = -1
x = np.linspace(0, len(test_data), len(test_data))
xticks = time_ticks(self.minute_interval, self.window_interval,
self.periodic_type)
plt.plot(x,
original_predict,
"-o",
label="Prediction Original TOF")
plt.plot(x,
reconstr_predict,
"-^",
label="Prediction Reconstruction TOF")
plt.title("Prediction for Region %s" % region)
plt.xticks(x, xticks, rotation="vertical")
plt.xlabel(
"One Week Period with %d minutes interval and %d window time" %
(self.minute_interval, self.window_interval))
plt.ylabel("Prediction (1=Anomalous, 0=Normal, -1=Unobserved)")
plt.ylim(ymin=-2, ymax=2)
plt.legend()
plt.show()
def prediction_accuracy(self, region, data, month, year, percentile=0.1, plot=False):
if self.spectrum_selection != 0:
reconstruction_tof, original_tof = self.reconstruct_tof_from_spectrum(
region, self.addition_technique, self.spectrum_selection
)
else:
reconstruction_tof, original_tof = self.reconstruct_tof_from_spectrum(
region, self.addition_technique
# region, False, 26
)
test_data = trajectories_full_dates_periodic(
data, month, year, self.length_of_periodicity,
self.window_interval, self.minute_interval
)
original_predict = self._get_prediction(original_tof, test_data, percentile)
reconstr_predict = self._get_prediction(reconstruction_tof, test_data, percentile)
_, clean_ori_pred, clean_recon_pred, indices = self._remove_unobserved_data(
test_data, reconstr_predict, original_predict
)
if len(clean_ori_pred) and len(clean_recon_pred):
mse_predict = mse(clean_ori_pred, clean_recon_pred)
else:
mse_predict = -1
rospy.loginfo(
"Calculated MSE for prediction between original and reconstruction for Region %s: %.2f" % (region, mse_predict)
)
if plot:
for index in indices:
original_predict[index] = -1
reconstr_predict[index] = -1
x = np.linspace(0, len(test_data), len(test_data))
xticks = time_ticks(self.minute_interval, self.window_interval, self.periodic_type)
plt.plot(
x, original_predict, "-o", label="Prediction Original TOF"
)
plt.plot(
x, reconstr_predict, "-^", label="Prediction Reconstruction TOF"
)
plt.title("Prediction for Region %s" % region)
plt.xticks(x, xticks, rotation="vertical")
plt.xlabel("One Week Period with %d minutes interval and %d window time" % (self.minute_interval, self.window_interval))
plt.ylabel("Prediction (1=Anomalous, 0=Normal, -1=Unobserved)")
plt.ylim(ymin=-2, ymax=2)
plt.legend()
plt.show()
def calculate_mse(self, region, data, month, year):
if self.spectrum_selection != 0:
reconstruction_tof, original_tof = self.reconstruct_tof_from_spectrum(
region, self.addition_technique, self.spectrum_selection)
else:
reconstruction_tof, original_tof = self.reconstruct_tof_from_spectrum(
region, self.addition_technique)
test_data = trajectories_full_dates_periodic(
data, month, year, self.length_of_periodicity,
self.window_interval, self.minute_interval)
reconstruction_tof = [
i for j, i in enumerate(reconstruction_tof) if j % 5 == 0
]
original_tof = [i for j, i in enumerate(original_tof) if j % 5 == 0]
test_data = [i for j, i in enumerate(test_data) if j % 5 == 0]
# Dropping -1 in test_data together with corresponding tof
test_data, reconstruction_tof, original_tof, _ = self._remove_unobserved_data(
test_data, reconstruction_tof, original_tof)
mse_recon = -1
mse_origin = -1
if len(reconstruction_tof) > 0 and len(original_tof) > 0:
mse_recon = mse(test_data, reconstruction_tof)
mse_origin = mse(test_data, original_tof)
rospy.loginfo("Calculated MSE for original tof Region %s: %.2f" %
(region, mse_origin))
rospy.loginfo("Calculated MSE for reconstruction tof Region %s: %.2f" %
(region, mse_recon))
# temp_recon = np.sqrt(mse_recon)
# sum_recon = 0
# for i in test_data:
# sum_recon += (i - temp_recon)**2
# print "std_dev: %f" % (np.sqrt(sum_recon / float(len(test_data) - 1)))
# temp_recon = np.sqrt(mse_origin)
# sum_recon = 0
# for i in test_data:
# sum_recon += (i - temp_recon)**2
# print "std_dev: %f" % (np.sqrt(sum_recon / float(len(test_data) - 1)))
return mse_recon, mse_origin
def calculate_mse(self, region, data, month, year):
if self.spectrum_selection != 0:
reconstruction_tof, original_tof = self.reconstruct_tof_from_spectrum(
region, self.addition_technique, self.spectrum_selection
)
else:
reconstruction_tof, original_tof = self.reconstruct_tof_from_spectrum(
region, self.addition_technique
)
test_data = trajectories_full_dates_periodic(
data, month, year, self.length_of_periodicity,
self.window_interval, self.minute_interval
)
reconstruction_tof = [i for j, i in enumerate(reconstruction_tof) if j % 5 == 0]
original_tof = [i for j, i in enumerate(original_tof) if j % 5 == 0]
test_data = [i for j, i in enumerate(test_data) if j % 5 == 0]
# Dropping -1 in test_data together with corresponding tof
test_data, reconstruction_tof, original_tof, _ = self._remove_unobserved_data(
test_data, reconstruction_tof, original_tof
)
mse_recon = -1
mse_origin = -1
if len(reconstruction_tof) > 0 and len(original_tof) > 0:
mse_recon = mse(test_data, reconstruction_tof)
mse_origin = mse(test_data, original_tof)
rospy.loginfo("Calculated MSE for original tof Region %s: %.2f" % (region, mse_origin))
rospy.loginfo("Calculated MSE for reconstruction tof Region %s: %.2f" % (region, mse_recon))
# temp_recon = np.sqrt(mse_recon)
# sum_recon = 0
# for i in test_data:
# sum_recon += (i - temp_recon)**2
# print "std_dev: %f" % (np.sqrt(sum_recon / float(len(test_data) - 1)))
# temp_recon = np.sqrt(mse_origin)
# sum_recon = 0
# for i in test_data:
# sum_recon += (i - temp_recon)**2
# print "std_dev: %f" % (np.sqrt(sum_recon / float(len(test_data) - 1)))
return mse_recon, mse_origin
