def gridsearch_reservoir_computing(self,
train_X,
train_y,
test_X,
test_y,
per_time_step=False,
error='mse',
gridsearch_training_frac=0.7):
tuned_parameters = {
'a': [0.6, 0.8],
'reservoir_size': [400, 700, 1000]
}
# tuned_parameters = {'a': [0.4], 'reservoir_size':[250]}
params = tuned_parameters.keys()
combinations = self.generate_parameter_combinations(
tuned_parameters, params)
split_point = int(gridsearch_training_frac * len(train_X.index))
train_params_X = train_X.ix[0:split_point, ]
test_params_X = train_X.ix[split_point:len(train_X.index), ]
train_params_y = train_y.ix[0:split_point, ]
test_params_y = train_y.ix[split_point:len(train_X.index), ]
if error == 'mse':
best_error = sys.float_info.max
elif error == 'accuracy':
best_error = 0
best_combination = []
for comb in combinations:
print comb
# Order of the keys might have changed.
keys = tuned_parameters.keys()
pred_train_y, pred_test_y, pred_train_y_prob, pred_test_y_prob = self.reservoir_computing(
train_params_X,
train_params_y,
test_params_X,
test_params_y,
reservoir_size=comb[keys.index('reservoir_size')],
a=comb[keys.index('a')],
per_time_step=per_time_step,
gridsearch=False)
if error == 'mse':
eval = RegressionEvaluation()
mse = eval.mean_squared_error(test_params_y, pred_test_y_prob)
if mse < best_error:
best_error = mse
best_combination = comb
elif error == 'accuracy':
eval = ClassificationEvaluation()
acc = eval.accuracy(test_params_y, pred_test_y)
if acc > best_error:
best_error = acc
best_combination = comb
print '-------'
print best_combination
print '-------'
return best_combination[keys.index(
'reservoir_size')], best_combination[keys.index('a')]
def gridsearch_recurrent_neural_network(self,
train_X,
train_y,
test_X,
test_y,
error='accuracy',
gridsearch_training_frac=0.7):
tuned_parameters = {
'n_hidden_neurons': [50, 100],
'iterations': [250, 500],
'outputbias': [True]
}
params = list(tuned_parameters.keys())
combinations = self.generate_parameter_combinations(
tuned_parameters, params)
split_point = int(gridsearch_training_frac * len(train_X.index))
train_params_X = train_X.iloc[0:split_point, ]
test_params_X = train_X.iloc[split_point:len(train_X.index), ]
train_params_y = train_y.iloc[0:split_point, ]
test_params_y = train_y.iloc[split_point:len(train_X.index), ]
if error == 'mse':
best_error = sys.float_info.max
elif error == 'accuracy':
best_error = 0
best_combination = []
for comb in combinations:
print(comb)
# Order of the keys might have changed.
keys = list(tuned_parameters.keys())
# print(keys)
pred_train_y, pred_test_y, pred_train_y_prob, pred_test_y_prob = self.recurrent_neural_network(
train_params_X,
train_params_y,
test_params_X,
test_params_y,
n_hidden_neurons=comb[keys.index('n_hidden_neurons')],
iterations=comb[keys.index('iterations')],
outputbias=comb[keys.index('outputbias')],
gridsearch=False)
if error == 'mse':
eval = RegressionEvaluation()
mse = eval.mean_squared_error(test_params_y, pred_test_y_prob)
if mse < best_error:
best_error = mse
best_combination = comb
elif error == 'accuracy':
eval = ClassificationEvaluation()
acc = eval.accuracy(test_params_y, pred_test_y)
if acc > best_error:
best_error = acc
best_combination = comb
print('-------')
print(best_combination)
print('-------')
return best_combination[params.index(
'n_hidden_neurons')], best_combination[params.index(
'iterations')], best_combination[params.index('outputbias')]
def backward_selection(self, max_features, X_train, y_train):
# First select all features.
selected_features = X_train.columns.tolist()
ra = RegressionAlgorithms()
re = RegressionEvaluation()
# Select from the features that are still in the selection.
for i in range(0, (len(X_train.columns) - max_features)):
best_perf = sys.float_info.max
worst_feature = ''
for f in selected_features:
temp_selected_features = copy.deepcopy(selected_features)
temp_selected_features.remove(f)
# Determine the score without the feature.
pred_y_train, pred_y_test = ra.decision_tree(X_train[temp_selected_features], y_train, X_train[temp_selected_features])
perf = re.mean_squared_error(y_train, pred_y_train)
# If we score better (i.e. a lower mse) without the feature than what we have seen so far
# this is the worst feature.
if perf < best_perf:
best_perf = perf
worst_feature = f
# Remove the worst feature.
selected_features.remove(worst_feature)
return selected_features
def forward_selection(
max_features: int, X_train: pd.DataFrame,
y_train: pd.Series) -> Tuple[List[str], List[str], List[float]]:
"""
Select the given number of features for regression, that show the best accuracy, using forward selection.
The method uses the given features and labels to train a decision tree and determine the mse of the
predictions. The method returns the selected features as well as the the scores.
:param max_features: Number of features to select.
:param X_train: Features as DataFrame.
:param y_train: True values corresponding to given features.
:return: Selected features and scores.
"""
ordered_features = []
ordered_scores = []
# Start with no features
selected_features = []
ra = RegressionAlgorithms()
re = RegressionEvaluation()
# Select the appropriate number of features
for i in range(0, max_features):
# Determine the features left to select
features_left = list(set(X_train.columns) - set(selected_features))
best_perf = sys.float_info.max
best_feature = ''
# Iterate over all features left
for f in features_left:
temp_selected_features = copy.deepcopy(selected_features)
temp_selected_features.append(f)
# Determine the mse of a decision tree learner when adding the feature
pred_y_train, pred_y_test = ra.decision_tree(
X_train[temp_selected_features], y_train,
X_train[temp_selected_features])
perf = re.mean_squared_error(y_train, pred_y_train)
# If the performance is better than seen so far (aiming for low mse) set the current feature to the best
# feature and the same for the best performance
if perf < best_perf:
best_perf = perf
best_feature = f
# Select the feature with the best performance
selected_features.append(best_feature)
ordered_features.append(best_feature)
ordered_scores.append(best_perf)
return selected_features, ordered_features, ordered_scores
def gridsearch_time_series(self,
train_X,
train_y,
test_X,
test_y,
error='mse',
gridsearch_training_frac=0.7):
tuned_parameters = {'ar': [0, 5], 'ma': [0, 5], 'd': [1]}
params = tuned_parameters.keys()
tc = TemporalClassificationAlgorithms()
combinations = tc.generate_parameter_combinations(
tuned_parameters, params)
split_point = int(gridsearch_training_frac * len(train_X.index))
train_params_X = train_X.ix[0:split_point, ]
test_params_X = train_X.ix[split_point:len(train_X.index), ]
train_params_y = train_y.ix[0:split_point, ]
test_params_y = train_y.ix[split_point:len(train_X.index), ]
if error == 'mse':
best_error = sys.float_info.max
elif error == 'accuracy':
best_error = 0
best_combination = []
for comb in combinations:
print comb
# Order of the keys might have changed.
keys = tuned_parameters.keys()
pred_train_y, pred_test_y = self.time_series(
train_params_X,
train_params_y,
test_params_X,
test_params_y,
ar=comb[keys.index('ar')],
ma=comb[keys.index('ma')],
d=comb[keys.index('d')],
gridsearch=False)
eval = RegressionEvaluation()
mse = eval.mean_squared_error(test_params_y, pred_test_y)
if mse < best_error:
best_error = mse
best_combination = comb
print '-------'
print best_combination
print '-------'
return best_combination[keys.index('ar')], best_combination[keys.index(
'ma')], best_combination[keys.index('d')]
def forward_selection(self, max_features, X_train, y_train):
ordered_features = []
ordered_scores = []
# Start with no features.
selected_features = []
ra = RegressionAlgorithms()
re = RegressionEvaluation()
prev_best_perf = sys.float_info.max
# Select the appropriate number of features.
for i in range(0, max_features):
print i
#Determine the features left to select.
features_left = list(set(X_train.columns) - set(selected_features))
best_perf = sys.float_info.max
best_feature = ''
# For all features we can still select...
for f in features_left:
temp_selected_features = copy.deepcopy(selected_features)
temp_selected_features.append(f)
# Determine the mse of a decision tree learner if we were to add
# the feature.
pred_y_train, pred_y_test = ra.decision_tree(
X_train[temp_selected_features], y_train,
X_train[temp_selected_features])
perf = re.mean_squared_error(y_train, pred_y_train)
# If the performance is better than what we have seen so far (we aim for low mse)
# we set the current feature to the best feature and the same for the best performance.
if perf < best_perf:
best_perf = perf
best_feature = f
# We select the feature with the best performance.
selected_features.append(best_feature)
prev_best_perf = best_perf
ordered_features.append(best_feature)
ordered_scores.append(best_perf)
return selected_features, ordered_features, ordered_scores
def forward_selection(max_features, X_train, y_train):
ordered_features = []
ordered_scores = []
# Start with no features.
selected_features = []
learner = TemporalRegressionAlgorithms()
eval = RegressionEvaluation()
prev_best_perf = sys.float_info.max
# Select the appropriate number of features.
for i in range(0, max_features):
print i
#Determine the features left to select.
features_left = list(set(X_train.columns) - set(selected_features))
best_perf = sys.float_info.max
best_feature = ''
# For all features we can still select...
for f in features_left:
temp_selected_features = copy.deepcopy(selected_features)
temp_selected_features.append(f)
# Determine the mse of a decision tree learner if we were to add
# the feature.
regr_train_y, regr_test_y = learner.recurrent_neural_network(
X_train, y_train, X_train, y_train, gridsearch=False)
perf, std_tr = eval.mean_squared_error_with_std(
train_y.ix[10:, ], regr_train_y.ix[10:, ])
# If the performance is better than what we have seen so far (we aim for low mse)
# we set the current feature to the best feature and the same for the best performance.
if perf < best_perf:
best_perf = perf
best_feature = f
# We select the feature with the best performance.
selected_features.append(best_feature)
prev_best_perf = best_perf
ordered_features.append(best_feature)
ordered_scores.append(best_perf)
return selected_features, ordered_features, ordered_scores
def backward_selection(max_features, X_train, y_train):
"""
Select the given number of features for regression, that show the best accuracy, using backward selection.
The method uses the given features and labels to train a decision tree and determine the mse of the
predictions.
:param max_features: Number of features to select.
:param X_train: Features as DataFrame.
:param y_train: True values corresponding to given features.
:return: Selected features.
"""
# First select all features
selected_features = X_train.columns.tolist()
ra = RegressionAlgorithms()
re = RegressionEvaluation()
# Select from the features that are still in the selection
for i in range(0, (len(X_train.columns) - max_features)):
best_perf = sys.float_info.max
worst_feature = ''
for f in selected_features:
temp_selected_features = copy.deepcopy(selected_features)
temp_selected_features.remove(f)
# Determine the score without the feature
pred_y_train, pred_y_test = ra.decision_tree(
X_train[temp_selected_features], y_train,
X_train[temp_selected_features])
perf = re.mean_squared_error(y_train, pred_y_train)
# If scoring better (i.e. a lower mse) without the feature than seen so far this is the worst feature
if perf < best_perf:
best_perf = perf
worst_feature = f
# Remove the worst feature
selected_features.remove(worst_feature)
return selected_features
def main():
# Read the result from the previous chapter and convert the index to datetime
try:
dataset = pd.read_csv(DATA_PATH / DATASET_FILENAME, index_col=0)
dataset.index = pd.to_datetime(dataset.index)
except IOError as e:
print('File not found, try to run previous crowdsignals scripts first!')
raise e
# Create an instance of visualization class to plot the results
DataViz = VisualizeDataset(__file__)
# Consider the second task, namely the prediction of the heart rate. Therefore create a dataset with the heart
# rate as target and split using timestamps, because this is considered as a temporal task.
print('\n- - - Loading dataset - - -')
prepare = PrepareDatasetForLearning()
learner = RegressionAlgorithms()
evaluation = RegressionEvaluation()
train_X, test_X, train_y, test_y = prepare.split_single_dataset_regression_by_time(dataset, 'hr_watch_rate',
'2016-02-08 18:28:56',
'2016-02-08 19:34:07',
'2016-02-08 20:07:50')
print('Training set length is: ', len(train_X.index))
print('Test set length is: ', len(test_X.index))
# Select subsets of the features
print('- - - Selecting subsets - - -')
basic_features = ['acc_phone_x', 'acc_phone_y', 'acc_phone_z', 'acc_watch_x', 'acc_watch_y', 'acc_watch_z',
'gyr_phone_x', 'gyr_phone_y', 'gyr_phone_z', 'gyr_watch_x', 'gyr_watch_y', 'gyr_watch_z',
'labelOnTable', 'labelSitting', 'labelWashingHands', 'labelWalking', 'labelStanding',
'labelDriving',
'labelEating', 'labelRunning', 'light_phone_lux', 'mag_phone_x', 'mag_phone_y', 'mag_phone_z',
'mag_watch_x', 'mag_watch_y', 'mag_watch_z', 'press_phone_pressure']
pca_features = ['pca_1', 'pca_2', 'pca_3', 'pca_4', 'pca_5', 'pca_6', 'pca_7']
time_features = [name for name in dataset.columns if ('temp_' in name and 'hr_watch' not in name)]
freq_features = [name for name in dataset.columns if (('_freq' in name) or ('_pse' in name))]
cluster_features = ['cluster']
print('#basic features: ', len(basic_features))
print('#PCA features: ', len(pca_features))
print('#time features: ', len(time_features))
print('#frequency features: ', len(freq_features))
print('#cluster features: ', len(cluster_features))
features_after_chapter_3 = list(set().union(basic_features, pca_features))
features_after_chapter_4 = list(set().union(features_after_chapter_3, time_features, freq_features))
features_after_chapter_5 = list(set().union(features_after_chapter_4, cluster_features))
if FLAGS.mode == 'selection' or FLAGS.mode == 'all':
# First, consider the Pearson correlations and see whether features can be selected based on them
fs = FeatureSelectionRegression()
print('\n- - - Running feature selection - - -')
features, correlations = fs.pearson_selection(10, train_X[features_after_chapter_5], train_y)
util.print_pearson_correlations(correlations)
# Select the 10 features with the highest correlation
selected_features = ['temp_pattern_labelOnTable', 'labelOnTable', 'temp_pattern_labelOnTable(b)labelOnTable',
'pca_2_temp_mean_ws_120', 'pca_1_temp_mean_ws_120', 'acc_watch_y_temp_mean_ws_120', 'pca_2',
'acc_phone_z_temp_mean_ws_120', 'gyr_watch_y_pse', 'gyr_watch_x_pse']
possible_feature_sets = [basic_features, features_after_chapter_3, features_after_chapter_4,
features_after_chapter_5, selected_features]
feature_names = ['initial set', 'Chapter 3', 'Chapter 4', 'Chapter 5', 'Selected features']
if FLAGS.mode == 'overall' or FLAGS.mode == 'all':
print('\n- - - Running test of all different regression algorithms - - -')
# First study the importance of the parameter settings. Therefore repeat the experiment a number of times to get
# a bit more robust data as the initialization of e.g. the NN is random
REPEATS = FLAGS.repeats
scores_over_all_algs = []
for i in range(0, len(possible_feature_sets)):
selected_train_X = train_X[possible_feature_sets[i]]
selected_test_X = test_X[possible_feature_sets[i]]
performance_tr_nn, performance_tr_nn_std = 0, 0
performance_tr_rf, performance_tr_rf_std = 0, 0
performance_te_nn, performance_te_nn_std = 0, 0
performance_te_rf, performance_te_rf_std = 0, 0
# First run non deterministic classifiers a number of times to average their score
for repeat in range(0, REPEATS):
print(f'Training NeuralNetwork run {repeat + 1}/{REPEATS} ... ')
regr_train_y, regr_test_y = learner.\
feedforward_neural_network(selected_train_X, train_y, selected_test_X, gridsearch=True)
mean_tr, std_tr = evaluation.mean_squared_error_with_std(train_y, regr_train_y)
mean_te, std_te = evaluation.mean_squared_error_with_std(test_y, regr_test_y)
performance_tr_nn += mean_tr
performance_tr_nn_std += std_tr
performance_te_nn += mean_te
performance_te_nn_std += std_te
print(f'Training RandomForest run {repeat + 1}/{REPEATS} ... ')
regr_train_y, regr_test_y = learner.random_forest(selected_train_X, train_y, selected_test_X,
gridsearch=True)
mean_tr, std_tr = evaluation.mean_squared_error_with_std(train_y, regr_train_y)
mean_te, std_te = evaluation.mean_squared_error_with_std(test_y, regr_test_y)
performance_tr_rf += mean_tr
performance_tr_rf_std += std_tr
performance_te_rf += mean_te
performance_te_rf_std += std_te
overall_performance_tr_nn = performance_tr_nn / REPEATS
overall_performance_tr_nn_std = performance_tr_nn_std / REPEATS
overall_performance_te_nn = performance_te_nn / REPEATS
overall_performance_te_nn_std = performance_te_nn_std / REPEATS
overall_performance_tr_rf = performance_tr_rf / REPEATS
overall_performance_tr_rf_std = performance_tr_rf_std / REPEATS
overall_performance_te_rf = performance_te_rf / REPEATS
overall_performance_te_rf_std = performance_te_rf_std / REPEATS
# Run deterministic algorithms:
print("Support Vector Regressor run 1/1 ... ")
# Convergence of the SVR does not always occur (even adjusting tolerance and iterations does not help)
regr_train_y, regr_test_y = learner.\
support_vector_regression_without_kernel(selected_train_X, train_y, selected_test_X, gridsearch=False)
mean_tr, std_tr = evaluation.mean_squared_error_with_std(train_y, regr_train_y)
mean_te, std_te = evaluation.mean_squared_error_with_std(test_y, regr_test_y)
performance_tr_svm = mean_tr
performance_tr_svm_std = std_tr
performance_te_svm = mean_te
performance_te_svm_std = std_te
print("Training Nearest Neighbor run 1/1 ... ")
regr_train_y, regr_test_y = learner.k_nearest_neighbor(selected_train_X, train_y, selected_test_X,
gridsearch=True)
mean_tr, std_tr = evaluation.mean_squared_error_with_std(train_y, regr_train_y)
mean_te, std_te = evaluation.mean_squared_error_with_std(test_y, regr_test_y)
performance_tr_knn = mean_tr
performance_tr_knn_std = std_tr
performance_te_knn = mean_te
performance_te_knn_std = std_te
print("Training Decision Tree run 1/1 ... ")
regr_train_y, regr_test_y = learner.\
decision_tree(selected_train_X, train_y, selected_test_X, gridsearch=True,
export_tree_path=EXPORT_TREE_PATH)
mean_tr, std_tr = evaluation.mean_squared_error_with_std(train_y, regr_train_y)
mean_te, std_te = evaluation.mean_squared_error_with_std(test_y, regr_test_y)
performance_tr_dt = mean_tr
performance_tr_dt_std = std_tr
performance_te_dt = mean_te
performance_te_dt_std = std_te
scores_with_sd = [(overall_performance_tr_nn, overall_performance_tr_nn_std, overall_performance_te_nn,
overall_performance_te_nn_std),
(overall_performance_tr_rf, overall_performance_tr_rf_std, overall_performance_te_rf,
overall_performance_te_rf_std),
(performance_tr_svm, performance_tr_svm_std, performance_te_svm, performance_te_svm_std),
(performance_tr_knn, performance_tr_knn_std, performance_te_knn, performance_te_knn_std),
(performance_tr_dt, performance_tr_dt_std, performance_te_dt, performance_te_dt_std)]
util.print_table_row_performances_regression(feature_names[i], scores_with_sd)
scores_over_all_algs.append(scores_with_sd)
# Plot the results
DataViz.plot_performances_regression(['NN', 'RF', 'SVM', 'KNN', 'DT'], feature_names, scores_over_all_algs)
if FLAGS.mode == 'detail' or FLAGS.mode == 'all':
print('\n- - - Running visualization of results - - -')
regr_train_y, regr_test_y = learner.random_forest(train_X[features_after_chapter_5], train_y,
test_X[features_after_chapter_5], gridsearch=False,
print_model_details=True)
DataViz.plot_numerical_prediction_versus_real(train_X.index, train_y, regr_train_y, test_X.index, test_y,
regr_test_y, 'heart rate')
feature_names = [
'initial set', 'Chapter 3', 'Chapter 4', 'Chapter 5', 'Selected features'
]
# Let us first study whether the time series is stationary and what the autocorrelations are.
dftest = adfuller(dataset['Azimuth'], autolag='AIC')
print dftest
autocorrelation_plot(dataset['Azimuth'])
plot.show()
# Now let us focus on the learning part.
learner = TemporalRegressionAlgorithms()
eval = RegressionEvaluation()
# We repeat the experiment a number of times to get a bit more robust data as the initialization of the NN is random.
repeats = 5
# we set a washout time to give the NN's the time to stabilize. We do not compute the error during the washout time.
washout_time = 10
scores_over_all_algs = []
for i in range(0, len(possible_feature_sets)):
selected_train_X = train_X[possible_feature_sets[i]]
selected_test_X = test_X[possible_feature_sets[i]]
def main():
# Read the result from the previous chapter and convert the index to datetime
try:
dataset = pd.read_csv(DATA_PATH / DATASET_FNAME, index_col=0)
dataset.index = pd.to_datetime(dataset.index)
except IOError as e:
print(
'File not found, try to run previous crowdsignals scripts first!')
raise e
# Create an instance of visualization class to plot the results
DataViz = VisualizeDataset(__file__)
# Consider the second task, namely the prediction of the heart rate. Therefore create a dataset with the heart
# rate as target and split using timestamps, because this is considered as a temporal task
print('\n- - - Loading dataset - - -')
prepare = PrepareDatasetForLearning()
train_X, test_X, train_y, test_y = prepare.split_single_dataset_regression_by_time(
dataset, 'hr_watch_rate', '2016-02-08 18:29:56', '2016-02-08 19:34:07',
'2016-02-08 20:07:50')
print('Training set length is: ', len(train_X.index))
print('Test set length is: ', len(test_X.index))
# Select subsets of the features
print('\n- - - Selecting subsets - - -')
basic_features = [
'acc_phone_x', 'acc_phone_y', 'acc_phone_z', 'acc_watch_x',
'acc_watch_y', 'acc_watch_z', 'gyr_phone_x', 'gyr_phone_y',
'gyr_phone_z', 'gyr_watch_x', 'gyr_watch_y', 'gyr_watch_z',
'labelOnTable', 'labelSitting', 'labelWashingHands', 'labelWalking',
'labelStanding', 'labelDriving', 'labelEating', 'labelRunning',
'light_phone_lux', 'mag_phone_x', 'mag_phone_y', 'mag_phone_z',
'mag_watch_x', 'mag_watch_y', 'mag_watch_z', 'press_phone_pressure'
]
pca_features = [
'pca_1', 'pca_2', 'pca_3', 'pca_4', 'pca_5', 'pca_6', 'pca_7'
]
time_features = [
name for name in dataset.columns
if ('temp_' in name and 'hr_watch' not in name)
]
freq_features = [
name for name in dataset.columns
if (('_freq' in name) or ('_pse' in name))
]
cluster_features = ['cluster']
print('#basic features: ', len(basic_features))
print('#PCA features: ', len(pca_features))
print('#time features: ', len(time_features))
print('#frequency features: ', len(freq_features))
print('#cluster features: ', len(cluster_features))
features_after_chapter_3 = list(set().union(basic_features, pca_features))
features_after_chapter_4 = list(set().union(features_after_chapter_3,
time_features, freq_features))
features_after_chapter_5 = list(set().union(features_after_chapter_4,
cluster_features))
selected_features = [
'temp_pattern_labelOnTable', 'labelOnTable',
'temp_pattern_labelOnTable(b)labelOnTable', 'cluster',
'pca_1_temp_mean_ws_120', 'pca_2_temp_mean_ws_120', 'pca_2',
'acc_watch_y_temp_mean_ws_120', 'gyr_watch_y_pse', 'gyr_watch_x_pse'
]
possible_feature_sets = [
basic_features, features_after_chapter_3, features_after_chapter_4,
features_after_chapter_5, selected_features
]
feature_names = [
'initial set', 'Chapter 3', 'Chapter 4', 'Chapter 5',
'Selected features'
]
if FLAGS.mode == 'correlation' or FLAGS.mode == 'all':
# First study whether the time series is stationary and what the autocorrelations are
adfuller(dataset['hr_watch_rate'], autolag='AIC')
plt.Figure()
autocorrelation_plot(dataset['hr_watch_rate'])
DataViz.save(plt)
plt.show()
# Now focus on the learning part
learner = TemporalRegressionAlgorithms()
evaluate = RegressionEvaluation()
if FLAGS.mode == 'overall' or FLAGS.mode == 'all':
# Repeat the experiment a number of times to get a bit more robust data as the initialization of e.g. the NN is
# random
repeats = FLAGS.repeats
# Set a washout time to give the NN's the time to stabilize (so don't compute the error during the washout time)
washout_time = FLAGS.washout
scores_over_all_algs = []
for i in range(0, len(possible_feature_sets)):
print(f'Evaluating for features {possible_feature_sets[i]}')
selected_train_X = train_X[possible_feature_sets[i]]
selected_test_X = test_X[possible_feature_sets[i]]
# First run non deterministic classifiers a number of times to average their score
performance_tr_res, performance_tr_res_std = 0, 0
performance_te_res, performance_te_res_std = 0, 0
performance_tr_rnn, performance_tr_rnn_std = 0, 0
performance_te_rnn, performance_te_rnn_std = 0, 0
for repeat in range(0, repeats):
print(f'--- run {repeat} ---')
regr_train_y, regr_test_y = learner.reservoir_computing(
selected_train_X,
train_y,
selected_test_X,
test_y,
gridsearch=True,
per_time_step=False)
mean_tr, std_tr = evaluate.mean_squared_error_with_std(
train_y.iloc[washout_time:, ],
regr_train_y.iloc[washout_time:, ])
mean_te, std_te = evaluate.mean_squared_error_with_std(
test_y.iloc[washout_time:, ],
regr_test_y.iloc[washout_time:, ])
performance_tr_res += mean_tr
performance_tr_res_std += std_tr
performance_te_res += mean_te
performance_te_res_std += std_te
regr_train_y, regr_test_y = learner.recurrent_neural_network(
selected_train_X,
train_y,
selected_test_X,
test_y,
gridsearch=True)
mean_tr, std_tr = evaluate.mean_squared_error_with_std(
train_y.iloc[washout_time:, ],
regr_train_y.iloc[washout_time:, ])
mean_te, std_te = evaluate.mean_squared_error_with_std(
test_y.iloc[washout_time:, ],
regr_test_y.iloc[washout_time:, ])
performance_tr_rnn += mean_tr
performance_tr_rnn_std += std_tr
performance_te_rnn += mean_te
performance_te_rnn_std += std_te
# Only apply the time series in case of the basis features
if feature_names[i] == 'initial set':
regr_train_y, regr_test_y = learner.time_series(
selected_train_X,
train_y,
selected_test_X,
test_y,
gridsearch=True)
mean_tr, std_tr = evaluate.mean_squared_error_with_std(
train_y.iloc[washout_time:, ],
regr_train_y.iloc[washout_time:, ])
mean_te, std_te = evaluate.mean_squared_error_with_std(
test_y.iloc[washout_time:, ],
regr_test_y.iloc[washout_time:, ])
overall_performance_tr_ts = mean_tr
overall_performance_tr_ts_std = std_tr
overall_performance_te_ts = mean_te
overall_performance_te_ts_std = std_te
else:
overall_performance_tr_ts = 0
overall_performance_tr_ts_std = 0
overall_performance_te_ts = 0
overall_performance_te_ts_std = 0
overall_performance_tr_res = performance_tr_res / repeats
overall_performance_tr_res_std = performance_tr_res_std / repeats
overall_performance_te_res = performance_te_res / repeats
overall_performance_te_res_std = performance_te_res_std / repeats
overall_performance_tr_rnn = performance_tr_rnn / repeats
overall_performance_tr_rnn_std = performance_tr_rnn_std / repeats
overall_performance_te_rnn = performance_te_rnn / repeats
overall_performance_te_rnn_std = performance_te_rnn_std / repeats
scores_with_sd = [
(overall_performance_tr_res, overall_performance_tr_res_std,
overall_performance_te_res, overall_performance_te_res_std),
(overall_performance_tr_rnn, overall_performance_tr_rnn_std,
overall_performance_te_rnn, overall_performance_te_rnn_std),
(overall_performance_tr_ts, overall_performance_tr_ts_std,
overall_performance_te_ts, overall_performance_te_ts_std)
]
util.print_table_row_performances_regression(
feature_names[i], scores_with_sd)
scores_over_all_algs.append(scores_with_sd)
DataViz.plot_performances_regression(
['Reservoir', 'RNN', 'Time series'], feature_names,
scores_over_all_algs)
if FLAGS.mode == 'detail' or FLAGS.mode == 'all':
regr_train_y, regr_test_y = learner.reservoir_computing(
train_X[features_after_chapter_5],
train_y,
test_X[features_after_chapter_5],
test_y,
gridsearch=False)
DataViz.plot_numerical_prediction_versus_real(
train_X.index, train_y, regr_train_y['hr_watch_rate'],
test_X.index, test_y, regr_test_y['hr_watch_rate'], 'heart rate')
regr_train_y, regr_test_y = learner.recurrent_neural_network(
train_X[basic_features],
train_y,
test_X[basic_features],
test_y,
gridsearch=True)
DataViz.plot_numerical_prediction_versus_real(
train_X.index, train_y, regr_train_y['hr_watch_rate'],
test_X.index, test_y, regr_test_y['hr_watch_rate'], 'heart rate')
regr_train_y, regr_test_y = learner.time_series(
train_X[basic_features],
train_y,
test_X[basic_features],
test_y,
gridsearch=True)
DataViz.plot_numerical_prediction_versus_real(
train_X.index, train_y, regr_train_y['hr_watch_rate'],
test_X.index, test_y, regr_test_y['hr_watch_rate'], 'heart rate')
if FLAGS.mode == 'dynamical' or FLAGS.mode == 'all':
# And now some example code for using the dynamical systems model with parameter tuning (note: focus on
# predicting accelerometer data):
train_X, test_X, train_y, test_y = prepare.split_single_dataset_regression(
copy.deepcopy(dataset), ['acc_phone_x', 'acc_phone_y'],
0.9,
filter_data=False,
temporal=True)
output_sets = learner. \
dynamical_systems_model_nsga_2(train_X, train_y, test_X, test_y,
['self.acc_phone_x', 'self.acc_phone_y', 'self.acc_phone_z'],
['self.a * self.acc_phone_x + self.b * self.acc_phone_y',
'self.c * self.acc_phone_y + self.d * self.acc_phone_z',
'self.e * self.acc_phone_x + self.f * self.acc_phone_z'],
['self.acc_phone_x', 'self.acc_phone_y'],
['self.a', 'self.b', 'self.c', 'self.d', 'self.e', 'self.f'],
pop_size=10, max_generations=10, per_time_step=True)
DataViz.plot_pareto_front(output_sets)
DataViz.plot_numerical_prediction_versus_real_dynsys_mo(
train_X.index, train_y, test_X.index, test_y, output_sets, 0,
'acc_phone_x')
regr_train_y, regr_test_y = learner. \
dynamical_systems_model_ga(train_X, train_y, test_X, test_y,
['self.acc_phone_x', 'self.acc_phone_y', 'self.acc_phone_z'],
['self.a * self.acc_phone_x + self.b * self.acc_phone_y',
'self.c * self.acc_phone_y + self.d * self.acc_phone_z',
'self.e * self.acc_phone_x + self.f * self.acc_phone_z'],
['self.acc_phone_x', 'self.acc_phone_y'],
['self.a', 'self.b', 'self.c', 'self.d', 'self.e', 'self.f'],
pop_size=5, max_generations=10, per_time_step=True)
DataViz.plot_numerical_prediction_versus_real(
train_X.index, train_y['acc_phone_x'], regr_train_y['acc_phone_x'],
test_X.index, test_y['acc_phone_x'], regr_test_y['acc_phone_x'],
'acc_phone_x')
regr_train_y, regr_test_y = learner. \
dynamical_systems_model_sa(train_X, train_y, test_X, test_y,
['self.acc_phone_x', 'self.acc_phone_y', 'self.acc_phone_z'],
['self.a * self.acc_phone_x + self.b * self.acc_phone_y',
'self.c * self.acc_phone_y + self.d * self.acc_phone_z',
'self.e * self.acc_phone_x + self.f * self.acc_phone_z'],
['self.acc_phone_x', 'self.acc_phone_y'],
['self.a', 'self.b', 'self.c', 'self.d', 'self.e', 'self.f'],
max_generations=10, per_time_step=True)
DataViz.plot_numerical_prediction_versus_real(
train_X.index, train_y['acc_phone_x'], regr_train_y['acc_phone_x'],
test_X.index, test_y['acc_phone_x'], regr_test_y['acc_phone_x'],
'acc_phone_x')