def test_esn_classifier_no_valid_params() -> None:
X, y = load_digits(return_X_y=True, as_sequence=True)
with pytest.raises(TypeError):
ESNClassifier(input_to_node=ESNRegressor()).fit(X, y)
with pytest.raises(TypeError):
ESNClassifier(node_to_node=ESNRegressor()).fit(X, y)
with pytest.raises(TypeError):
ESNClassifier(input_to_node=ESNRegressor()).fit(X, y)
with pytest.raises(ValueError):
ESNClassifier(requires_sequence="True").fit(X, y)
with pytest.raises(TypeError):
ESNClassifier(regressor=InputToNode()).fit(X, y)
def test_esn_regressor_requires_sequence() -> None:
print('\ntest_esn_regressor_requires_sequence():')
X, y = mackey_glass(n_timesteps=8000)
X_train = np.empty(shape=(10, ), dtype=object)
y_train = np.empty(shape=(10, ), dtype=object)
X_test = np.empty(shape=(10, ), dtype=object)
y_test = np.empty(shape=(10, ), dtype=object)
splitter = TimeSeriesSplit(n_splits=10)
for k, (train_index, test_index) in enumerate(splitter.split(X, y)):
X_train[k] = X[train_index].reshape(-1, 1)
y_train[k] = y[train_index]
X_test[k] = X[test_index].reshape(-1, 1)
y_test[k] = y[test_index]
param_grid = {
'hidden_layer_size': [20, 50],
'input_scaling': [1.],
'bias_scaling': [10.],
'input_activation': ['identity'],
'random_state': [42],
'spectral_radius': [0.],
'reservoir_activation': ['tanh'],
'alpha': [1e-2, 1e-5],
}
esn = GridSearchCV(ESNRegressor(),
param_grid,
scoring=make_scorer(mean_squared_error,
greater_is_better=False))
esn.fit(X_train, y_train, n_jobs=2)
np.testing.assert_equal(esn.best_estimator_.requires_sequence, True)
def test_esn_regressor_jobs() -> None:
print('\ntest_esn_regressor_jobs():')
X, y = mackey_glass(n_timesteps=8000)
X_train, X_test, y_train, y_test = train_test_split(X, y, shuffle=False)
param_grid = {
"input_to_node": [
InputToNode(bias_scaling=.1,
hidden_layer_size=10,
input_activation='identity',
random_state=42),
InputToNode(bias_scaling=.1,
hidden_layer_size=50,
input_activation='identity',
random_state=42)
],
"node_to_node": [
NodeToNode(spectral_radius=0.,
hidden_layer_size=10,
random_state=42),
NodeToNode(spectral_radius=1,
hidden_layer_size=50,
random_state=42)
],
"regressor":
[IncrementalRegression(alpha=.0001),
IncrementalRegression(alpha=.01)],
'random_state': [42]
}
esn = GridSearchCV(estimator=ESNRegressor(), param_grid=param_grid)
esn.fit(X_train.reshape(-1, 1), y_train, n_jobs=2)
y_esn = esn.predict(X_test.reshape(-1, 1))
print("tests - esn:\n sin | cos \n {0}".format(y_test - y_esn))
print("best_params_: {0}".format(esn.best_params_))
print("best_score: {0}".format(esn.best_score_))
np.testing.assert_allclose(1, esn.best_score_, atol=1e-1)
def test_esn_regressor_chunk():
print('\ntest_elm_regressor_chunk():')
X = np.linspace(0, 10, 2000)
y = np.hstack((np.sin(X).reshape(-1, 1), np.cos(X).reshape(-1, 1)))
X_train, X_test, y_train, y_test = train_test_split(X,
y,
test_size=10,
random_state=42)
param_grid = {
'input_to_node__hidden_layer_size': [20, 50],
'input_to_node__input_scaling': [1.],
'input_to_node__bias_scaling': [10.],
'input_to_node__activation': ['identity'],
'input_to_node__random_state': [42],
'node_to_node__hidden_layer_size': [20, 50],
'node_to_node__spectral_radius': [0.],
'node_to_node__bias_scaling': [0.],
'node_to_node__activation': ['tanh'],
'node_to_node__random_state': [42],
'chunk_size': [500],
'regressor__alpha': [1e-2, 1e-5],
'random_state': [42]
}
esn = GridSearchCV(ESNRegressor(), param_grid)
esn.fit(X_train.reshape(-1, 1), y_train, n_jobs=2)
y_esn = esn.predict(X_test.reshape(-1, 1))
print("tests - esn:\n sin | cos \n {0}".format(y_test - y_esn))
print("best_params_: ".format(esn.best_params_))
print("best_score: ".format(esn.best_score_))
np.testing.assert_allclose(y_test, y_esn, atol=1e-1)
def train_esn(base_input_to_node, base_node_to_node, base_reg, frame_length,
file_list):
print(frame_length)
try:
load("models/esn_500u_" + str(frame_length) + ".joblib")
except FileNotFoundError:
input_to_node = clone(base_input_to_node)
node_to_node = clone(base_node_to_node)
reg = clone(base_reg)
esn = ESNRegressor(input_to_nodes=[('default', input_to_node)],
nodes_to_nodes=[('default', node_to_node)],
regressor=reg,
random_state=0)
for file in file_list[:7]:
X, y = extract_features(file,
sr=4000.,
frame_length=frame_length,
target_widening=True)
esn.partial_fit(X=X, y=y, update_output_weights=False)
X, y = extract_features(file_list[8],
sr=4000.,
frame_length=frame_length,
target_widening=True)
esn.partial_fit(X=X, y=y, update_output_weights=True)
dump(esn, "esn_500b_" + str(frame_length) + ".joblib")
def test_esn_regressor_requires_no_sequence() -> None:
print('\ntest_esn_regressor_requires_sequence():')
X, y = mackey_glass(n_timesteps=8000)
X_train, X_test, y_train, y_test = train_test_split(X,
y,
test_size=10,
random_state=42)
param_grid = {
'hidden_layer_size': [20, 50],
'input_scaling': [1.],
'bias_scaling': [10.],
'input_activation': ['identity'],
'random_state': [42],
'spectral_radius': [0.],
'reservoir_activation': ['tanh'],
'alpha': [1e-2, 1e-5]
}
esn = GridSearchCV(ESNRegressor(), param_grid)
esn.fit(X_train.reshape(-1, 1), y_train, n_jobs=2)
np.testing.assert_equal(esn.best_estimator_.requires_sequence, False)
def test_esn_regressor_jobs():
print('\ntest_esn_regressor_jobs():')
X = np.linspace(0, 10, 2000)
y = np.hstack((np.sin(X).reshape(-1, 1), np.cos(X).reshape(-1, 1)))
X_train, X_test, y_train, y_test = train_test_split(X,
y,
test_size=10,
random_state=42)
param_grid = {
'input_to_node': [[('default',
InputToNode(bias_scaling=10.,
hidden_layer_size=10,
activation='identity',
random_state=42))],
[('default',
InputToNode(bias_scaling=10.,
hidden_layer_size=50,
activation='identity',
random_state=42))]],
'node_to_node': [[('default',
NodeToNode(spectral_radius=0.,
bias_scaling=0.,
hidden_layer_size=10,
random_state=42))],
[('default',
NodeToNode(spectral_radius=0.,
bias_scaling=0.,
hidden_layer_size=50,
random_state=42))]],
'regressor':
[IncrementalRegression(alpha=.0001),
IncrementalRegression(alpha=.01)],
'random_state': [42]
}
esn = GridSearchCV(ESNRegressor(), param_grid)
esn.fit(X_train.reshape(-1, 1), y_train, n_jobs=2)
y_esn = esn.predict(X_test.reshape(-1, 1))
print("tests - esn:\n sin | cos \n {0}".format(y_test - y_esn))
print("best_params_: ".format(esn.best_params_))
print("best_score: ".format(esn.best_score_))
np.testing.assert_allclose(y_test, y_esn, atol=1e-1)
def test_esn_regressor_wrong_sequence_format() -> None:
print('\ntest_esn_regressor_requires_sequence():')
X, y = mackey_glass(n_timesteps=8000)
X_train = np.empty(shape=(10, 1000, 1))
y_train = np.empty(shape=(10, 1000, 1))
splitter = TimeSeriesSplit(n_splits=10)
for k, (train_index, test_index) in enumerate(splitter.split(X, y)):
X_train[k, :, :] = X[:1000].reshape(-1, 1)
y_train[k, :, :] = y[:1000].reshape(-1, 1)
param_grid = {
'hidden_layer_size': 50,
'input_scaling': 1.,
'bias_scaling': 10.,
'input_activation': 'identity',
'random_state': 42,
'spectral_radius': 0.,
'reservoir_activation': 'tanh',
'alpha': 1e-5
}
with pytest.raises(ValueError):
ESNRegressor(verbose=True, **param_grid)\
.fit(X_train, y_train, n_jobs=2)
train_len = 3000
future_len = 1
future_total = len(prices) - train_len
# Echo State Network preparation
# In[7]:
base_input_to_nodes = InputToNode(hidden_layer_size=100, activation='identity', k_in=1, input_scaling=0.6, bias_scaling=0.0)
base_nodes_to_nodes = NodeToNode(hidden_layer_size=100, spectral_radius=0.9, leakage=1.0, bias_scaling=0.0, k_rec=10)
esn = ESNRegressor(input_to_node=base_input_to_nodes,
node_to_node=base_nodes_to_nodes,
regressor=IncrementalRegression(alpha=1e-8), random_state=10)
# Training and Prediction.
# In[8]:
train_in = prices[0:train_len, :]
train_out = prices[0+1:train_len+1, :]
test_in = prices[0:train_len+future_total - future_len, :]
test_out = prices[future_len:train_len+future_total, :]
esn.fit(X=train_in, y=train_out.ravel())
train_pred = esn.predict(X=train_in)
plt.ylabel('X[n]')
plt.grid()
plt.show()
# At first, show the impact of different input scaling factors.
#
# Therefore, we neutralize the other hyper-parameters, i.e., no recurrent connections ($\rho = 0$), no bias ($\alpha_{\mathrm{b}} = 0$) and no leakage ($\lambda = 1$).
# In[3]:
esn = ESNRegressor(input_to_node=InputToNode(hidden_layer_size=50,
activation='identity',
k_in=1,
input_scaling=0.1,
bias_scaling=0.0),
node_to_node=NodeToNode(hidden_layer_size=50,
spectral_radius=0.0,
leakage=1.0,
bias_scaling=0.0,
k_rec=10),
regressor=Ridge(alpha=1e-6),
random_state=10)
esn.fit(X=X, y=y)
_ = esn.predict(X=X)
# Visualizing this, we can see exactly what we might expect. We have chosen an input scaling factor of 0.1. Thus, the reservoir state is non-zero for exactly one sample. We can see that all reservoir states are zero all the times except for $n=5$, when the impulse is fed into the ESN.
#
# The absolute values of the reservoir states lie between 0 and 0.1.
# In[5]:
"""
base_input_to_node = InputToNode(hidden_layer_size=500,
activation='identity',
k_in=5,
input_scaling=14.6,
bias_scaling=0.0,
random_state=1)
base_node_to_node = NodeToNode(hidden_layer_size=500,
spectral_radius=0.8,
leakage=0.5,
bias_scaling=0.5,
k_rec=16,
bi_directional=True,
random_state=1)
base_reg = FastIncrementalRegression(alpha=1.7e-10)
base_esn = ESNRegressor(input_to_node=[('default', base_input_to_node)],
node_to_node=[('default', base_node_to_node)],
regressor=base_reg,
random_state=0)
esn = base_esn
t1 = time.time()
Parallel(n_jobs=1,
verbose=50)(delayed(train_esn)(base_input_to_node, base_node_to_node,
base_reg, frame_length, all_wavs_m)
for frame_length in [5, 7, 9, 11, 21, 31, 41, 81])
print("Finished in {0} seconds!".format(time.time() - t1))
exit(0)
# u[n]| _ _ _ _ _ _ _|r'[n]|_ _ _ _ _ _ _|r[n] | _ _ _ _ _ _ _ |
# U R_i2n R_n2n |
# Initialize, fit and apply NodeToOutput
y_pred = Ridge().fit(R_n2n, y).predict(R_n2n)
print(y_pred.shape)
# Predicting the Mackey-Glass equation
# Load the dataset
X, y = mackey_glass(n_timesteps=5000)
# Define Train/Test lengths
trainLen = 1900
X_train, y_train = X[:trainLen], y[:trainLen]
X_test, y_test = X[trainLen:], y[trainLen:]
# Initialize and train an ELMRegressor and an ESNRegressor
esn = ESNRegressor().fit(X=X_train.reshape(-1, 1), y=y_train)
elm = ELMRegressor(regressor=skRidge()).fit(X=X_train.reshape(-1, 1),
y=y_train)
print("Fitted models")
# Build Reservoir Computing Networks with PyRCN
U, y = make_blobs(n_samples=100, n_features=10)
# Vanilla ELM for regression tasks with input_scaling
# _ _ _ _ _ _ _ _ _ _ _ _ _ _
# | | | |
# ----|Input-to-Node |-----|Node-to-Output |------
# u[n]| _ _ _ _ _ _ _|r'[n]| _ _ _ _ _ _ _ |y[n]
# y_pred
#
vanilla_elm = ELMRegressor(input_scaling=0.9)
# Visualization
fix, axs = plt.subplots()
sns.lineplot(data=X_train.ravel(), ax=axs)
sns.lineplot(data=y_train.ravel(), ax=axs)
axs.set_xlim([0, 1900])
axs.set_xlabel('n')
axs.set_ylabel('u[n]')
plt.legend(["Input", "Target"])
plt.show()
# Training and Prediction using vanilla ESNs and ELMs
# In[ ]:
# initialize an ESNRegressor
esn = ESNRegressor()
# initialize an ELMRegressor
elm = ELMRegressor(regressor=Ridge())
# train a model
esn.fit(X=X_train.reshape(-1, 1), y=y_train)
elm.fit(X=X_train.reshape(-1, 1), y=y_train)
# evaluate the models
y_test_pred = esn.predict(X=X_test)
print(mean_squared_error(y_test, y_test_pred))
y_test_pred = elm.predict(X=X_test)
print(mean_squared_error(y_test, y_test_pred))
# Hyperparameter optimization ESN
print(len(all_wavs_m))
all_wavs_n = glob.glob(r"C:\Temp\SpLxDataLondonStudents2008\N\*.wav")
print(len(all_wavs_n))
base_input_to_node = InputToNode(hidden_layer_size=500,
input_activation='identity',
k_in=5,
input_scaling=14.6,
bias_scaling=0.0,
random_state=1)
base_node_to_node = NodeToNode(hidden_layer_size=500,
spectral_radius=0.8,
leakage=0.5,
k_rec=16,
bidirectional=True,
random_state=1)
base_reg = IncrementalRegression(alpha=1.7e-10)
base_esn = ESNRegressor(input_to_node=base_input_to_node,
node_to_node=base_node_to_node,
regressor=base_reg)
esn = base_esn
t1 = time.time()
Parallel(n_jobs=1, verbose=50)(delayed(train_esn)(
base_input_to_node, base_node_to_node, base_reg, frame_length, all_wavs_m)
for frame_length in [7, 9, 11, 21, 31, 41, 81])
print("Finished in {0} seconds!".format(time.time() - t1))
exit(0)
'scoring': gpe_scorer
}
kwargs_step4 = {
'n_iter': 50,
'random_state': 42,
'verbose': 1,
'n_jobs': -1,
'scoring': gpe_scorer
}
searches = [('step1', RandomizedSearchCV, step1_esn_params, kwargs_step1),
('step2', RandomizedSearchCV, step2_esn_params, kwargs_step2),
('step3', RandomizedSearchCV, step3_esn_params, kwargs_step3),
('step4', RandomizedSearchCV, step4_esn_params, kwargs_step4)]
base_esn = ESNRegressor(input_to_node=input_to_node).set_params(
**initially_fixed_params)
try:
sequential_search = load("../f0/sequential_search_f0_mel_km_500.joblib")
except FileNotFoundError:
print(FileNotFoundError)
sequential_search = SequentialSearchCV(base_esn, searches=searches).fit(
X_train, y_train)
dump(sequential_search, "../f0/sequential_search_f0_mel_km_500.joblib")
print(sequential_search)
param_grid = {
'hidden_layer_size': [
50, 100, 200, 400, 500, 800, 1000, 1600, 2000, 3200, 4000, 6400, 8000,
16000
# In[7]:
base_input_to_nodes = InputToNode(hidden_layer_size=100,
activation='identity',
k_in=1,
input_scaling=0.6,
bias_scaling=0.0)
base_nodes_to_nodes = NodeToNode(hidden_layer_size=100,
spectral_radius=0.9,
leakage=1.0,
bias_scaling=0.0,
k_rec=10)
esn = ESNRegressor(input_to_node=base_input_to_nodes,
node_to_node=base_nodes_to_nodes,
regressor=IncrementalRegression(alpha=1e-8),
random_state=10)
# Training and Prediction.
# In[8]:
X_train = scaler.transform(X[0:train_len])
y_train = scaler.transform(X[1:train_len + 1])
X_test = scaler.transform(X[train_len + 1:-1])
y_test = scaler.transform(X[train_len + 1 + future_len:])
fig = plt.figure()
plt.plot(scaler.transform(X.reshape(-1, 1)))
plt.xlabel("n")
plt.xlim([0, len(X)])
X_test = X[trainLen:trainLen + testLen]
y_test = y[trainLen:trainLen + testLen]
fig = plt.figure()
plt.plot(X_train, label="Training input")
plt.plot(y_train, label="Training target")
plt.xlabel("n")
plt.xlim([0, 200])
plt.ylabel("u[n]")
plt.grid()
plt.legend()
fig.set_size_inches(4, 2.5)
plt.savefig('input_data.pdf', bbox_inches='tight', pad_inches=0)
# initialize an ESNRegressor
esn = ESNRegressor() # IncrementalRegression()
# initialize an ELMRegressor
elm = ELMRegressor(regressor=Ridge()) # Ridge()
# train a model
esn.fit(X=X_train, y=y_train)
elm.fit(X=X_train, y=y_train)
# evaluate the models
y_test_pred = esn.predict(X=X_test)
print(mean_squared_error(y_test, y_test_pred))
y_test_pred = elm.predict(X=X_test)
print(mean_squared_error(y_test, y_test_pred))
# create a unit impulse to record the impulse response of the reservoir