def train_ESN(X, Y, Xte, Yte, embedding_method, n_dim, w_ridge, n_internal_units=None, spectral_radius=None, connectivity=None, input_scaling=None, noise_level=None, reservoir=None): num_classes = Y.shape[1] # Initialize reservoir if reservoir is None: reservoir = Reservoir(n_internal_units, spectral_radius, connectivity, input_scaling, noise_level) elif n_internal_units is None \ or spectral_radius is None \ or connectivity is None \ or input_scaling is None \ or noise_level is None: raise RuntimeError('Reservoir parameters missing') # Initialize timer time_tr_start = time.time() # Compute reservoir states res_states = reservoir.get_states(X, embedding=embedding_method, n_dim=n_dim, train=True, bidir=False) # Readout training readout = Ridge(alpha=w_ridge) readout.fit(res_states, Y) training_time = (time.time() - time_tr_start) / 60 # Test res_states_te = reservoir.get_states(Xte, embedding=embedding_method, n_dim=n_dim, train=False, bidir=False) logits = readout.predict(res_states_te) pred_class = np.argmax(logits, axis=1) true_class = np.argmax(Yte, axis=1) accuracy = accuracy_score(true_class, pred_class) if num_classes > 2: f1 = f1_score(true_class, pred_class, average='weighted') else: f1 = f1_score(true_class, pred_class, average='binary') return accuracy, f1, training_time
def echo_state_network(data_matrix, label, reservoir): # Initialize reservoir if reservoir is None: reservoir = Reservoir(n_internal_units, spectral_radius, connectivity, input_scaling, noise_level) res_states = reservoir.get_states(data_matrix, embedding=embedding_method, n_dim=None, train=True, bidir=False) # Readout training readout = Ridge(alpha=w_ridge) readout.fit(res_states, label) return reservoir, readout
def train_BDESN(X, Y, Xte, Yte, embedding_method, n_dim, fc_layout, batch_size, num_epochs, p_drop, w_l2, learning_rate, seed=None, n_internal_units=None, spectral_radius=None, connectivity=None, input_scaling=None, noise_level=None, reservoir=None): num_classes = Y.shape[1] # Compute reservoir states if reservoir is None: reservoir = Reservoir(n_internal_units, spectral_radius, connectivity, input_scaling, noise_level) elif n_internal_units is None \ or spectral_radius is None \ or connectivity is None \ or input_scaling is None \ or noise_level is None: raise RuntimeError('Reservoir parameters missing') res_states = reservoir.get_states(X, embedding=embedding_method, n_dim=n_dim, train=True, bidir=True) res_states_te = reservoir.get_states(Xte, embedding=embedding_method, n_dim=n_dim, train=False, bidir=True) n_data, input_size = res_states.shape graph = tf.Graph() with graph.as_default(): if seed is not None: tf.set_random_seed(seed) # ============= MLP ============== # tf Graph input nn_input = tf.placeholder(shape=(None, input_size), dtype=tf.float32) nn_output = tf.placeholder(shape=(None, num_classes), dtype=tf.float32) keep_prob = tf.placeholder(dtype=tf.float32) # MLP logits = build_network(nn_input, input_size, fc_layout, num_classes, keep_prob) loss_track, pred_class, training_time = \ train_tf_model('BDESN', graph, res_states, Y, res_states_te, Yte, n_data, batch_size, num_epochs, nn_input, keep_prob, logits, nn_output, w_l2, learning_rate, p_drop) true_class = np.argmax(Yte, axis=1) accuracy = accuracy_score(true_class, pred_class) if num_classes > 2: f1 = f1_score(true_class, pred_class, average='weighted') else: f1 = f1_score(true_class, pred_class, average='binary') return loss_track, accuracy, f1, training_time
class RC_model(object): def __init__( self, # reservoir reservoir=None, n_internal_units=None, spectral_radius=None, leak=None, connectivity=None, input_scaling=None, noise_level=None, n_drop=None, bidir=False, circle=False, # dim red dimred_method=None, n_dim=None, # representation mts_rep=None, w_ridge_embedding=None, # readout readout_type=None, w_ridge=None, mlp_layout=None, num_epochs=None, w_l2=None, nonlinearity=None, svm_gamma=1.0, svm_C=1.0, ): """ Build and evaluate a RC-based classifier. The training and test MTS are multidimensional arrays of shape [N,T,V], with - N = number of samples - T = number of time steps in each sample - V = number of variables in each sample Training and test labels have shape [N,C], with C the number of classes The dataset consists of: X, Y = training data and respective labels Xte, Yte = test data and respective labels Reservoir parameters: reservoir = precomputed reservoir (oject of class 'Reservoir'); if None, the following structural hyperparameters must be specified n_internal_units = processing units in the reservoir spectral_radius = largest eigenvalue of the reservoir matrix of connection weights leak = amount of leakage in the reservoir state update (optional) connectivity = percentage of nonzero connection weights input_scaling = scaling of the input connection weights noise_level = deviation of the Gaussian noise injected in the state update n_drop = number of transient states to drop bidir = use a bidirectional reservoir (True or false) Dimensionality reduction parameters: dimred_method = procedure for reducing the number of features in the sequence of reservoir states; possible options are: None (no dimensionality reduction), 'pca' or 'tenpca' n_dim = number of resulting dimensions after the dimensionality reduction procedure Representation parameters: mts_rep = type of MTS representation. It can be 'last' (last state), 'output' (output model space), or 'reservoir' (reservoir model space) w_ridge_embedding = regularization parameter of the ridge regression in the output model space and reservoir model space representation; ignored if mts_rep == None Readout parameters: readout_type = type of readout used for classification. It can be 'lin' (ridge regression), 'mlp' (multiplayer perceptron), 'svm' (support vector machine), or None. If None, the input representations will be saved instead: this is useful for clustering and visualization. w_ridge = regularization parameter of the ridge regression readout (only for readout_type=='lin') mlp_layout = tuple with the sizes of MLP layers, e.g. (20, 10) defines a MLP with 2 layers of 20 and 10 units respectively. (only for readout_type=='mlp') num_epochs = number of iterations during the optimization (only for readout_type=='mlp') w_l2 = weight of the L2 regularization (only for readout_type=='mlp') nonlinearity = type of activation function {'relu', 'tanh', 'logistic', 'identity'} (only for readout_type=='mlp') svm_gamma = bandwith of the RBF kernel (only for readout_type=='svm') svm_C = regularization for SVM hyperplane (only for readout_type=='svm') """ self.n_drop = n_drop self.bidir = bidir self.dimred_method = dimred_method self.mts_rep = mts_rep self.readout_type = readout_type self.svm_gamma = svm_gamma # Initialize reservoir if reservoir is None: self._reservoir = Reservoir(n_internal_units=n_internal_units, spectral_radius=spectral_radius, leak=leak, connectivity=connectivity, input_scaling=input_scaling, noise_level=noise_level, circle=circle) else: self._reservoir = reservoir # Initialize dimensionality reduction method if dimred_method is not None: if dimred_method.lower() == 'pca': self._dim_red = PCA(n_components=n_dim) elif dimred_method.lower() == 'tenpca': self._dim_red = tensorPCA(n_components=n_dim) else: raise RuntimeError('Invalid dimred method ID') # Initialize ridge regression model if mts_rep == 'output' or mts_rep == 'reservoir': self._ridge_embedding = Ridge(alpha=w_ridge_embedding, fit_intercept=True) # Initialize readout type if self.readout_type is not None: if self.readout_type == 'lin': # Ridge regression self.readout = Ridge(alpha=w_ridge) elif self.readout_type == 'svm': # SVM readout self.readout = SVC(C=svm_C, kernel='precomputed') elif readout_type == 'mlp': # MLP (deep readout) # pass self.readout = MLPClassifier( hidden_layer_sizes=mlp_layout, activation=nonlinearity, alpha=w_l2, batch_size=32, learning_rate='adaptive', # 'constant' or 'adaptive' learning_rate_init=0.001, max_iter=num_epochs, early_stopping=False, # if True, set validation_fraction > 0 validation_fraction=0.0 # used for early stopping ) else: raise RuntimeError('Invalid readout type') def train(self, X, Y=None): time_start = time.time() # ============ Compute reservoir states ============ res_states = self._reservoir.get_states(X, n_drop=self.n_drop, bidir=self.bidir) # ============ Dimensionality reduction of the reservoir states ============ if self.dimred_method.lower() == 'pca': # matricize N_samples = res_states.shape[0] res_states = res_states.reshape(-1, res_states.shape[2]) # ..transform.. red_states = self._dim_red.fit_transform(res_states) # ..and put back in tensor form red_states = red_states.reshape(N_samples, -1, red_states.shape[1]) elif self.dimred_method.lower() == 'tenpca': red_states = self._dim_red.fit_transform(res_states) else: # Skip dimensionality reduction red_states = res_states # ============ Generate representation of the MTS ============ coeff_tr = [] biases_tr = [] # Output model space representation if self.mts_rep == 'output': if self.bidir: X = np.concatenate((X, X[:, ::-1, :]), axis=2) for i in range(X.shape[0]): self._ridge_embedding.fit(red_states[i, 0:-1, :], X[i, self.n_drop + 1:, :]) coeff_tr.append(self._ridge_embedding.coef_.ravel()) biases_tr.append(self._ridge_embedding.intercept_.ravel()) input_repr = np.concatenate( (np.vstack(coeff_tr), np.vstack(biases_tr)), axis=1) # Reservoir model space representation elif self.mts_rep == 'reservoir': for i in range(X.shape[0]): self._ridge_embedding.fit(red_states[i, 0:-1, :], red_states[i, 1:, :]) coeff_tr.append(self._ridge_embedding.coef_.ravel()) biases_tr.append(self._ridge_embedding.intercept_.ravel()) input_repr = np.concatenate( (np.vstack(coeff_tr), np.vstack(biases_tr)), axis=1) # Last state representation elif self.mts_rep == 'last': input_repr = red_states[:, -1, :] # Mean state representation elif self.mts_rep == 'mean': input_repr = np.mean(red_states, axis=1) else: raise RuntimeError('Invalid representation ID') # ============ Apply readout ============ if self.readout_type == None: # Just store the input representations self.input_repr = input_repr elif self.readout_type == 'lin': # Ridge regression self.readout.fit(input_repr, Y) elif self.readout_type == 'svm': # SVM readout Ktr = squareform(pdist(input_repr, metric='sqeuclidean')) Ktr = np.exp(-self.svm_gamma * Ktr) self.readout.fit(Ktr, np.argmax(Y, axis=1)) self.input_repr_tr = input_repr # store them to build test kernel elif self.readout_type == 'mlp': # MLP (deep readout) self.readout.fit(input_repr, Y) tot_time = (time.time() - time_start) / 60 return tot_time def test(self, Xte, Yte): # ============ Compute reservoir states ============ res_states_te = self._reservoir.get_states(Xte, n_drop=self.n_drop, bidir=self.bidir) # ============ Dimensionality reduction of the reservoir states ============ if self.dimred_method.lower() == 'pca': # matricize N_samples_te = res_states_te.shape[0] res_states_te = res_states_te.reshape(-1, res_states_te.shape[2]) # ..transform.. red_states_te = self._dim_red.transform(res_states_te) # ..and put back in tensor form red_states_te = red_states_te.reshape(N_samples_te, -1, red_states_te.shape[1]) elif self.dimred_method.lower() == 'tenpca': red_states_te = self._dim_red.transform(res_states_te) else: # Skip dimensionality reduction red_states_te = res_states_te # ============ Generate representation of the MTS ============ coeff_te = [] biases_te = [] # Output model space representation if self.mts_rep == 'output': if self.bidir: Xte = np.concatenate((Xte, Xte[:, ::-1, :]), axis=2) for i in range(Xte.shape[0]): self._ridge_embedding.fit(red_states_te[i, 0:-1, :], Xte[i, self.n_drop + 1:, :]) coeff_te.append(self._ridge_embedding.coef_.ravel()) biases_te.append(self._ridge_embedding.intercept_.ravel()) input_repr_te = np.concatenate( (np.vstack(coeff_te), np.vstack(biases_te)), axis=1) # Reservoir model space representation elif self.mts_rep == 'reservoir': for i in range(Xte.shape[0]): self._ridge_embedding.fit(red_states_te[i, 0:-1, :], red_states_te[i, 1:, :]) coeff_te.append(self._ridge_embedding.coef_.ravel()) biases_te.append(self._ridge_embedding.intercept_.ravel()) input_repr_te = np.concatenate( (np.vstack(coeff_te), np.vstack(biases_te)), axis=1) # Last state representation elif self.mts_rep == 'last': input_repr_te = red_states_te[:, -1, :] # Mean state representation elif self.mts_rep == 'mean': input_repr_te = np.mean(red_states_te, axis=1) else: raise RuntimeError('Invalid representation ID') # ============ Apply readout ============ if self.readout_type == 'lin': # Ridge regression logits = self.readout.predict(input_repr_te) pred_class = np.argmax(logits, axis=1) elif self.readout_type == 'svm': # SVM readout Kte = cdist(input_repr_te, self.input_repr_tr, metric='sqeuclidean') Kte = np.exp(-self.svm_gamma * Kte) pred_class = self.readout.predict(Kte) elif self.readout_type == 'mlp': # MLP (deep readout) pred_class = self.readout.predict(input_repr_te) pred_class = np.argmax(pred_class, axis=1) accuracy, f1 = compute_test_scores(pred_class, Yte) return accuracy, f1