def __init__(self, inputs, outputs, numHiddenNeurons, activationFunction, LN=True, outputWeightForgettingFactor=0.999, inputWeightForgettingFactor=0.999, AE=True, ORTH=False): self.activationFunction = activationFunction self.inputs = inputs self.outputs = outputs self.numHiddenNeurons = numHiddenNeurons # input to hidden weights self.inputWeights = np.random.random((self.numHiddenNeurons, self.inputs)) # bias of hidden units #self.bias = np.random.random((1, self.numHiddenNeurons)) * 2 - 1 self.bias = np.zeros([1,self.numHiddenNeurons]) # hidden to output layer connection self.beta = np.random.random((self.numHiddenNeurons, self.outputs)) # auxiliary matrix used for sequential learning self.M = None self.LN = LN self.AE = AE self.ORTH = ORTH self.forgettingFactor =outputWeightForgettingFactor self.FOSELM_AE = FOSELM(inputs= inputs, outputs = inputs, numHiddenNeurons= numHiddenNeurons, activationFunction= activationFunction, LN=True, forgettingFactor=inputWeightForgettingFactor, ORTH=self.ORTH)
def __init__(self, inputs, outputs, numHiddenNeurons, activationFunction, LN=True, AE=True, ORTH=True, inputWeightForgettingFactor=0.999, outputWeightForgettingFactor=0.999, hiddenWeightForgettingFactor=0.999): self.activationFunction = activationFunction self.inputs = inputs self.outputs = outputs self.numHiddenNeurons = numHiddenNeurons # input to hidden weights self.inputWeights = np.random.random( (self.numHiddenNeurons, self.inputs)) # hidden layer to hidden layer wieghts self.hiddenWeights = np.random.random( (self.numHiddenNeurons, self.numHiddenNeurons)) # initial hidden layer activation self.initial_H = np.random.random((1, self.numHiddenNeurons)) * 2 - 1 self.H = self.initial_H self.LN = LN self.AE = AE self.ORTH = ORTH # bias of hidden units self.bias = np.random.random((1, self.numHiddenNeurons)) * 2 - 1 # hidden to output layer connection self.beta = np.random.random((self.numHiddenNeurons, self.outputs)) # auxiliary matrix used for sequential learning self.M = inv(0.00001 * np.eye(self.numHiddenNeurons)) self.forgettingFactor = outputWeightForgettingFactor self.trace = 0 self.thresReset = 0.001 if self.AE: self.inputAE = FOSELM(inputs=inputs, outputs=inputs, numHiddenNeurons=numHiddenNeurons, activationFunction=activationFunction, LN=LN, forgettingFactor=inputWeightForgettingFactor, ORTH=ORTH) self.hiddenAE = FOSELM(inputs=numHiddenNeurons, outputs=numHiddenNeurons, numHiddenNeurons=numHiddenNeurons, activationFunction=activationFunction, LN=LN, ORTH=ORTH)
def __init__(self, inputs, outputs, numHiddenNeurons, activationFunction, min_t, max_t, LN=True, AE=True, ORTH=True, inputWeightForgettingFactor=0.999, outputWeightForgettingFactor=0.999, hiddenWeightForgettingFactor=0.999): self.min_t = min_t self.max_t = max_t self.activationFunction = activationFunction self.inputs = inputs self.outputs = outputs self.numHiddenNeurons = numHiddenNeurons # input to hidden weights self.inputWeights = np.random.random((self.numHiddenNeurons, self.inputs)) # hidden layer to hidden layer weights self.hiddenWeights = np.random.random((self.numHiddenNeurons, self.numHiddenNeurons)) # initial hidden layer activation self.initial_H = np.random.random((1, self.numHiddenNeurons)) * 2 - 1 self.H = self.initial_H self.LN = LN self.AE = AE self.ORTH = ORTH # bias of hidden units self.bias = np.random.random((1, self.numHiddenNeurons)) * 2 - 1 self.forgettingFactor = outputWeightForgettingFactor self.FFmin = 0.9 self.FFmax = 0.999 self.trace = 0 self.thresReset = 0.001 if self.AE: self.inputAE = FOSELM(inputs=inputs, outputs=inputs, numHiddenNeurons=numHiddenNeurons, activationFunction=activationFunction, LN=LN, forgettingFactor=inputWeightForgettingFactor, ORTH=ORTH ) self.hiddenAE = FOSELM(inputs=numHiddenNeurons, outputs=numHiddenNeurons, numHiddenNeurons=numHiddenNeurons, activationFunction=activationFunction, LN=LN, ORTH=ORTH )
class NAOSELM(object): def __init__(self, inputs, outputs, numHiddenNeurons, activationFunction, LN=True, outputWeightForgettingFactor=0.999, inputWeightForgettingFactor=0.999, AE=True, ORTH=False): self.activationFunction = activationFunction self.inputs = inputs self.outputs = outputs self.numHiddenNeurons = numHiddenNeurons # input to hidden weights self.inputWeights = np.random.random((self.numHiddenNeurons, self.inputs)) # bias of hidden units #self.bias = np.random.random((1, self.numHiddenNeurons)) * 2 - 1 self.bias = np.zeros([1,self.numHiddenNeurons]) # hidden to output layer connection self.beta = np.random.random((self.numHiddenNeurons, self.outputs)) # auxiliary matrix used for sequential learning self.M = None self.LN = LN self.AE = AE self.ORTH = ORTH self.forgettingFactor =outputWeightForgettingFactor self.FOSELM_AE = FOSELM(inputs= inputs, outputs = inputs, numHiddenNeurons= numHiddenNeurons, activationFunction= activationFunction, LN=True, forgettingFactor=inputWeightForgettingFactor, ORTH=self.ORTH) def layerNormalization(self, H, scaleFactor=1, biasFactor=0): H_normalized = (H-H.mean())/(np.sqrt(H.var() + 0.00001)) H_normalized = scaleFactor*H_normalized+biasFactor return H_normalized def __calculateInputWeightsUsingAE(self, features): self.FOSELM_AE.train(features=features,targets=features) return self.FOSELM_AE.beta def calculateHiddenLayerActivation(self, features): """ Calculate activation level of the hidden layer :param features feature matrix with dimension (numSamples, numInputs) :return: activation level (numSamples, numHiddenNeurons) """ if self.AE: self.inputWeights = self.__calculateInputWeightsUsingAE(features) if self.activationFunction is "sig": V = linear(features, self.inputWeights,self.bias) if self.LN: V = self.layerNormalization(V) H = sigmoidActFunc(V) else: print " Unknown activation function type" raise NotImplementedError return H def initializePhase(self, lamb=0.0001): """ Step 1: Initialization phase :param features feature matrix with dimension (numSamples, numInputs) :param targets target matrix with dimension (numSamples, numOutputs) """ self.FOSELM_AE.initializePhase(lamb=lamb) self.M = inv(lamb*np.eye(self.numHiddenNeurons)) self.beta = np.zeros([self.numHiddenNeurons,self.outputs]) def train(self, features, targets,VFF_RLS=False): """ Step 2: Sequential learning phase :param features feature matrix with dimension (numSamples, numInputs) :param targets target matrix with dimension (numSamples, numOutputs) """ (numSamples, numOutputs) = targets.shape assert features.shape[0] == targets.shape[0] H = self.calculateHiddenLayerActivation(features) Ht = np.transpose(H) try: scale = 1/(self.forgettingFactor) self.M = scale*self.M - np.dot(scale*self.M, np.dot(Ht, np.dot( pinv(np.eye(numSamples) + np.dot(H, np.dot(scale*self.M, Ht))), np.dot(H, scale*self.M)))) #self.beta = (self.forgettingFactor)*self.beta + np.dot(self.M, np.dot(Ht, targets - np.dot(H, (self.forgettingFactor)*self.beta))) self.beta = self.beta + np.dot(self.M, np.dot(Ht, targets - np.dot(H, self.beta))) except np.linalg.linalg.LinAlgError: print "SVD not converge, ignore the current training cycle" # else: # raise RuntimeError def predict(self, features): """ Make prediction with feature matrix :param features: feature matrix with dimension (numSamples, numInputs) :return: predictions with dimension (numSamples, numOutputs) """ H = self.calculateHiddenLayerActivation(features) prediction = np.dot(H, self.beta) return prediction
class ORELM(object): def __init__(self, inputs, outputs, numHiddenNeurons, activationFunction, LN=True, AE=True, ORTH=True, inputWeightForgettingFactor=0.999, outputWeightForgettingFactor=0.999, hiddenWeightForgettingFactor=0.999): self.activationFunction = activationFunction self.inputs = inputs self.outputs = outputs self.numHiddenNeurons = numHiddenNeurons # input to hidden weights self.inputWeights = np.random.random((self.numHiddenNeurons, self.inputs)) # hidden layer to hidden layer wieghts self.hiddenWeights = np.random.random((self.numHiddenNeurons, self.numHiddenNeurons)) # initial hidden layer activation self.initial_H = np.random.random((1, self.numHiddenNeurons)) * 2 -1 self.H = self.initial_H self.LN = LN self.AE = AE self.ORTH = ORTH # bias of hidden units self.bias = np.random.random((1, self.numHiddenNeurons)) * 2 - 1 # hidden to output layer connection self.beta = np.random.random((self.numHiddenNeurons, self.outputs)) # auxiliary matrix used for sequential learning self.M = inv(0.00001 * np.eye(self.numHiddenNeurons)) self.forgettingFactor = outputWeightForgettingFactor self.trace=0 self.thresReset=0.001 if self.AE: self.inputAE = FOSELM(inputs = inputs, outputs = inputs, numHiddenNeurons = numHiddenNeurons, activationFunction = activationFunction, LN= LN, forgettingFactor=inputWeightForgettingFactor, ORTH = ORTH ) self.hiddenAE = FOSELM(inputs = numHiddenNeurons, outputs = numHiddenNeurons, numHiddenNeurons = numHiddenNeurons, activationFunction=activationFunction, LN= LN, ORTH = ORTH ) def layerNormalization(self, H, scaleFactor=1, biasFactor=0): H_normalized = (H-H.mean())/(np.sqrt(H.var() + 0.000001)) H_normalized = scaleFactor*H_normalized+biasFactor return H_normalized def __calculateInputWeightsUsingAE(self, features): self.inputAE.train(features=features,targets=features) return self.inputAE.beta def __calculateHiddenWeightsUsingAE(self, features): self.hiddenAE.train(features=features,targets=features) return self.hiddenAE.beta def calculateHiddenLayerActivation(self, features): """ Calculate activation level of the hidden layer :param features feature matrix with dimension (numSamples, numInputs) :return: activation level (numSamples, numHiddenNeurons) """ if self.activationFunction is "sig": if self.AE: self.inputWeights = self.__calculateInputWeightsUsingAE(features) self.hiddenWeights = self.__calculateHiddenWeightsUsingAE(self.H) V = linear_recurrent(features=features, inputW=self.inputWeights, hiddenW=self.hiddenWeights, hiddenA=self.H, bias= self.bias) if self.LN: V = self.layerNormalization(V) self.H = sigmoidActFunc(V) else: print " Unknown activation function type" raise NotImplementedError return self.H def initializePhase(self, lamb=0.0001): """ Step 1: Initialization phase :param features feature matrix with dimension (numSamples, numInputs) :param targets target matrix with dimension (numSamples, numOutputs) """ if self.activationFunction is "sig": self.bias = np.random.random((1, self.numHiddenNeurons)) * 2 - 1 else: print " Unknown activation function type" raise NotImplementedError self.M = inv(lamb*np.eye(self.numHiddenNeurons)) self.beta = np.zeros([self.numHiddenNeurons,self.outputs]) # randomly initialize the input->hidden connections self.inputWeights = np.random.random((self.numHiddenNeurons, self.inputs)) self.inputWeights = self.inputWeights * 2 - 1 if self.AE: self.inputAE.initializePhase(lamb=0.00001) self.hiddenAE.initializePhase(lamb=0.00001) else: # randomly initialize the input->hidden connections self.inputWeights = np.random.random((self.numHiddenNeurons, self.inputs)) self.inputWeights = self.inputWeights * 2 - 1 if self.ORTH: if self.numHiddenNeurons > self.inputs: self.inputWeights = orthogonalization(self.inputWeights) else: self.inputWeights = orthogonalization(self.inputWeights.transpose()) self.inputWeights = self.inputWeights.transpose() # hidden layer to hidden layer wieghts self.hiddenWeights = np.random.random((self.numHiddenNeurons, self.numHiddenNeurons)) self.hiddenWeights = self.hiddenWeights * 2 - 1 if self.ORTH: self.hiddenWeights = orthogonalization(self.hiddenWeights) def reset(self): self.H = self.initial_H def train(self, features, targets,RESETTING=False): """ Step 2: Sequential learning phase :param features feature matrix with dimension (numSamples, numInputs) :param targets target matrix with dimension (numSamples, numOutputs) """ (numSamples, numOutputs) = targets.shape assert features.shape[0] == targets.shape[0] H = self.calculateHiddenLayerActivation(features) Ht = np.transpose(H) try: scale = 1/(self.forgettingFactor) self.M = scale*self.M - np.dot(scale*self.M, np.dot(Ht, np.dot( pinv(np.eye(numSamples) + np.dot(H, np.dot(scale*self.M, Ht))), np.dot(H, scale*self.M)))) if RESETTING: beforeTrace=self.trace self.trace=self.M.trace() print np.abs(beforeTrace - self.trace) if np.abs(beforeTrace - self.trace) < self.thresReset: print self.M eig,_=np.linalg.eig(self.M) lambMin=min(eig) lambMax=max(eig) #lamb = (lambMax+lambMin)/2 lamb = lambMax lamb = lamb.real self.M= lamb*np.eye(self.numHiddenNeurons) print "reset" print self.M self.beta = (self.forgettingFactor)*self.beta + np.dot(self.M, np.dot(Ht, targets - np.dot(H, (self.forgettingFactor)*self.beta))) #self.beta = self.beta + np.dot(self.M, np.dot(Ht, targets - np.dot(H, self.beta))) except np.linalg.linalg.LinAlgError: print "SVD not converge, ignore the current training cycle" # else: # raise RuntimeError def predict(self, features): """ Make prediction with feature matrix :param features: feature matrix with dimension (numSamples, numInputs) :return: predictions with dimension (numSamples, numOutputs) """ H = self.calculateHiddenLayerActivation(features) prediction = np.dot(H, self.beta) return prediction