class MLP(object): """ Multi-Layer Perceptron Class with multiple hidden layers. This class is used for pretraining the second phase neural network. Intermediate layers have tanh activation function or the sigmoid function (defined here by a ``SigmoidalLayer`` class) while the top layer is a softmax layer (defined here by a ``LogisticRegression`` class). """ def __init__(self, input, n_in=64, n_hiddens=[400, 500], n_out=11, normalize_inputs=False, use_adagrad=True, activation=NeuralActivations.Rectifier, exp_id=1, quiet=False, rng=None): """ Initialize the parameters for the multilayer perceptron :type rng: numpy.random.RandomState :param rng: a random number generator used to initialize weights :type input: theano.tensor.TensorType :param input: symbolic variable that describes the input of the architecture (one minibatch) :type n_in: int :param n_in: number of input units, the dimension of the space in which the datapoints lie. :type n_hidden: int :param n_hidden: number of hidden units :type n_out: int :param n_out: number of output units, the dimension of the space in which the labels lie. """ self.input = input if rng == None: rng = numpy.random.RandomState(1234) self.normalize_inputs = normalize_inputs self.learning_rate = 0.001 self.exp_id = exp_id # Since we are dealing with a one hidden layer MLP, this will # translate into a TanhLayer connected to the LogisticRegression # layer; this can be replaced by a SigmoidalLayer, or a layer # implementing any other nonlinearity self.rng = rng self.ds = Dataset() self.n_out = n_out self.n_in = n_in self.n_hiddens = n_hiddens self.n_hidden_layers = len(n_hiddens) self.hiddenLayers = [] self.state = "train" self.out_dir = "out/" self.grads = [] self.test_scores = [] #Whether to turn on or off the messages. self.quiet = quiet self.setup_pkl_paths() self.setup_hidden_layers(activation, n_in=n_in, n_out=n_out, n_hiddens=n_hiddens) self.use_adagrad = use_adagrad #Error for patches with object in it: self.obj_patch_error_percent = 0 self.reset_classified_patches() self.setup_patch_save_file() self.data_dict = { 'Ws':[], 'costs':[], 'test_probs':[], 'train_probs':[], 'test_scores':[] } def setup_patch_save_file(self, path="/RQusagers/gulcehre/codes/python/experiments/arcade_ds_exps/pretrained_mlp/prmlp_clean/out/"): self.npy_well_classified = path + "_well_classified_patches.npy" self.npy_misclassified = path + "_misclassified_patches.npy" def setup_pkl_paths(self, name_prefix="premlp_train_"): self.train_pkl_file = name_prefix + "n_hidden_" + str(self.n_hiddens) + \ "_n_out_" + str(self.n_out) + "_learning_rate_" + str(self.learning_rate) + \ "_exp_id_" + str(self.exp_id) self.test_pkl_file = name_prefix + "n_hidden_" + str(self.n_hiddens) + \ "_n_out_" + str(self.n_out) + "_learning_rate_" + str(self.learning_rate) + \ "_exp_id_" + str(self.exp_id) def setup_hidden_layers(self, activation, n_in=0, n_out=0, n_hiddens=0): """ Setup the hidden layers with the specified number of hidden units. """ act_fn = T.tanh if activation == NeuralActivations.Rectifier: act_fn = self.rectifier_act if n_in == 0: n_in = self.n_in if n_out == 0: n_out = self.n_out if n_hiddens == 0: n_hiddens = self.n_hiddens self.rng.seed(1985) #Create the hidden layers. self.hiddenLayers.append(HiddenLayer(rng=self.rng, input=self.input, n_in=n_in, n_out=n_hiddens[0], activation=act_fn)) for i in xrange(1, self.n_hidden_layers): self.rng.seed(2012) self.hiddenLayers.append(HiddenLayer(rng=self.rng, input=self.hiddenLayers[i-1].output, n_in=n_hiddens[i-1], n_out=n_hiddens[i], activation=act_fn)) # The logistic regression layer gets as input the hidden units # of the hidden layer self.logRegressionLayer = LogisticRegressionLayer( input=self.hiddenLayers[-1].output, n_in=n_hiddens[-1], n_out=n_out, rng=self.rng) self.initialize_regularization() # negative log likelihood of the MLP is given by the negative # log likelihood of the output of the model, computed in the # logistic regression layer self.negative_log_likelihood = self.logRegressionLayer.negative_log_likelihood # negative log likelihood of the MLP is given by the # crossentropy of the output of the model, computed in the # logistic regression layer self.crossentropy = self.logRegressionLayer.crossentropy self.crossentropy_categorical = self.logRegressionLayer.crossentropy_categorical # same holds for the function computing the number of errors self.errors = self.logRegressionLayer.errors self.raw_prediction_errors =\ self.logRegressionLayer.raw_prediction_errors self.p_y_given_x = self.logRegressionLayer.p_y_given_x # Class memberships hidden_outputs = self.hiddenLayers[0].get_outputs(self.input) for i in xrange(1, self.n_hidden_layers): hidden_outputs = self.hiddenLayers[i].get_outputs(hidden_outputs) self.class_memberships = self.logRegressionLayer.get_class_memberships(hidden_outputs) self.initialize_params() def initialize_params(self): # the parameters of the model are the parameters of the all hidden # and logistic regression layers it is made out of self.params = self.hiddenLayers[0].params for i in xrange(1, self.n_hidden_layers): self.params += self.hiddenLayers[i].params self.params += self.logRegressionLayer.params def initialize_regularization(self): # L1 norm ; one regularization option is to enforce L1 norm to # be small self.L1 = 0 # square of L2 norm; # one regularization option is to enforce # square of L2 norm to be small self.L2_sqr = 0 for i in xrange(self.n_hidden_layers): self.L1 += abs(self.hiddenLayers[i].W).sum() self.L2_sqr += (self.hiddenLayers[i].W ** 2).sum() self.L1 += abs(self.logRegressionLayer.W).sum() self.L2_sqr += (self.logRegressionLayer.W ** 2).sum() def normalize_in_data(self, data): return (data.T / data.max(axis=1)).T def rectifier_act(self, x, mask=False): """ Activation function for rectifier hidden units. """ if mask: activation = numpy.asarray(numpy.sign(numpy.random.uniform(low=-1, high=1, size=(10,))), dtype=theano.config.floatX) * T.maximum(x, 0) else: activation = T.maximum(x, 0) return activation def do_brain_transfer(self, t_params): """ Copy the weights of one neural network to another one. """ cnt_nonparams = 0 for i in xrange(len(self.params)): if self.params[i].name == "W": self.params[i].set_value(t_params[i - cnt_nonparams].get_value()) else: cnt_nonparams += 1 def sgd_updates(self, cost, learning_rate): """ Using this function, specify how to update the parameters of the model as a dictionary """ updates = {} if self.use_adagrad: updates = self.sgd_updates_adagrad(cost, learning_rate) else: for param in self.params: grad = T.grad(cost, param) self.grads.append(grad) # given two list the zip A = [a1, a2, a3, a4] and B = [b1, b2, b3, b4] of # same length, zip generates a list C of same size, where each element # is a pair formed from the two lists : # C = [(a1, b1), (a2, b2), (a3, b3) , (a4, b4)] for param, grad in zip(self.params, self.grads): updates[param] = param - learning_rate * grad return updates def sgd_updates_adagrad(self, cost, learning_rate): """ Return the dictionary of parameter specific learning rate updates using adagrad algorithm. """ #Initialize the variables accumulators = {} e0s = {} learn_rates = [] ups = {} #initialize the accumulator and the epsilon_0 for param in self.params: accumulators[param] = theano.shared(value=as_floatX(0.), name="acc_%s" % param.name) e0s[param] = as_floatX(learning_rate) self.grads = [T.grad(cost, p) for p in self.params] #Compute the learning rates for param, gp in zip(self.params, self.grads): acc = accumulators[param] ups[acc] = T.sqrt((gp ** 2).sum()) learn_rates.append(e0s[param] / ups[acc]) #Find the updates based on the parameters updates = [(p, p - step * gp) for (step, p, gp) in zip(learn_rates, self.params, self.grads)] p_up = dict(updates) safe_update(ups, p_up) return ups def standardize_weights(self): for param in self.params: if param.name == "W": val = param.get_value(borrow=True) if val.mean(0).nonzero()[0].shape[0] > 0: val = (val - val.mean(0)) if val.std(0).nonzero()[0].shape[0] > 0: val /= val.std(0) param.set_value(val) def normalize_weights(self): """ TODO: Add the weight constraint for the weight normalization that G. Hinton mentions about. """ for param in self.params: if param.name == "W": val = param.get_value(borrow=True) val = self.normalize_in_data(val) param.set_value(val) def dropout(self, prob=0.5): """ Randomly dropout the hidden units wrt given probability as a binomial prob. """ for param in self.params: if param.name == "W": val = param.get_value(borrow=True) #throw a coin for weight: dropouts = numpy.random.binomial(1, prob, (val.shape[0], val.shape[1])) new_param = param * dropouts param.set_value(val) def dropout_raw(self, prob=0.5): """ Randomly dropout the hidden units wrt given probability as a binomial prob.Avoid using that function, unoptimized and very naive. Just for testing. """ for param in self.params: if param.name == "W": val = param.get_value(borrow=True) #throw a coin for weight: dropouts = numpy.random.binomial(1, prob, (val.shape[0], val.shape[1])) for i in xrange(val.shape[0]): for j in xrange(val.shape[1]): if dropouts[i][j] == 0: val[i][j] = 0 param.set_value(val) def tangent_propagation(self, learning_rate): """ TODO: Implement the tangent propagation for invariant recognition. """ return NotImplementedError() def get_cost_function(self, cost_type, y, L1_reg, L2_reg): if cost_type == Costs.Crossentropy: if self.n_out == 1: cost = self.crossentropy(y) \ + L1_reg * self.L1 \ + L2_reg * self.L2_sqr else: cost = self.crossentropy_categorical(y) \ + L1_reg * self.L1 \ + L2_reg * self.L2_sqr elif cost_type == Costs.NegativeLikelihood: cost = self.negative_log_likelihood(y) \ + L1_reg * self.L1 \ + L2_reg * self.L2_sqr return cost def flip_sign(self, val, prob=0.1): p = 1 - prob return -val if numpy.random.rand() > p else val def get_inhibitory_mat(self, size=None, prob=0.1): if size is None: raise Exception("Please enter a shape value") id_mat = numpy.identity(size) for i in xrange(size): id_mat[i] = self.flip_sign(id_mat[i], prob=prob) return id_mat def flip_weight_matrices_sign(self, prob=0.1): for i in xrange(len(self.params)): if self.params[i].name == "W": weights = self.params[i].get_value() inhibit_mat = self.get_inhibitory_mat(size=weights.shape[1], prob=prob) weights = numpy.asarray(numpy.dot(weights, inhibit_mat), dtype=theano.config.floatX) self.params[i].set_value(weights) def record_classified_examples(self, patches, errors): # import ipdb; ipdb.set_trace() for i in xrange(errors.shape[0]): if errors[i] == 1: self.misclassifieds.append(patches[i]) else: self.well_classifieds.append(patches[i]) def save_classified_patches(self): misclassifieds = numpy.asarray(self.misclassifieds, dtype="uint8") well_classifieds = numpy.asarray(self.well_classifieds, dtype="uint8") numpy.save(self.npy_well_classified, well_classifieds) numpy.save(self.npy_misclassified, misclassifieds) def save_data(self): output = None if self.state == "train": output = open(self.out_dir + self.train_pkl_file + ".pkl", 'wb') else: output = open(self.out_dir + self.test_pkl_file + ".pkl", 'wb') pkl.dump(self.data_dict, output) self.data_dict['Ws'] = [] self.data_dict['costs'] = [] self.data_dict['test_probs'] = [] self.data_dict['train_probs'] = [] self.data_dict['test_scores'] = [] def report_object_patch_statistics(self): """ Report the generalization error for the patches containing objects. """ conf_mat = self.logRegressionLayer.conf_mat #import ipdb; ipdb.set_trace() correct_class = 0 incorrect_class = 0 for i in xrange(1, self.n_out): for j in xrange(self.n_out): if i == j: correct_class += conf_mat[i][j] else: incorrect_class += conf_mat[i][j] self.obj_patch_error_percent = (correct_class/ (incorrect_class + correct_class)) * 100 print "Number of incorrectly classified patch objects: " + str(incorrect_class) print "Percentage of correctly classified patch objects: " + str(self.obj_patch_error_percent) def reset_confusion_mat(self): """ Reset the confusion matrix. """ self.logRegressionLayer.reset_conf_mat() def reset_classified_patches(self): self.well_classifieds = [] self.misclassifieds = []
class MLP(object): """ Multi-Layer Perceptron Class with multiple hidden layers. This class is used for pretraining the second phase neural network. Intermediate layers have tanh activation function or the sigmoid function (defined here by a ``SigmoidalLayer`` class) while the top layer is a softmax layer (defined here by a ``LogisticRegression`` class). """ def __init__(self, input, n_in=64, n_hiddens=[400, 500], n_out=11, normalize_inputs=False, use_adagrad=True, activation=NeuralActivations.Rectifier, exp_id=1, quiet=False, rng=None): """ Initialize the parameters for the multilayer perceptron :type rng: numpy.random.RandomState :param rng: a random number generator used to initialize weights :type input: theano.tensor.TensorType :param input: symbolic variable that describes the input of the architecture (one minibatch) :type n_in: int :param n_in: number of input units, the dimension of the space in which the datapoints lie. :type n_hidden: int :param n_hidden: number of hidden units :type n_out: int :param n_out: number of output units, the dimension of the space in which the labels lie. """ self.input = input if rng == None: rng = numpy.random.RandomState(1234) self.normalize_inputs = normalize_inputs self.learning_rate = 0.001 self.exp_id = exp_id # Since we are dealing with a one hidden layer MLP, this will # translate into a TanhLayer connected to the LogisticRegression # layer; this can be replaced by a SigmoidalLayer, or a layer # implementing any other nonlinearity self.rng = rng self.ds = Dataset() self.n_out = n_out self.n_in = n_in self.n_hiddens = n_hiddens self.n_hidden_layers = len(n_hiddens) self.hiddenLayers = [] self.state = "train" self.out_dir = "out/" self.grads = [] self.test_scores = [] #Whether to turn on or off the messages. self.quiet = quiet self.setup_pkl_paths() self.setup_hidden_layers(activation, n_in=n_in, n_out=n_out, n_hiddens=n_hiddens) self.use_adagrad = use_adagrad #Error for patches with object in it: self.obj_patch_error_percent = 0 self.reset_classified_patches() self.setup_patch_save_file() self.data_dict = { 'Ws': [], 'costs': [], 'test_probs': [], 'train_probs': [], 'test_scores': [] } def setup_patch_save_file( self, path="/RQusagers/gulcehre/codes/python/experiments/arcade_ds_exps/pretrained_mlp/prmlp_clean/out/" ): self.npy_well_classified = path + "_well_classified_patches.npy" self.npy_misclassified = path + "_misclassified_patches.npy" def setup_pkl_paths(self, name_prefix="premlp_train_"): self.train_pkl_file = name_prefix + "n_hidden_" + str(self.n_hiddens) + \ "_n_out_" + str(self.n_out) + "_learning_rate_" + str(self.learning_rate) + \ "_exp_id_" + str(self.exp_id) self.test_pkl_file = name_prefix + "n_hidden_" + str(self.n_hiddens) + \ "_n_out_" + str(self.n_out) + "_learning_rate_" + str(self.learning_rate) + \ "_exp_id_" + str(self.exp_id) def setup_hidden_layers(self, activation, n_in=0, n_out=0, n_hiddens=0): """ Setup the hidden layers with the specified number of hidden units. """ act_fn = T.tanh if activation == NeuralActivations.Rectifier: act_fn = self.rectifier_act if n_in == 0: n_in = self.n_in if n_out == 0: n_out = self.n_out if n_hiddens == 0: n_hiddens = self.n_hiddens self.rng.seed(1985) #Create the hidden layers. self.hiddenLayers.append( HiddenLayer(rng=self.rng, input=self.input, n_in=n_in, n_out=n_hiddens[0], activation=act_fn)) for i in xrange(1, self.n_hidden_layers): self.rng.seed(2012) self.hiddenLayers.append( HiddenLayer(rng=self.rng, input=self.hiddenLayers[i - 1].output, n_in=n_hiddens[i - 1], n_out=n_hiddens[i], activation=act_fn)) # The logistic regression layer gets as input the hidden units # of the hidden layer self.logRegressionLayer = LogisticRegressionLayer( input=self.hiddenLayers[-1].output, n_in=n_hiddens[-1], n_out=n_out, rng=self.rng) self.initialize_regularization() # negative log likelihood of the MLP is given by the negative # log likelihood of the output of the model, computed in the # logistic regression layer self.negative_log_likelihood = self.logRegressionLayer.negative_log_likelihood # negative log likelihood of the MLP is given by the # crossentropy of the output of the model, computed in the # logistic regression layer self.crossentropy = self.logRegressionLayer.crossentropy self.crossentropy_categorical = self.logRegressionLayer.crossentropy_categorical # same holds for the function computing the number of errors self.errors = self.logRegressionLayer.errors self.raw_prediction_errors =\ self.logRegressionLayer.raw_prediction_errors self.p_y_given_x = self.logRegressionLayer.p_y_given_x # Class memberships hidden_outputs = self.hiddenLayers[0].get_outputs(self.input) for i in xrange(1, self.n_hidden_layers): hidden_outputs = self.hiddenLayers[i].get_outputs(hidden_outputs) self.class_memberships = self.logRegressionLayer.get_class_memberships( hidden_outputs) self.initialize_params() def initialize_params(self): # the parameters of the model are the parameters of the all hidden # and logistic regression layers it is made out of self.params = self.hiddenLayers[0].params for i in xrange(1, self.n_hidden_layers): self.params += self.hiddenLayers[i].params self.params += self.logRegressionLayer.params def initialize_regularization(self): # L1 norm ; one regularization option is to enforce L1 norm to # be small self.L1 = 0 # square of L2 norm; # one regularization option is to enforce # square of L2 norm to be small self.L2_sqr = 0 for i in xrange(self.n_hidden_layers): self.L1 += abs(self.hiddenLayers[i].W).sum() self.L2_sqr += (self.hiddenLayers[i].W**2).sum() self.L1 += abs(self.logRegressionLayer.W).sum() self.L2_sqr += (self.logRegressionLayer.W**2).sum() def normalize_in_data(self, data): return (data.T / data.max(axis=1)).T def rectifier_act(self, x, mask=False): """ Activation function for rectifier hidden units. """ if mask: activation = numpy.asarray( numpy.sign(numpy.random.uniform(low=-1, high=1, size=(10, ))), dtype=theano.config.floatX) * T.maximum(x, 0) else: activation = T.maximum(x, 0) return activation def do_brain_transfer(self, t_params): """ Copy the weights of one neural network to another one. """ cnt_nonparams = 0 for i in xrange(len(self.params)): if self.params[i].name == "W": self.params[i].set_value(t_params[i - cnt_nonparams].get_value()) else: cnt_nonparams += 1 def sgd_updates(self, cost, learning_rate): """ Using this function, specify how to update the parameters of the model as a dictionary """ updates = {} if self.use_adagrad: updates = self.sgd_updates_adagrad(cost, learning_rate) else: for param in self.params: grad = T.grad(cost, param) self.grads.append(grad) # given two list the zip A = [a1, a2, a3, a4] and B = [b1, b2, b3, b4] of # same length, zip generates a list C of same size, where each element # is a pair formed from the two lists : # C = [(a1, b1), (a2, b2), (a3, b3) , (a4, b4)] for param, grad in zip(self.params, self.grads): updates[param] = param - learning_rate * grad return updates def sgd_updates_adagrad(self, cost, learning_rate): """ Return the dictionary of parameter specific learning rate updates using adagrad algorithm. """ #Initialize the variables accumulators = {} e0s = {} learn_rates = [] ups = {} #initialize the accumulator and the epsilon_0 for param in self.params: accumulators[param] = theano.shared(value=as_floatX(0.), name="acc_%s" % param.name) e0s[param] = as_floatX(learning_rate) self.grads = [T.grad(cost, p) for p in self.params] #Compute the learning rates for param, gp in zip(self.params, self.grads): acc = accumulators[param] ups[acc] = T.sqrt((gp**2).sum()) learn_rates.append(e0s[param] / ups[acc]) #Find the updates based on the parameters updates = [(p, p - step * gp) for (step, p, gp) in zip(learn_rates, self.params, self.grads)] p_up = dict(updates) safe_update(ups, p_up) return ups def standardize_weights(self): for param in self.params: if param.name == "W": val = param.get_value(borrow=True) if val.mean(0).nonzero()[0].shape[0] > 0: val = (val - val.mean(0)) if val.std(0).nonzero()[0].shape[0] > 0: val /= val.std(0) param.set_value(val) def normalize_weights(self): """ TODO: Add the weight constraint for the weight normalization that G. Hinton mentions about. """ for param in self.params: if param.name == "W": val = param.get_value(borrow=True) val = self.normalize_in_data(val) param.set_value(val) def dropout(self, prob=0.5): """ Randomly dropout the hidden units wrt given probability as a binomial prob. """ for param in self.params: if param.name == "W": val = param.get_value(borrow=True) #throw a coin for weight: dropouts = numpy.random.binomial(1, prob, (val.shape[0], val.shape[1])) new_param = param * dropouts param.set_value(val) def dropout_raw(self, prob=0.5): """ Randomly dropout the hidden units wrt given probability as a binomial prob.Avoid using that function, unoptimized and very naive. Just for testing. """ for param in self.params: if param.name == "W": val = param.get_value(borrow=True) #throw a coin for weight: dropouts = numpy.random.binomial(1, prob, (val.shape[0], val.shape[1])) for i in xrange(val.shape[0]): for j in xrange(val.shape[1]): if dropouts[i][j] == 0: val[i][j] = 0 param.set_value(val) def tangent_propagation(self, learning_rate): """ TODO: Implement the tangent propagation for invariant recognition. """ return NotImplementedError() def get_cost_function(self, cost_type, y, L1_reg, L2_reg): if cost_type == Costs.Crossentropy: if self.n_out == 1: cost = self.crossentropy(y) \ + L1_reg * self.L1 \ + L2_reg * self.L2_sqr else: cost = self.crossentropy_categorical(y) \ + L1_reg * self.L1 \ + L2_reg * self.L2_sqr elif cost_type == Costs.NegativeLikelihood: cost = self.negative_log_likelihood(y) \ + L1_reg * self.L1 \ + L2_reg * self.L2_sqr return cost def flip_sign(self, val, prob=0.1): p = 1 - prob return -val if numpy.random.rand() > p else val def get_inhibitory_mat(self, size=None, prob=0.1): if size is None: raise Exception("Please enter a shape value") id_mat = numpy.identity(size) for i in xrange(size): id_mat[i] = self.flip_sign(id_mat[i], prob=prob) return id_mat def flip_weight_matrices_sign(self, prob=0.1): for i in xrange(len(self.params)): if self.params[i].name == "W": weights = self.params[i].get_value() inhibit_mat = self.get_inhibitory_mat(size=weights.shape[1], prob=prob) weights = numpy.asarray(numpy.dot(weights, inhibit_mat), dtype=theano.config.floatX) self.params[i].set_value(weights) def record_classified_examples(self, patches, errors): # import ipdb; ipdb.set_trace() for i in xrange(errors.shape[0]): if errors[i] == 1: self.misclassifieds.append(patches[i]) else: self.well_classifieds.append(patches[i]) def save_classified_patches(self): misclassifieds = numpy.asarray(self.misclassifieds, dtype="uint8") well_classifieds = numpy.asarray(self.well_classifieds, dtype="uint8") numpy.save(self.npy_well_classified, well_classifieds) numpy.save(self.npy_misclassified, misclassifieds) def save_data(self): output = None if self.state == "train": output = open(self.out_dir + self.train_pkl_file + ".pkl", 'wb') else: output = open(self.out_dir + self.test_pkl_file + ".pkl", 'wb') pkl.dump(self.data_dict, output) self.data_dict['Ws'] = [] self.data_dict['costs'] = [] self.data_dict['test_probs'] = [] self.data_dict['train_probs'] = [] self.data_dict['test_scores'] = [] def report_object_patch_statistics(self): """ Report the generalization error for the patches containing objects. """ conf_mat = self.logRegressionLayer.conf_mat #import ipdb; ipdb.set_trace() correct_class = 0 incorrect_class = 0 for i in xrange(1, self.n_out): for j in xrange(self.n_out): if i == j: correct_class += conf_mat[i][j] else: incorrect_class += conf_mat[i][j] self.obj_patch_error_percent = ( correct_class / (incorrect_class + correct_class)) * 100 print "Number of incorrectly classified patch objects: " + str( incorrect_class) print "Percentage of correctly classified patch objects: " + str( self.obj_patch_error_percent) def reset_confusion_mat(self): """ Reset the confusion matrix. """ self.logRegressionLayer.reset_conf_mat() def reset_classified_patches(self): self.well_classifieds = [] self.misclassifieds = []
def setup_hidden_layers(self, activation, n_in=0, n_out=0, n_hiddens=0): """ Setup the hidden layers with the specified number of hidden units. """ act_fn = T.tanh if activation == NeuralActivations.Rectifier: act_fn = self.rectifier_act if n_in == 0: n_in = self.n_in if n_out == 0: n_out = self.n_out if n_hiddens == 0: n_hiddens = self.n_hiddens self.rng.seed(1985) #Create the hidden layers. self.hiddenLayers.append(HiddenLayer(rng=self.rng, input=self.input, n_in=n_in, n_out=n_hiddens[0], activation=act_fn)) for i in xrange(1, self.n_hidden_layers): self.rng.seed(2012) self.hiddenLayers.append(HiddenLayer(rng=self.rng, input=self.hiddenLayers[i-1].output, n_in=n_hiddens[i-1], n_out=n_hiddens[i], activation=act_fn)) # The logistic regression layer gets as input the hidden units # of the hidden layer self.logRegressionLayer = LogisticRegressionLayer( input=self.hiddenLayers[-1].output, n_in=n_hiddens[-1], n_out=n_out, rng=self.rng) self.initialize_regularization() # negative log likelihood of the MLP is given by the negative # log likelihood of the output of the model, computed in the # logistic regression layer self.negative_log_likelihood = self.logRegressionLayer.negative_log_likelihood # negative log likelihood of the MLP is given by the # crossentropy of the output of the model, computed in the # logistic regression layer self.crossentropy = self.logRegressionLayer.crossentropy self.crossentropy_categorical = self.logRegressionLayer.crossentropy_categorical # same holds for the function computing the number of errors self.errors = self.logRegressionLayer.errors self.raw_prediction_errors =\ self.logRegressionLayer.raw_prediction_errors self.p_y_given_x = self.logRegressionLayer.p_y_given_x # Class memberships hidden_outputs = self.hiddenLayers[0].get_outputs(self.input) for i in xrange(1, self.n_hidden_layers): hidden_outputs = self.hiddenLayers[i].get_outputs(hidden_outputs) self.class_memberships = self.logRegressionLayer.get_class_memberships(hidden_outputs) self.initialize_params()
def setup_hidden_layers(self, activation, n_in=0, n_out=0, n_hiddens=0): """ Setup the hidden layers with the specified number of hidden units. """ act_fn = T.tanh if activation == NeuralActivations.Rectifier: act_fn = self.rectifier_act if n_in == 0: n_in = self.n_in if n_out == 0: n_out = self.n_out if n_hiddens == 0: n_hiddens = self.n_hiddens self.rng.seed(1985) #Create the hidden layers. self.hiddenLayers.append( HiddenLayer(rng=self.rng, input=self.input, n_in=n_in, n_out=n_hiddens[0], activation=act_fn)) for i in xrange(1, self.n_hidden_layers): self.rng.seed(2012) self.hiddenLayers.append( HiddenLayer(rng=self.rng, input=self.hiddenLayers[i - 1].output, n_in=n_hiddens[i - 1], n_out=n_hiddens[i], activation=act_fn)) # The logistic regression layer gets as input the hidden units # of the hidden layer self.logRegressionLayer = LogisticRegressionLayer( input=self.hiddenLayers[-1].output, n_in=n_hiddens[-1], n_out=n_out, rng=self.rng) self.initialize_regularization() # negative log likelihood of the MLP is given by the negative # log likelihood of the output of the model, computed in the # logistic regression layer self.negative_log_likelihood = self.logRegressionLayer.negative_log_likelihood # negative log likelihood of the MLP is given by the # crossentropy of the output of the model, computed in the # logistic regression layer self.crossentropy = self.logRegressionLayer.crossentropy self.crossentropy_categorical = self.logRegressionLayer.crossentropy_categorical # same holds for the function computing the number of errors self.errors = self.logRegressionLayer.errors self.raw_prediction_errors =\ self.logRegressionLayer.raw_prediction_errors self.p_y_given_x = self.logRegressionLayer.p_y_given_x # Class memberships hidden_outputs = self.hiddenLayers[0].get_outputs(self.input) for i in xrange(1, self.n_hidden_layers): hidden_outputs = self.hiddenLayers[i].get_outputs(hidden_outputs) self.class_memberships = self.logRegressionLayer.get_class_memberships( hidden_outputs) self.initialize_params()