def add_biaffine(self, inputs): ## inputs [seq_len, batch_size, units] ## first define four different MLPs arc_roles = ['arc-dep', 'arc-head'] rel_roles = ['rel-dep', 'rel-head'] joint_roles = ['jk', 'stag'] vectors = {} for arc_role in arc_roles: for i in xrange(self.opts.mlp_num_layers): if i == 0: inputs_dim = self.outputs_dim vector_mlp = inputs else: inputs_dim = self.opts.arc_mlp_units weights = get_mlp_weights('{}_MLP_Layer{}'.format(arc_role, i), inputs_dim, self.opts.arc_mlp_units) vector_mlp = self.add_dropout(tf.map_fn(lambda x: mlp(x, weights), vector_mlp), self.mlp_prob) ## [seq_len, batch_size, 2*mlp_units] vectors[arc_role] = vector_mlp weights = get_arc_weights('arc', self.opts.arc_mlp_units) arc_output = arc_equation(vectors['arc-head'], vectors['arc-dep'], weights) # [batch_size, seq_len, seq_len] dim 1: deps, dim 2: heads # arc_predictions = get_arcs(arc_output, self.test_opts) # [batch_size, seq_len] arc_predictions = tf.argmax(arc_output, 2) # [batch_size, seq_len] for rel_role in rel_roles: for i in xrange(self.opts.mlp_num_layers): if i == 0: inputs_dim = self.outputs_dim vector_mlp = inputs else: inputs_dim = self.opts.rel_mlp_units weights = get_mlp_weights('{}_MLP_Layer{}'.format(rel_role, i), inputs_dim, self.opts.rel_mlp_units) vector_mlp = self.add_dropout(tf.map_fn(lambda x: mlp(x, weights), vector_mlp), self.mlp_prob) ## [seq_len, batch_size, 2*mlp_units] vectors[rel_role] = vector_mlp weights = get_rel_weights('rel', self.opts.rel_mlp_units, self.loader.nb_rels) rel_output, rel_scores = rel_equation(vectors['rel-head'], vectors['rel-dep'], weights, arc_predictions) #[batch_size, seq_len, nb_rels] ## joint stagging for joint_role in joint_roles: for i in xrange(self.opts.mlp_num_layers): if i == 0: inputs_dim = self.outputs_dim vector_mlp = inputs else: inputs_dim = self.opts.joint_mlp_units weights = get_mlp_weights('{}_MLP_Layer{}'.format(joint_role, i), inputs_dim, self.opts.joint_mlp_units) vector_mlp = self.add_dropout(tf.map_fn(lambda x: mlp(x, weights), vector_mlp), self.mlp_prob) ## [seq_len, batch_size, 2*mlp_units] vectors[joint_role] = vector_mlp weights = get_joint_weights('stag', self.opts.joint_mlp_units, self.loader.nb_stags) self.stag_embeddings = tf.transpose(weights['W-joint'], [1,0]) joint_output = joint_equation(vectors['stag'], weights) # [batch_size, seq_len, nb_stags] weights = get_joint_weights('jk', self.opts.joint_mlp_units, self.loader.nb_jk) joint_output_jk = joint_equation(vectors['jk'], weights) # [batch_size, seq_len, nb_stags] return arc_output, rel_output, rel_scores, joint_output, joint_output_jk
def testMLP(dataFileName, nhidden): dataText = np.loadtxt(dataFileName, delimiter=',') target = np.zeros((np.shape(dataText)[0], 2)) indices = np.where(dataText[:, -1] == 0) target[indices, 0] = 1 indices = np.where(dataText[:, -1] == 1) target[indices, 1] = 1 #print dataText # 5 fold validation val_error = [] TP = [] TN = [] FP = [] FN = [] order = range(np.shape(dataText)[0]) np.random.shuffle(order) dataText = dataText[order, :] dataText = dataText[:, :-1] target = target[order, :] for i in range(5): valid = dataText[i * len(dataText) / 5:(i + 1) * len(dataText) / 5] validt = target[i * len(target) / 5:(i + 1) * len(target) / 5] train = np.vstack((dataText[:i * len(dataText) / 5], dataText[(i + 1) * len(dataText) / 5:])) traint = np.vstack( (target[:i * len(target) / 5], target[(i + 1) * len(target) / 5:])) net = mlp.mlp(train, traint, nhidden, outtype='softmax') val_error.append(net.earlystopping(train, traint, valid, validt, 0.3)) cm = net.confmat(valid, validt) TP.append(cm[0][0]) FN.append(cm[0][1]) FP.append(cm[1][0]) TN.append(cm[1][1]) return np.mean(val_error), np.sum(TP), np.sum(FP), np.sum(TN), np.sum(FN)
def main(): print("***EXECUTION INITIATED***") # train the data training_data = get_vectors()['training1and2'] # create MLP with 12 hidden nodes and 8 output nodes MLP = mlp(12, 8, training_data) # test the holdout set holdout_data = get_vectors()['holdout'] for example in holdout_data: MLP.get_classification(example) print() # printing the weights and the epochs MLP.print_weights() MLP.print_epochs() #Validation print("----------Validation----------") validation_set = get_vectors()['training2'] print("\nConfusion Matrix Rates on Holdout:") MLP.print_rates_for_class(3, holdout_data) MLP.print_rates_for_class(2, holdout_data) print("\nConfusion Matrix Rates on Validation Set:") MLP.print_rates_for_class(3, validation_set) MLP.print_rates_for_class(2, validation_set) accuracy_rate = MLP.get_accuracy(holdout_data) error_rate = 1 - accuracy_rate print("MLP Accuracy and Error Rates for Holdout:") print("Accuracy rate =", accuracy_rate) print("Error rate =", error_rate) accuracy_rate = MLP.get_accuracy(validation_set) error_rate = 1 - accuracy_rate print("\nMLP Accuracy and Error Rates for Validation:") print("Accuracy rate =", accuracy_rate) print("Error rate =", error_rate)
def __init__(self, layers, mean=None, rng_seed=None): ''' layers is a list specifying the dimension of each network layer: layers = [#n_visible, #n_hidden1, #n_hidden2, ...] The final layer is a softmax layer, with one node per class label E.g., layers = [20,10,2] implements binary classification with inputs of dimension 20, and 10 units in the first hidden layer The DNN is trained by first calling pre_train() to perform unsupervised pre-training, and then fine_tune() to perform discriminative (supervised) fine tuning ''' self.layers = layers self.n_hid = len(layers) - 1 self.mlp = mlp.mlp(layers, rng_seed) # initialize multi-layer perceptron self.rbm = (self.n_hid - 1) * [None] # initialize RBMs if 1: self.rbm[0] = rbm.rbm(layers[0], layers[1], input_type='binary', mean=mean) else: self.rbm[0] = rbm.rbm(layers[0], layers[1], input_type='gaussian') for k in xrange(1, self.n_hid - 1): # skip last layer which uses softmax instead of binary units self.rbm[k] = rbm.rbm(layers[k], layers[k+1], input_type='binary')
def main(): preprocessFacebook('dataset_Facebook.csv', 'facebook.csv') data = np.loadtxt('facebook.csv', delimiter=';') data = data - data.mean(axis=0) imax = np.concatenate((data.max(axis=0) * np.ones( (1, 19)), np.abs(data.min(axis=0) * np.ones((1, 19)))), axis=0).max(axis=0) data = data / imax train = data[::2, :] trainTarget = data[::2, 18].reshape((np.shape(train)[0]), 1) valid = data[1::4, :] validTarget = data[1::4, 18].reshape((np.shape(valid)[0]), 1) test = data[3::4, :] testTarget = data[3::4, 18].reshape((np.shape(test)[0]), 1) net = mlp.mlp(train, trainTarget, 30, outtype='linear') net.earlystopping(train, trainTarget, valid, validTarget, 0.4, 500) test = np.concatenate((test, -np.ones((np.shape(test)[0], 1))), axis=1) testout = net.mlpfwd(test) pl.figure() pl.plot(np.arange(np.shape(test)[0]), testout, '.') pl.plot(np.arange(np.shape(test)[0]), testTarget, 'x') pl.legend(('Predictions', 'Targets')) pl.show()
def trainData(): with open('fixeddata.csv') as f: numCol = csv.reader(f, delimiter=',') first_row = next(numCol) numCol = len(first_row) print(numCol) cancerData = np.loadtxt('fixeddata.csv', delimiter=',', usecols=range(1, 11)).astype(np.float) print(np.shape(cancerData)) cancerData = cancerData / cancerData.max(axis=0) np.random.shuffle(cancerData) traindata = cancerData[0:349, 0:9] traindatat = cancerData[0:349, 9:10] datavalid = cancerData[349:523, 0:9] datavalidt = cancerData[349:523, 9:10] datatest = cancerData[523:699, 0:9] datatestt = cancerData[523:699, 9:10] #net = mlp.mlp(traindata,traindatat,40,outtype='linear') #net.mlptrain(traindata,traindatat,.005,650) net = mlp.mlp(traindata, traindatat, 10, outtype='linear') #net.mlptrain(traindata,traindatat,.4,1000) net.earlystopping(traindata, traindatat, datavalid, datavalidt, 0.35) net.confmat(datatest, datatestt)
def kfolds(movements, target, iterations=10, hidden=6, k=5, outtype='logistic'): n_networks = [] for i in range(k): test = movements[i::k,0:40] test_targets = target[i::k] valid = movements[((i+1)%k)::k,0:40] valid_targets = target[((i+1)%k)::k] train = [item for sublist in [movements[((i + j) % k)::k,0:40] for j in range(2, k)] for item in sublist] train_targets = [item for sublist in [target[((i + j) % k)::k] for j in range(2, k)] for item in sublist] # Initialize the network: net = mlp.mlp(train, train_targets, hidden, beta, momentum, outtype) # Store error and percentage correct: net.earlystopping(train, train_targets, valid, valid_targets,learning_rate) net.confusion(test, valid_targets) # Store the neural network: n_networks.append(net) percentages = [] errors = [] for i in range(len(n_networks)): percentages.append(n_networks[i].percentage) errors.append(n_networks[i].errors) print("Average Percentage: {}".format(statistics.mean(percentages))) print("Standard deviation: {}".format(statistics.stdev(percentages)))
def normal(movements, target, iterations=10, hidden=6, outtype='logistic'): # Split data into 3 sets # Training updates the weights of the network and thus improves the network train = movements[::2,0:40] train_targets = target[::2] # Validation checks how well the network is performing and when to stop valid = movements[1::4,0:40] valid_targets = target[1::4] # Test data is used to evaluate how good the completely trained network is. test = movements[3::4,0:40] test_targets = target[3::4] # Initialize the network: net = mlp.mlp(train, train_targets, hidden, beta, momentum, outtype) # Run training: net.earlystopping(train, train_targets, valid, valid_targets,learning_rate,iterations) # Check how well the network performed: print("HN: {} - Iter: {} - LR: {}".format(hidden, iterations, learning_rate)) print("Momentum: {} - Type: {}" .format(momentum, outtype)) net.confusion(test,test_targets)
def do(beta, eta, iterations, color = 'k'): nhidden = 2 momentum = 0.9 print "%s_beta-%f_eta-%f_iterations-%f" % (test, beta, eta, iterations) tron = mlp.mlp(classC, targetC, nhidden, beta, momentum, 'logistic') tron.mlptrain(classC, targetC, eta, iterations) xrange = numpy.arange( -4 , 4 , 0.1 ) yrange = numpy.arange(-4 , 4 , 0.1 ) xgrid,ygrid = numpy.meshgrid ( xrange , yrange ) noOfPoints = xgrid.shape[ 0 ] * xgrid.shape [ 1 ] xcoords = xgrid.reshape( ( noOfPoints , 1 ) ) ycoords = ygrid.reshape( ( noOfPoints , 1 ) ) samples = numpy.concatenate( ( xcoords , ycoords) , axis=1) ones = -numpy.ones ( xcoords.shape ) samples = numpy.concatenate ( ( samples , ones ) , axis =1) indicator = tron.mlpfwd ( samples ) indicator = indicator.reshape ( xgrid . shape ) pylab.contour ( xrange , yrange , indicator , ( 0.5 , ), colors=color )
def unseen_tissue_learn(self, unseen_train_loader, unseen_test_loader): unseen_tissue_model = mlp(self.feature_num, self.layer, self.hidden) # First need to copy the original meta learning model unseen_tissue_model.copy_weights(self.observed_tissue_model) unseen_tissue_model.cuda() #unseen_tissue_model.train() unseen_tissue_model.eval() unseen_opt = torch.optim.SGD(unseen_tissue_model.parameters(), lr=self.inner_lr) #unseen_opt = torch.optim.Adam(unseen_tissue_model.parameters(), lr=self.inner_lr, betas=(0.9, 0.99), eps=1e-05) # Here test_feature and test_label contains only one tissue info #unseen_train_loader, unseen_test_loader = get_unseen_data_loader(test_feature, test_label, K, args.inner_batch_size) for i in range(self.num_inner_updates): in_, target = unseen_train_loader.__iter__().next() loss, _ = forward_pass(unseen_tissue_model, in_, target) unseen_opt.zero_grad() loss.backward() unseen_opt.step() # Test on the rest of cell lines in this tissue (unseen_test_loader) mtrain_loss, mtrain_pear_corr, mtrain_spearman_corr, _, _ = evaluate_new( unseen_tissue_model, unseen_train_loader, 1) mtest_loss, mtest_pear_corr, mtest_spearman_corr, test_prediction, test_true_label = evaluate_new( unseen_tissue_model, unseen_test_loader, 0) return mtrain_loss, mtrain_pear_corr, mtrain_spearman_corr, mtest_loss, mtest_pear_corr, mtest_spearman_corr, test_prediction, test_true_label
def trainData(): with open('firesarranged.data') as f: numCol = len(f.readline().split()) fireData = np.loadtxt('firesarranged.data', dtype='S', delimiter=',', skiprows=1, usecols=range(13)).astype(np.float) #fireData[:,:13] = fireData[:,:13]-fireData[:,:13].mean(axis=0) #print(np.shape(fireData), np.shape(targets)) #imax = np.concatenate((fireData.max(axis=0)*np.ones((1,13)), np.abs(fireData.min(axis=0))*np.ones((1,13))), axis=0).max(axis=0) #targets[:,:1] = targets[:,:1]-targets[:,:1].mean(axis=0) #imax2 = np.concatenate((targets.max(axis=0)*np.ones((1)), np.abs(targets.min(axis=0))*np.ones((1))), axis=0).max(axis=0) #fireData[:,:13] = fireData[:,:13]/imax[:13] #print(np.shape(fireData)) #print("Before", fireData) #print(fireData) #targets [:,:1] = targets[:,:1]/imax2[:1] #print(np.shape(fireData), np.shape(targets)) #pl.plot(fireData,targets,'.') #pl.show() fireData = np.delete(fireData, np.where(fireData[0:517, 12:13] <= 0), 0) fireData[0:517, 12:13] = np.log(fireData[0:517, 12:13] + 1) fireData = fireData / fireData.max(axis=0) #targets = fireData[0:517,12:13] #print(np.shape(fireData[0:517,12:13])) #print(np.shape(targets)) #fireData = fireData[0:517,12:13][fireData[0:517,12:13] > 0] #print(np.shape(fireData)) #print(np.amin(fireData[0:270,12:13])) n, bins, patches = plt.hist(fireData[0:270, 12:13], 50, facecolor='green', alpha=0.75, rwidth=50) np.random.shuffle(fireData) plt.show() #print(imax) #print("Item", fireData.item(233), "Target", targets.item(233)) #print(np.shape(fireData[:,:12])) train = fireData[0:135, 2:12] traintarget = fireData[0:135, 12:13] test = fireData[135:202, 2:12] testtarget = fireData[135:202, 12:13] valid = fireData[202:267, 2:12] validtarget = fireData[202:267, 12:13] #net = mlp.mlp(train,traintarget,105,outtype='linear') #net.mlptrain(train,traintarget,0.1,1001) net = mlp.mlp(train, traintarget, 15, outtype='linear') net.mlptrain(train, traintarget, .3, 450) #net = mlp.mlp(traindata,traindatat,10,outtype='linear') #net.mlptrain(traindata,traindatat,.4,1000) net.earlystopping(train, traintarget, valid, validtarget, 0.3)
def build(self, x): ''' Build the coupling layers Args: x: tensor like, input tensor ''' name = 'coupling_'+str(self.ctr) with tf.variable_scope(name): # mask self.mask = tf.Variable(self.maskval, dtype=tf.float64, trainable=False) # input tensor xpp = tf.identity(x) self.xd = tf.multiply(xpp, self.mask) # defining s(•) and t(•) (MLPs) self.s = mlp(0, self.xd, self.size, self.depth, self.width, tf.nn.tanh) self.t = mlp(1, self.xd, self.size, self.depth, self.width) # feed forward output if (self.mode == 'ff'): self.xa = tf.multiply(xpp, tf.exp(self.s.y)) + self.t.y self.y = self.xd + tf.multiply(self.xa, 1-self.mask) # feed backward output elif (self.mode == 'fb'): self.xa = tf.multiply(tf.exp(-self.s.y), xpp-self.t.y) self.y = self.xd + tf.multiply(self.xa, 1-self.mask) else: raise ValueError('Inexistent mode for coupling layer') # jacobian determinant computation for the forward function (closed form) if (self.mode == 'ff'): self.detjac = tf.exp(tf.reduce_sum( tf.multiply(1-self.mask, self.s.y), axis=1)) self.detjactest = tf.matrix_determinant( tf.stack([tf.gradients(self.y[:, idx], x)[0] for idx in range(self.size)], axis=1)) elif (self.mode == 'fb'): self.detjac = tf.exp(-tf.reduce_sum( tf.multiply(1-self.mask, self.s.y), axis=1)) self.detjactest = tf.matrix_determinant( tf.stack([tf.gradients(self.y[:, idx], x)[0] for idx in range(self.size)], axis=1)) else: raise ValueError('Inexistent mode for coupling layer')
def add_biaffine_global(self, inputs): ## inputs [seq_len, batch_size, units] = [n, b, d] ## first define four different MLPs arc_roles = ['arc-dep', 'arc-head'] rel_roles = ['rel-dep', 'rel-head'] vectors = {} for arc_role in arc_roles: for i in xrange(self.opts.mlp_num_layers): if i == 0: inputs_dim = self.outputs_dim vector_mlp = inputs else: inputs_dim = self.opts.arc_mlp_units weights = get_mlp_weights('{}_MLP_Layer{}'.format(arc_role, i), inputs_dim, self.opts.arc_mlp_units) vector_mlp = self.add_dropout( tf.map_fn(lambda x: mlp(x, weights), vector_mlp), self.mlp_prob) vectors[arc_role] = vector_mlp ## [seq_len, batch_size, arc_mlp_units] arc_weights = get_arc_weights('arc', self.opts.arc_mlp_units) for rel_role in rel_roles: for i in xrange(self.opts.mlp_num_layers): if i == 0: inputs_dim = self.outputs_dim vector_mlp = inputs else: inputs_dim = self.opts.rel_mlp_units weights = get_mlp_weights('{}_MLP_Layer{}'.format(rel_role, i), inputs_dim, self.opts.rel_mlp_units) vector_mlp = self.add_dropout( tf.map_fn(lambda x: mlp(x, weights), vector_mlp), self.mlp_prob) vectors[rel_role] = vector_mlp ## [seq_len, batch_size, rel_mlp_units] rel_weights = get_rel_weights('rel', self.opts.rel_mlp_units, self.loader.nb_rels) global_weights = get_global_weights('global', self.opts.arc_mlp_units, self.loader.nb_rels) arc_output, rel_output, rel_scores = global_equation( vectors, arc_weights, rel_weights, global_weights) return arc_output, rel_output, rel_scores
def __init__(self, meta_dataset, fs_dataset, K, meta_lr, inner_lr, layer, hidden, tissue_num, meta_batch_size, inner_batch_size, num_updates, num_inner_updates, tissue_index_list, patience=3, num_trials=10): super(self.__class__, self).__init__() self.meta_dataset = meta_dataset self.fs_dataset = fs_dataset self.meta_batch_size = meta_batch_size self.inner_batch_size = inner_batch_size self.num_updates = num_updates self.num_inner_updates = num_inner_updates self.num_trials = num_trials self.hidden = hidden self.patience = patience self.feature_num = self.fs_dataset.feature.shape[1] self.K = K self.meta_lr = meta_lr self.inner_lr = inner_lr self.layer = layer self.hidden = hidden self.tissue_index_list = tissue_index_list self.tissue_num = tissue_num self.observed_tissue_model = mlp(self.feature_num, layer, hidden) self.observed_opt = torch.optim.Adam( self.observed_tissue_model.parameters(), lr=self.meta_lr, betas=(0.9, 0.99), eps=1e-05) self.inner_net = InnerLoop(self.num_inner_updates, self.inner_lr, self.feature_num, layer, hidden) #torch.cuda.manual_seed(args.seed) self.observed_tissue_model.cuda() self.inner_net.cuda()
def getFitness(string): n = int(string.replace('.',''), 2) # print "Checking fitness for: ", string # print "Converted to decimal: ", n net = mlp.mlp(train_in,train_tgt,n,outtype='softmax') train = np.concatenate((train_in,-np.ones((np.shape(train_in)[0],1))),axis=1) fitness = net.mlpfwd(train) fitness = ((0.5*np.sum(fitness**2))) if (math.isnan(fitness) or fitness == 0): fitness = 1 # fitness = net.confmat(test_in,test_tgt) #print "Fitness is: ", fitness return fitness
def train_and_eval(num_iterations, learning_rate, num_neurons, momentum): split = int(len(data) * .9) #no ten-fold. Use 90% for training and rest for validation train = data[:split, :] train_tgt = targets[:split] validation = data[split:, :] val_tgt = targets[split:] p = mlp.mlp(train, train_tgt, num_neurons, momentum=momentum, outtype='logistic') p.mlptrain(train, train_tgt, learning_rate, num_iterations) conf_mat = p.confmat(validation, val_tgt) acc = get_accuracy(conf_mat) return acc
def zero_shot_test(self, unseen_train_loader, unseen_vali_loader, unseen_test_loader): unseen_tissue_model = mlp(self.feature_num, self.layer, self.hidden) # First need to copy the original meta learning model unseen_tissue_model.copy_weights(self.observed_tissue_model) unseen_tissue_model.cuda() unseen_tissue_model.eval() train_performance = evaluate_cv(unseen_tissue_model, unseen_train_loader) vali_performance = evaluate_cv(unseen_tissue_model, unseen_vali_loader) test_performance = evaluate_cv(unseen_tissue_model, unseen_test_loader) return train_performance, vali_performance, test_performance, np.mean( tissue_loss)
def __init__(self, options, channel): """ options: a dictionary contains all the configurations channel: jobman channel """ # Step 0. Load data print 'Loading data' data = numpy.load(options['data']) self.options = options self.channel = channel # Step 1. Construct Model print 'Constructing Model' if options['model'] == 'mlp': model = mlp(options, channel, data) elif options['model'] == 'daa': model = daa(options, channel, data) self.model = model print 'Constructing algo' # Step 2. Construct optimization technique if options['algo'] == 'natSGD_basic': algo = natSGD(options, channel, data, model) elif options['algo'] == 'natSGD_jacobi': algo = natSGD_jacobi(options, channel, data, model) elif options['algo'] == 'natSGD_ls': algo = natSGD_linesearch(options, channel, data, model) elif options['algo'] == 'natNCG': algo = natNCG(options, channel, data, model) elif options['algo'] == 'krylov': algo = KrylovDescent(options, channel, data, model) elif options['algo'] == 'hf': raise NotImplemented elif options['algo'] == 'hf_jacobi': raise NotImplemented elif options['algo'] == 'sgd': algo = SGD(options, channel, data, model) self.algo = algo self.options['validscore'] = 1e20 self.train_timing = numpy.zeros((options['loopIters'], 13), dtype='float32') self.valid_timing = numpy.zeros((options['loopIters'], 2), dtype='float32') if self.channel is not None: self.channel.save() self.start_time = time.time() self.batch_start_time = time.time()
def mlp_process(data, labels): w = 2 h = len(data) target = [[0 for model in range(w)] for y in range(h)] target = np.array(target) # print(labels) for i in range(data.shape[0]): if (labels[i] == 1): target[i, 0] = 1 elif (labels[i] == 2): target[i, 1] = 1 # elif(labels[i] == 3): # target[i,2] = 1 # elif(labels[i] == 4): # target[i,3] = 1 # elif(labels[i] == 5): # target[i,4] = 1 # elif(labels[i] == 6): # target[i,5] = 1 # elif(labels[i] == 7): # target[i,6] = 1 # elif(labels[i] == 8): # target[i,7] = 1 # print(target) #split the data into train, test and validation data for mlp #Set up Neural Network # x_train, y_train, x_valid, y_valid = train_test_split(data, target, test_size = 0.3, random_state = 0) x_train = data[0::2, :] x_valid = data[1::4, :] x_test = data[3::4, :] y_train = target[0::2, :] y_valid = target[1::4, :] y_test = target[3::4, :] hidden_layers = 50 learning_rate = 0.1 net = mlp.mlp(x_train, y_train, hidden_layers, outtype='softmax') net.earlystopping(x_train, y_train, x_valid, y_valid, learning_rate) net.confmat(x_test, y_test) return net
def learn_nonlinear_mapping(hid_layers=4): hid_layers=3 train=CBK[0::2,:]/100. SFR=vectorize_codebook(STK,FRA,REG) target=SFR[0::2,-3:] print train.shape, target.shape net = mlp(train,target,hid_layers,outtype='linear') valid=CBK[1::4,:]/100. valtarg=SFR[1::4,-3:] print valid.shape, valtarg.shape #net.mlptrain(train,target,0.9,100) net.earlystopping(train,target,valid,valtarg,0.9)#eta was 0.25 test=CBK[3::4,:]/100. testtarg=SFR[3::4,-3:] print test.shape, testtarg.shape exptest = np.concatenate((test,-np.ones((np.shape(test)[0],1))),axis=1) testout=net.mlpfwd(exptest) print testtarg[:3,:] print testout[:3,:]
def __init__(self, options, channel): # Step 0. Load data print 'Loading data' data = numpy.load(options['data']) self.options = options self.channel = channel # Step 1. Construct Model print 'Constructing Model' if options['model'] == 'mlp': model = mlp(options, channel, data) elif options['model'] == 'daa': model = daa(options, channel, data) self.model = model print 'Constructing algo' # Step 2. Construct optimization technique if options['algo'] == 'natSGD_basic': algo = natSGD(options, channel, data, model) elif options['algo'] == 'natSGD_jacobi': algo = natSGD_jacobi(options, channel, data, model) elif options['algo'] == 'natSGD_ls': algo = natSGD_linesearch(options, channel, data, model) elif options['algo'] == 'natNCG': algo = natNCG(options, channel, data, model) elif options['algo'] == 'krylov': algo = KrylovDescent(options, channel, data, model) elif options['algo'] == 'hf': raise NotImplemented elif options['algo'] == 'hf_jacobi': raise NotImplemented elif options['algo'] == 'sgd': algo = SGD(options, channel, data, model) self.algo = algo self.options['validscore'] = 1e20 self.options['testscore'] = 1e20 self.train_timing = numpy.zeros((options['loopIters'], 15), dtype='float32') self.valid_timing = numpy.zeros((options['loopIters'],), dtype='float32') self.test_timing = numpy.zeros((options['loopIters'],), dtype='float32') if self.channel is not None: self.channel.save() self.start_time = time.time() self.batch_start_time = time.time()
def cross_validation(data, targets, test, test_targets, nhidden=5, k=10): data_sp = np.split(data, k) size = int(len(data) / k) targets_sp = np.split(targets, k) errs = np.empty(k) accs = np.empty(k) for i in range(k): t_ind = np.arange(len(data)) t_ind = np.delete(t_ind, t_ind[i * size:(i + 1) * size]) print(t_ind) valid = data_sp[i] valid_targets = targets_sp[i] train = data[t_ind] train_targets = targets[t_ind] print(train.shape) print(train_targets.shape) net = mlp.mlp(train, train_targets, nhidden) val_err, val_acc, tr_err, tr_acc = net.earlystopping( train, train_targets, valid, valid_targets, 50) plt.plot(val_err) plt.figure() plt.plot(val_acc) plt.figure() plt.plot(tr_err) plt.figure() plt.plot(tr_acc) plt.show() errs[i], accs[i] = net.test(test, test_targets) print("Error of %d fold is: " % (i + 1), errs[i]) print("Accuracy of %d fold is: " % (i + 1), accs[i]) print("Mean error is: ", np.mean(errs)) print("Standard deviation of error is: ", np.std(errs)) print("Mean accuracy is: ", np.mean(accs)) print("Standard deviation of accuracy is: ", np.std(accs))
def getMatrix(nHidden, momentum): info = np.loadtxt('normalizeOutput.txt', delimiter=',') np.random.shuffle(info) ###START Split between test and valid sample### trainInputs = [] trainTargets = [] validInputs = [] validTargets = [] testInputs = [] testTargets = [] for sample in info[:len(info) // 5]: new = [0, 0] new[int(sample[-1])] = 1 np.array(new) validInputs.append(sample[:-1]) validTargets.append(new) for sample in info[len(info) // 5:2 * (len(info) // 5)]: new = [0, 0] new[int(sample[-1])] = 1 np.array(new) testInputs.append(sample[:-1]) testTargets.append(new) for sample in info[2 * (len(info) // 5):]: new = [0, 0] new[int(sample[-1])] = 1 np.array(new) trainInputs.append(sample[:-1]) trainTargets.append(new) ###END Split between test and valid sample### trainInputs = np.array(trainInputs) trainTargets = np.array(trainTargets) trainInputs = np.array(trainInputs) trainTargets = np.array(trainTargets) testInputs = np.array(testInputs) testTargets = np.array(testTargets) net = mlp.mlp(trainInputs, trainTargets, nHidden, momentum=momentum) net.earlystopping(trainInputs, trainTargets, validInputs, validTargets, 0.3) net.confmat(testInputs, testTargets)
def train_eval_ten_fold(num_iterations, learning_rate, num_neurons, momentum): total = 0 leave = 0 for i in range(10): start = int(len(data) * leave) end = int(len(data) * (leave + .1)) train = np.concatenate((data[:start, :], data[end:, :])) train_tgt = np.concatenate((targets[:start, :], targets[end:, :])) validation = data[start:end, :] val_tgt = targets[start:end, :] p = mlp.mlp(train, train_tgt, num_neurons, momentum=momentum, outtype='logistic') p.mlptrain(train, train_tgt, learning_rate, num_iterations) conf_mat = p.confmat(validation, val_tgt) acc = get_accuracy(conf_mat) total += acc return total / 10
def fine_tune(self, data, target, depth='deep', batch_size=1, learning_rate=0.1, epochs=10, momentum=0, weight_decay=0): ''' Discriminative fine tuning of MLP weights. In most cases pre_train() should be called first ''' # fine-tune all layers if depth=='deep': error = self.mlp.train(data, target, batch_size, learning_rate, epochs, momentum, weight_decay) # fine-tune deepest layer only elif depth=='shallow': perceptron = mlp.mlp([self.layers[-2], self.layers[-1]]) error = perceptron.train( self.calc_rep(data), target, batch_size, learning_rate, epochs, momentum ) # copy weights into last layer self.mlp.W[-1] = np.copy(perceptron.W[-1]) self.mlp.b[-1] = np.copy(perceptron.b[-1]) else: raise Exception("Unknown depth. Currently recognized options are 'deep' or 'shallow'") return error
def initialise_data (): # Preprocessing data data = np.loadtxt('spambase/spambase.data',delimiter=',') data[:,:57] = data[:,:57]-data[:,:57].mean(axis=0) imax = np.concatenate((data.max(axis=0)*np.ones((1,58)),np.abs(data.min(axis=0)*np.ones((1,58)))),axis=0).max(axis=0) data[:,:57] = data[:,:57]/imax[:57] # 1-of-N encoding target = np.zeros((np.shape(data)[0],2)); indices = np.where(data[:,57]==0) target[indices,0] = 1 indices = np.where(data[:,57]==1) target[indices,1] = 1 #Take half of data for testing and compute fitness test = data[::2,0:57] testt = target[::2] #initialise MLP model,and using 2 hiddnes nodes and 'softmax' activation function net = mlp.mlp(test,testt,1,outtype='softmax') return net,test,testt
def evaluateLearning(): """Main runner method, can be modified to loop through different hidden layers, note I have changed to MLPtrain method to return the x and y values of the errors. (Iteration and error amount) """ figure(1) #Change this to use the mlp with different layers #for x in [1,2,3,5,10,15]: #for x in [13,14,15,16,17]: for x in [15]: print x net = mlp.mlp(inputs, inputs, x, outtype='logistic') y, x1 = net.mlptrain(inputs, inputs, 0.25, 20000) net.confmat(inputs, inputs) net.confmat(noiseinputs1, inputs) net.confmat(noiseinputs2, inputs) plot(x1, y, '.') xlabel('Number of Learning Iterations') ylabel('Learning Error (Log 10)') show()
def main(argv): args = parseCmd(argv) initLogging(args) # Set up the data # x,t = getData1() x, t = getData2() # # Split into training, testing, and validation sets train = x[0::2, :] test = x[1::4, :] valid = x[3::4, :] traintarget = t[0::2, :] testtarget = t[1::4, :] validtarget = t[3::4, :] # Perform basic training with a small MLP net = mlp.mlp(train, traintarget, 20, outtype='linear') error = net.earlystopping(train, traintarget, valid, validtarget, 0.25) # show result for test values biasedTest = np.concatenate((test, -np.ones((np.shape(test)[0], 1))), axis=1) outputs = net.mlpfwd(biasedTest) print "Error: " + str(error) print "Test error: " + str((0.5 * sum((outputs - testtarget) ** 2))[0]) with open("result.dat", "w") as f: for i in range(0, len(biasedTest)): f.write(str(biasedTest[i][0]) + " " + str(biasedTest[i][1]) + " " + str(outputs[i][0]) + "\n") with open("network_in.dat", "w") as f: for i, w1 in enumerate(net.weights1): for j, w in enumerate(w1): f.write(str(i) + " " + str(j) + " " + str(w) + "\n") f.write("\n") with open("network_hidden.dat", "w") as f: for w2 in net.weights2: f.write(str(w2[0]) + "\n")
def getFitness(string): # Gather the first 5 bytes to calculate the number of hidden nodes, set one at random to off hiddenNodesActivation=[0,0,0,0,0] weightAct = [] for i in xrange(5): hiddenNodesActivation[i]=string[i] x = random.randint(0,4) hiddenNodesActivation[x]=0 array=np.array([16,8,4,2,1]) hiddenNodesActivation=np.array(hiddenNodesActivation) hiddenNodes=np.sum(hiddenNodesActivation*array) hiddenNodes = int(hiddenNodes)+1 # create the activation matric for the weights, depends in the number of hidden nodes, set one to off at random for i in range(44*hiddenNodes): weightAct.append(string[i+5]) x = random.randint(0,len(weightAct)-1) weightAct[x] = 0 weightAct = np.array(weightAct).reshape((44,hiddenNodes)) # create mlp and turn of weights ass set above net = mlp.mlp(train_in,train_tgt,hiddenNodes,outtype='softmax') weight1 = net.weights1 net.weights1 = weight1*weightAct # Take a few steps through the MLP and then check the percent correct, return that as fitness # edit MLP to return % correct valid = np.concatenate((train_in,-np.ones((np.shape(train_in)[0],1))),axis=1) count = 0 while (count <50): net.mlpfwd(valid) count +=1 return net.confmat(test_in,test_tgt)
def evaluateLearning(): """Main runner method, can be modified to loop through different hidden layers, note I have changed to MLPtrain method to return the x and y values of the errors. (Iteration and error amount) """ figure(1) #Change this to use the mlp with different layers #for x in [1,2,3,5,10,15]: #for x in [13,14,15,16,17]: for x in [15]: print x net = mlp.mlp(inputs,inputs,x,outtype='logistic') y, x1 = net.mlptrain(inputs,inputs,0.25,20000) net.confmat(inputs,inputs) net.confmat(noiseinputs1,inputs) net.confmat(noiseinputs2,inputs) plot(x1,y,'.') xlabel('Number of Learning Iterations') ylabel('Learning Error (Log 10)') show()
def run_mlp(): ''' Run a MLP on using pre-processed pollen data. The percentage correct is stored in an array. ''' x = preprocess.preprocess('Pollens') pollen = np.array(x.create_one_file(SIMPLE_GRASS)) pollen = x.normalise_max(pollen) train_set, train_set_target, test_set, test_set_target, validation_set, validation_set_target = x.make_groups( pollen, LABEL_SIZE, train_size=350, test_size=150, validation_size=150) p = mlp.mlp(train_set, train_set_target, 30, momentum=0.9, outtype='softmax') error = p.earlystopping(train_set, train_set_target, validation_set, validation_set_target, 0.1, niterations=200) correct = p.confmat(test_set, test_set_target)
def cross_validation(self, X, Y, M, lr, max_iters, batch_size, k, act_func): fold_size = int(np.floor(X.shape[0] / k)) accuracy_ls = [] folds_X, folds_Y = self.create_mini_batch(X, Y, fold_size) folds_X = np.asarray(folds_X) folds_Y = np.asarray(folds_Y) for i in range(k): X_test = folds_X[i] Y_test = folds_Y[i] X_train = np.concatenate(np.delete(folds_X, i, 0), axis=0) Y_train = np.concatenate(np.delete(folds_Y, i, 0), axis=0) N, D = X_train.shape N, K = Y_train.shape W = np.random.randn(M, K) * 0.01 V = np.random.randn(D, M) * 0.01 mlp_nn = mlp.mlp(W, V) mlp_nn.fit(X_train, Y_train, M, lr, max_iters, batch_size, act_func) predictions, accuracy = mlp_nn.predict(X_test, Y_test, act_func) accuracy_ls.append(accuracy) # if (act_func == 'ReLu'): # # elif (act_func == 'Tanh'): # # do something # print('Using Tanh') # # elif(act_func == 'Sigmoid'): # # do something # print('Using Sigmoid') return accuracy_ls, np.mean(accuracy_ls)
def k_fold_cross_validation(self, inputs, targets, early_stop_test, early_stop_test_target, num_folds, eta): n = np.shape(inputs)[0] size_list = range(100, n, num_folds) average_accuracy_list = [] for size in size_list: indices = range(0, size, int(size / num_folds)) accuracy_list = [] mlp_helper = mlp.mlp(7, int(size / num_folds) * (num_folds - 1), 1, 19) for i in indices: validation_set = inputs[i:i + int(size / num_folds), :] validation_targets = targets[i:i + int(size / num_folds), :] train_set = np.concatenate((inputs[:i, :], inputs[i + int(size / num_folds):size, :])) train_targets = np.concatenate((targets[:i, :], targets[i + int(size / num_folds):size, :])) accuracy_list.append(float( self.early_stop(train_set, train_targets, validation_set, validation_targets, eta, mlp_helper, early_stop_test, early_stop_test_target))) average_accuracy_list.append(np.mean(accuracy_list)) print(average_accuracy_list) plt.plot(size_list, average_accuracy_list, 'g^') plt.axis([size_list[0], size_list[len(size_list) - 1], 0, 100]) plt.show() size_list = np.transpose(np.array(list([size_list]))) average_accuracy_list = np.transpose(np.array(list([average_accuracy_list]))) table = tabulate(np.concatenate((size_list, average_accuracy_list), axis=1), headers=['apps taken', 'average accuracy(%)'], floatfmt='.3f') with open('k_fold_X_validation_observations.txt', 'w') as f: f.write(table)
valid_targets = target[1::4] # Test data is used to evaluate how good the completely trained network is. test = movements[3::4,0:40] test_targets = target[3::4] return train, train_targets, valid, valid_targets, test, test_targets #end prepare_data ######### ## Run # ####### input_n_epoch = 10000 # input("Type number of epochs (tested on 10, 100 and 1000): ") input_n_neuron = 18# input("Type number of neurons in hidden layer (tested on 6, 8 and 12): ") train, train_targets, valid, valid_targets, test, test_targets = prepare_data() X = train target = train_targets number_of_neurons_in_input = X.shape[1] number_of_targets = target.shape[1] number_of_neurons_in_hidden = int(input_n_neuron) #12 #should test with 6, 8, 12 number_of_epochs = int(input_n_epoch) #1000 #10, 100, 1000 net = mlp(number_of_neurons_in_input, number_of_neurons_in_hidden, number_of_targets, beta=5) net.early_stopping(X, target, valid, valid_targets, number_of_epochs) threshold_value = 0.5 net.confusion(test, test_targets, threshold_value)
def irismain(): #1. make iris.data in usable form #2. make input set and output set out of it #3. make setpool out of the dataset #4. make pcn and train it #5. test on validation and testing set convert_iris() irisdata = np.loadtxt( "/home/swapnil/forgit/neuro-evolution/05/dataset/iris/newiris.data", delimiter=',') nin = 4 # for four features of iris nout = 3 # for 3 sets of iris flowers minerr = 10000000 lis = [] order = np.arange(np.shape(irisdata)[0]) np.random.shuffle(order) irisdata = irisdata[order, :] irisdata[:, :4] = irisdata[:, :4] - irisdata[:, :4].mean(axis=0) imax = np.concatenate((irisdata.max(axis=0) * np.ones( (1, 7)), np.abs(irisdata.min(axis=0)) * np.ones((1, 7))), axis=0).max(axis=0) irisdata[:, :4] = irisdata[:, :4] / imax[:4] errcal = 'confmat' eta = 0.29 niterations = 500 tlis = [] for niterations in range(10, 1000, 10): minitererr = 10000000 flag = 0 for nhidden in range(1, 6): # range for number of hidden nodes minnhiddenerr = 100000000 for tupoftup in nextpartition(irisdata, nin, nout): train, traintarget = tupoftup[0] valid, validtarget = tupoftup[ 1] #each row of setpool is input and their targets so we need to seperate them test, testtarget = tupoftup[2] #np.concatenate((train,valid),axis=0) #np.concatenate((traintarget,validtarget),axis=0) #valid is of no use on perceptron because perceptron can not overfit!! and neither is early-stopping. net = mlp.mlp(train, traintarget, nhidden, outtype='logistic') net.mlptrain(train, traintarget, eta, niterations // 2) validmeanerr = net.earlystopping(train, traintarget, valid, validtarget, eta, niterations // 10, errcaltype=errcal) print("no. of nodes", nhidden) lis.append(validmeanerr) if errcal == 'squaremean': trainmeanerr = net.findmeantrainerr(train, traintarget) else: trainmeanerr = net.confmat(train, traintarget) print("validation error: %f trainerr error:%f" % (validmeanerr, trainmeanerr)) #y=np.concatenate((x,-np.ones((np.shape(x)[0],1))),axis=1) minnhiddenerr = min(minnhiddenerr, validmeanerr) if validmeanerr < minerr: #see I can't use equal to here so that this way it will select one with lowest num of nodes minerr = validmeanerr bestnet = trainedmlp.trainedmlp(net, test, testtarget, trainmeanerr, validmeanerr, nhidden) if minnhiddenerr < 0.07: tempnhidden = nhidden flag = 1 break if not flag: tempnhidden = 10 tlis.append((niterations, tempnhidden)) print(lis) niterationslis = [i[0] for i in tlis] temphiddenlis = [i[1] for i in tlis] iterarr = np.array(niterationslis) * np.ones((len(niterationslis), 1)) nhiddenarr = np.array(temphiddenlis) * np.ones((len(temphiddenlis), 1)) pl.plot(iterarr, nhiddenarr, '.') #pl.plot(x,,'o') pl.xlabel('iter') pl.ylabel('nhidden') pl.show() print("\n best network is attained with no. of nodes as ", bestnet.numnodes) leasterr = bestnet.test() print("error on test is %f while on valid is %f" % (leasterr, bestnet.validmeanerr))
import numpy as np from mlp import mlp and_data = np.array([[0, 0, 0], [0, 1, 0], [1, 0, 0], [1, 1, 1]]) xor_data = np.array([[0, 0, 0], [0, 1, 1], [1, 0, 1], [1, 1, 1]]) X_train = and_data[:, :2] y_train = and_data[:, 2:] eta = 0.25 n_iterations = 1001 p = mlp(X_train, y_train, 2) p.mlptrain(X_train, y_train, eta, n_iterations) p.confmat(X_train, y_train)
# Split into training, testing, and validation sets train = x[0::2,:] test = x[1::4,:] valid = x[3::4,:] traintarget = t[0::2,:] testtarget = t[1::4,:] validtarget = t[3::4,:] # Plot the data plot(x,t,'o') xlabel('x') ylabel('t') # Perform basic training with a small MLP import mlp net = mlp.mlp(train,traintarget,3,outtype='linear') net.mlptrain(train,traintarget,0.25,101) # Use early stopping net.earlystopping(train,traintarget,valid,validtarget,0.25) # Test out different sizes of network #count = 0 #out = zeros((10,7)) #for nnodes in [1,2,3,5,10,25,50]: # for i in range(10): # net = mlp.mlp(train,traintarget,nnodes,outtype='linear') # out[i,count] = net.earlystopping(train,traintarget,valid,validtarget,0.25) # count += 1 # #test = concatenate((test,-ones((shape(test)[0],1))),axis=1)
# by Stephen Marsland (http://seat.massey.ac.nz/personal/s.r.marsland/MLBook.html) # You are free to use, change, or redistribute the code in any way you wish for # non-commercial purposes, but please maintain the name of the original author. # This code comes with no warranty of any kind. # Stephen Marsland, 2008 # Kristian Valentin <*****@*****.**>, 2011 import numpy as np import mlp anddata = np.array([[0,0,0],[0,1,0],[1,0,0],[1,1,1]]) xordata = np.array([[0,0,0],[0,1,1],[1,0,1],[1,1,0]]) p = mlp.mlp(anddata[:,0:2],anddata[:,2:3],1) p.mlptrain(anddata[:,0:2],anddata[:,2:3],0.25,1001) p.confmat(anddata[:,0:2],anddata[:,2:3]) q = mlp.mlp(xordata[:,0:2],xordata[:,2:3],2,beta=.6,outtype='logistic') q.mlptrain(xordata[:,0:2],xordata[:,2:3],0.2,5001,verbose=True) q.confmat(xordata[:,0:2],xordata[:,2:3]) #anddata = np.array([[0,0,1,0],[0,1,1,0],[1,0,1,0],[1,1,0,1]]) #xordata = np.array([[0,0,1,0],[0,1,0,1],[1,0,0,1],[1,1,1,0]]) # #p = mlp.mlp(anddata[:,0:2],anddata[:,2:4],2,outtype='linear') #p.mlptrain(anddata[:,0:2],anddata[:,2:4],0.25,1001) #p.confmat(anddata[:,0:2],anddata[:,2:4]) # #q = mlp.mlp(xordata[:,0:2],xordata[:,2:4],2,outtype='linear')
target = target[order, :] # Split data into 3 sets # Training updates the weights of the network and thus improves the network train = movements[::2, 0:40] train_targets = target[::2] # Validation checks how well the network is performing and when to stop valid = movements[1::4, 0:40] valid_targets = target[1::4] # Test data is used to evaluate how good the completely trained network is. test = movements[3::4, 0:40] test_targets = target[3::4] # Try networks with different number of hidden nodes: hidden = 12 # Initialize the network: net = mlp.mlp(train, train_targets, hidden) # Run training: net.earlystopping(train, train_targets, valid, valid_targets) # NOTE: You can also call train method from here, # and make train use earlystopping method. # This is a matter of preference. # Check how well the network performed: net.confusion(test, test_targets)
mlp_train_score = [] mlp_valid_score = [] test_scores = [] for train_indices, valid_indices, test_indices in kf: train = iris[train_indices] train_tgt = targets[train_indices] valid = iris[valid_indices] valid_tgt = targets[valid_indices] test = iris[test_indices] test_tgt = targets[test_indices] """MLP""" mlpnet = mlp.mlp(train, train_tgt, 20, beta=1, momentum=0.9, outtype='logistic') mlpnet.mlptrain(train, train_tgt, 0.01, 1000, valid, valid_tgt) #Get training and validation scores on this Fold mlp_train_score.append(mlpnet.train_scores_list) mlp_valid_score.append(mlpnet.valid_scores_list) test_scores.append(mlpnet.score(test, test_tgt)) print test_scores print np.sum(test_scores) mlp_train_score = np.array(mlp_train_score) mlp_valid_score = np.array(mlp_valid_score)
target = zeros((shape(data)[0],3)) indices = where(data[:,p]==0) target[indices,0] = 0 indices = where(data[:,p]==1) target[indices,0] = 1 indices = where(data[:,p]==2) target[indices,0] = 2 # Randomly order the data order = range(shape(data)[0]) random.shuffle(order) data = data[order,:] target = target[order,:] # Split into training, validation, and test sets train = data[::2,0:p] traint = target[::2] valid = data[1::4,0:p] validt = target[1::4] test = data[3::4,0:p] testt = target[3::4] #print train.max(axis=0), train.min(axis=0) # Train the network import mlp net = mlp.mlp(train,traint,5,outtype='softmax') net.earlystopping(train,traint,valid,validt,0.1) net.confmat(test,testt)
train = movements[::2,0:40] traint = target[::2] valid = movements[1::4,0:40] validt = target[1::4] test = movements[3::4,0:40] testt = target[3::4] # Train the network k_fold = False fold_number = 4 if not k_fold: net = mlp.mlp(train,traint) net.earlystopping(train, traint, valid, validt) net.confusion(test,testt) else: movements = movements[::,0:40] movements = movements.tolist() target = target.tolist() folds = [] tfolds = [] fold_size = len(movements) / fold_number for i in range(0, fold_number): fold = [] tfold = [] for j in range(0, fold_size): select = random.randint(0, len(movements) - 1) fold.append(movements[select])
f = gzip.open('../Data/mnist.pkl.gz','rb') tset, vset, teset = cPickle.load(f) f.close() nread = 200 # Just use the first few images train_in = tset[0][:nread,:] # This is a little bit of work -- 1 of N encoding # Make sure you understand how it does it train_tgt = np.zeros((nread,10)) for i in range(nread): train_tgt[i,tset[1][i]] = 1 test_in = teset[0][:nread,:] test_tgt = np.zeros((nread,10)) for i in range(nread): test_tgt[i,teset[1][i]] = 1 # We will need the validation set valid_in = vset[0][:nread,:] valid_tgt = np.zeros((nread,10)) for i in range(nread): valid_tgt[i,vset[1][i]] = 1 for i in [1,2,5,10,20]: print "----- "+str(i) net = mlp.mlp(train_in,train_tgt,i,outtype='softmax') net.earlystopping(train_in,train_tgt,valid_in,valid_tgt,0.1) net.confmat(test_in,test_tgt)
# You are free to use, change, or redistribute the code in any way you wish for # non-commercial purposes, but please maintain the name of the original author. # This code comes with no warranty of any kind. # Stephen Marsland, 2008 from numpy import * import mlp anddata = array([[0,0,0],[0,1,0],[1,0,0],[1,1,1]]) xordata = array([[0,0,0],[0,1,1],[1,0,1],[1,1,0]]) p = mlp.mlp(anddata[:,0:2],anddata[:,2:3],2) p.mlptrain(anddata[:,0:2],anddata[:,2:3],0.25,1001) p.confmat(anddata[:,0:2],anddata[:,2:3]) q = mlp.mlp(xordata[:,0:2],xordata[:,2:3],2,outtype='logistic') q.mlptrain(xordata[:,0:2],xordata[:,2:3],0.25,5001) q.confmat(xordata[:,0:2],xordata[:,2:3]) #anddata = array([[0,0,1,0],[0,1,1,0],[1,0,1,0],[1,1,0,1]]) #xordata = array([[0,0,1,0],[0,1,0,1],[1,0,0,1],[1,1,1,0]]) # #p = mlp.mlp(anddata[:,0:2],anddata[:,2:4],2,outtype='linear') #p.mlptrain(anddata[:,0:2],anddata[:,2:4],0.25,1001) #p.confmat(anddata[:,0:2],anddata[:,2:4]) # #q = mlp.mlp(xordata[:,0:2],xordata[:,2:4],2,outtype='linear')
pima = np.loadtxt('C:\Users\subha\PycharmProjects\MLP_Pima/pima-indians-diabetes.data',delimiter=',') # Plot the first and second values for the two classes indices0 = np.where(pima[:,8]==0) indices1 = np.where(pima[:,8]==1) pima[:,:8] = pima[:,:8]-pima[:,:8].mean(axis=0) imax = np.concatenate((pima.max(axis=0)*np.ones((1,9)),np.abs(pima.min(axis=0)*np.ones((1,9)))),axis=0).max(axis=0) pima[:,:8] = pima[:,:8]/imax[:8] print"\n" p = mlp.mlp(pima[:,:8],pima[:,8:9],5,outtype='logistic') p.mlptrain(pima[:,:8],pima[:,8:9],0.25,500) print"\n" p.confmat(pima[:,:8],pima[:,8:9]) print "output after preprocessing the data" train = pima[::2,:8] traint = pima[::2,8:9] valid = pima[1::8,0:8] validt = pima[1::8,8:9] test = pima[1::2,:8] testt = pima[1::2,8::9] import mlp
# Split into training, validation, and test sets target = zeros((shape(iris)[0], 3)) indices = where(iris[:, 4] == 0) target[indices, 0] = 1 indices = where(iris[:, 4] == 1) target[indices, 1] = 1 indices = where(iris[:, 4] == 2) target[indices, 2] = 1 # Randomly order the data order = range(shape(iris)[0]) random.shuffle(order) iris = iris[order, :] target = target[order, :] train = iris[::2, 0:4] traint = target[::2] valid = iris[1::4, 0:4] validt = target[1::4] test = iris[3::4, 0:4] testt = target[3::4] # print train.max(axis=0), train.min(axis=0) # Train the network import mlp net = mlp.mlp(train, traint, 5, outtype="softmax") net.earlystopping(train, traint, valid, validt, 0.1) net.confmat(test, testt)
target = target[order,:] # Split data into 3 sets # Training updates the weights of the network and thus improves the network train = movements[::2,0:40] train_targets = target[::2] # Validation checks how well the network is performing and when to stop valid = movements[1::4,0:40] valid_targets = target[1::4] # Test data is used to evaluate how good the completely trained network is. test = movements[3::4,0:40] test_targets = target[3::4] # Try networks with different number of hidden nodes: hidden = 12 # Initialize the network: net = mlp.mlp(train, train_targets, hidden) # Run training: net.earlystopping(train, train_targets, valid, valid_targets) # NOTE: You can also call train method from here, # and make train use earlystopping method. # This is a matter of preference. # Check how well the network performed: net.confusion(test,test_targets)
for i in range(len(set_size)): if i > 0: set_size[i] = set_size[i] + set_size[i - 1] train = data[:set_size[0],0:col-output_size] traint = target[:set_size[0]] valid = data[set_size[0]:set_size[1],0:col-output_size] validt = target[set_size[0]:set_size[1]] test = data[set_size[1]:set_size[2],0:col-output_size] testt = target[set_size[1]:set_size[2]] # Train the network # MLP if run[0]: import mlp as mlp_module mlp = mlp_module.mlp(train,traint,nhidden,outtype='softmax') mlp.earlystopping(train,traint,valid,validt,eta) mlp_correct_pct = mlp.confmat(test,testt) if save_results: mlp.get_data(resultPath, filename, SEED) print 'MLP weights saved' #RBF if run[1]: import rbf as rbf_module rbf = rbf_module.rbf(train,traint,nRBF,1,1) rbf.rbftrain(train,traint,0.25,2000) rbf_correct_pct = rbf.confmat(test,testt) if save_results: rbf.get_data(resultPath + filename) print 'RBF weights saved'
# Split into training, validation, and test sets target = np.zeros((np.shape(iris)[0], 3)); indices = np.where(iris[:, 4] == 0) target[indices, 0] = 1 indices = np.where(iris[:, 4] == 1) target[indices, 1] = 1 indices = np.where(iris[:, 4] == 2) target[indices, 2] = 1 # Randomly order the data order = list(range(np.shape(iris)[0])) np.random.shuffle(order) iris = iris[order, :] target = target[order, :] train = iris[::2, 0:4] traint = target[::2] valid = iris[1::4, 0:4] validt = target[1::4] test = iris[3::4, 0:4] testt = target[3::4] # print train.max(axis=0), train.min(axis=0) # Train the network import mlp net = mlp.mlp(train, traint, 5, outtype='logistic') net.earlystopping(train, traint, valid, validt, 0.1) net.confmat(test, testt)
# Read the dataset in (code from sheet) f = gzip.open('mnist.pkl.gz', 'rb') tset, vset, teset = pickle.load(f, encoding='iso-8859-1') f.close() nread = 200 # Just use the first few images train_in = tset[0][:nread, :] # This is a little bit of work -- 1 of N encoding # Make sure you understand how it does it train_tgt = np.zeros((nread, 10)) for i in range(nread): train_tgt[i, tset[1][i]] = 1 test_in = teset[0][:nread, :] test_tgt = np.zeros((nread, 10)) for i in range(nread): test_tgt[i, teset[1][i]] = 1 # We will need the validation set valid_in = vset[0][:nread, :] valid_tgt = np.zeros((nread, 10)) for i in range(nread): valid_tgt[i, vset[1][i]] = 1 for i in [1, 2, 5, 10, 20]: print("----- " + str(i)) net = mlp.mlp(train_in, train_tgt, i, outtype='softmax') net.earlystopping(train_in, train_tgt, valid_in, valid_tgt, 0.1) net.confmat(test_in, test_tgt)
print_NN_params() #remind us what architecture was tested start = time.time() if train_func == 'B': '''BackPropagation makes use of mlp.py and mlp2.py, codes for 1 and 2 layer Networks respectively, written by Stephen Marsland (check modules for ref) ''' import mlp, mlp2 #, mlp_threaded - Not implemented yet signal.signal(signal.SIGINT, signal_handler) #register the signal handler #Build the network if nlayers > 1: net = mlp2.mlp(train,traint, nhidden, nhiddeno, outtype='softmax') elif nlayers == 1: net = mlp.mlp(train,traint, nhidden, outtype='softmax') print 'training mlp...' net.mlptrain(train,traint, 0.1, 700*percent_dataset_usage) #train some iterations before calling earlystopping!! net.earlystopping(train,traint,valid,validt,0.1) #train until validation error start increasing net.confmat(test,testt) #autopsy report elif train_func == 'R': ''' Resilient Propagation makes use of pyBrain framework (must be installed). Significantly faster than Bprop!! ''' #Signal handler (Ctrl-C) not implemented yet for Rprop... meaning if stopped its lost # Train the network from pybrain.datasets import ClassificationDataSet from pybrain.utilities import percentError from pybrain.tools.shortcuts import buildNetwork
# FUSE CLASSES INTO ONE DATASET dataset = numpy.concatenate((class1, class2), axis=0) # SHUFFLE THE DATASET order = range(numpy.shape(dataset)[0]) random.shuffle(order) dataset = dataset[order, :] # CReATE TRAINDATA AND TARGETDATA train = dataset[0:nrOfPoints, 0:2] traint = numpy.zeros((len(train), 1)) indices = numpy.where(dataset[0:nrOfPoints, 2] == 1) traint[indices, 0] = 1 # TRAINING!!! ANN = mlp.mlp(train, traint, nhidden, beta, momentum) ANN.mlptrain(train, traint, eta, iterations) ANN.confmat(train, traint) # PLOTTING? xRange = numpy.arange(-4, 4, 0.1) ### yRange = numpy.arange(-4, 4, 0.1) # xgrid, ygrid = numpy.meshgrid(xRange, yRange) # # noOfPoints = xgrid.shape[0] * xgrid.shape[1] # xcoords = xgrid.reshape((noOfPoints, 1)) ### CAN BE DONE WITH 2 FORLOOPS ycoords = ygrid.reshape((noOfPoints, 1)) # samples = numpy.concatenate((xcoords, ycoords), axis=1) # # ones = -numpy.ones(xcoords.shape) # samples = numpy.concatenate((samples, ones), axis=1) ###