def main(name, num_epochs): train_stream = ServerDataStream(('features', 'labels'), produces_examples=False) valid_stream = ServerDataStream(('features', 'labels'), produces_examples=False, port=5558) X = tensor.ftensor4('images') y = tensor.imatrix('targets') prediction_train, prediction_test, params = get_model(X) loss = lasagne.objectives.binary_crossentropy(prediction_train, y) loss = loss.mean() prediction_01 = tensor.ge(prediction_train, numpy.float32(.5)) f2 = f2_score(prediction_01, y) f2_diff = f2_score(prediction_train, y) loss = -f2_diff updates = lasagne.updates.nesterov_momentum(loss, params, learning_rate=1e-3, momentum=0.9) train_fn = function([X, y], loss, updates=updates) valid_fn = function([X, y], f2) best_valid_score = 0 patience = 0 all_train_loss = [] iteration = 0 for epoch in range(num_epochs): f2_valid_loss = [] f2_train_loss = [] for imgs, targets in train_stream.get_epoch_iterator(): f2_train_loss.append(train_fn(imgs, targets)) iteration += 1 all_train_loss.append(f2_train_loss) train_score = -numpy.mean(numpy.asarray(f2_train_loss)) print('Iteration %d' % (iteration, )) print('train score : {0}'.format(train_score)) for imgs, targets in valid_stream.get_epoch_iterator(): f2_valid_loss.append(valid_fn(imgs, targets)) valid_score = numpy.mean(numpy.asarray(f2_valid_loss)) print('valid score : {0}'.format(valid_score)) if best_valid_score < valid_score: best_valid_score = valid_score patience = 0 param_values = [p.get_value() for p in params] numpy.savez_compressed('%s.npz' % (name, ), param_values) pickle.dump(all_train_loss, open('%s.pkl' % (name, ), 'wb')) else: patience += 1 if patience == 5: break print('patience : {0}'.format(patience)) print('\n')
class TestServer(object): def setUp(self): self.server_process = Process( target=start_server, args=(get_stream(),)) self.server_process.start() self.stream = ServerDataStream(('f', 't'), False) def tearDown(self): self.server_process.terminate() self.stream = None def test_server(self): server_data = self.stream.get_epoch_iterator() expected_data = get_stream().get_epoch_iterator() for _, s, e in zip(range(3), server_data, expected_data): for data in zip(s, e): assert_allclose(*data) assert_raises(StopIteration, next, server_data) def test_pickling(self): try: self.stream = cPickle.loads(cPickle.dumps(self.stream)) # regression test: pickling of an unpickled stream used it fail cPickle.dumps(self.stream) server_data = self.stream.get_epoch_iterator() expected_data = get_stream().get_epoch_iterator() for _, s, e in zip(range(3), server_data, expected_data): for data in zip(s, e): assert_allclose(*data, rtol=1e-3) except AssertionError as e: raise SkipTest("Skip test_that failed with: {}".format(e)) assert_raises(StopIteration, next, server_data) def test_value_error_on_request(self): assert_raises(ValueError, self.stream.get_data, [0, 1]) def test_close(self): self.stream.close() def test_next_epoch(self): self.stream.next_epoch() def test_reset(self): self.stream.reset()
class TestServer(object): def setUp(self): self.server_process = Process( target=start_server, args=(get_stream(),)) self.server_process.start() self.stream = ServerDataStream(('f', 't'), False) def tearDown(self): self.server_process.terminate() self.stream = None def test_server(self): server_data = self.stream.get_epoch_iterator() expected_data = get_stream().get_epoch_iterator() for _, s, e in zip(range(3), server_data, expected_data): for data in zip(s, e): assert_allclose(*data) assert_raises(StopIteration, next, server_data) def test_pickling(self): self.stream = cPickle.loads(cPickle.dumps(self.stream)) server_data = self.stream.get_epoch_iterator() expected_data = get_stream().get_epoch_iterator() for _, s, e in zip(range(3), server_data, expected_data): for data in zip(s, e): #pass assert_allclose(*data, rtol=1e-5) assert_raises(StopIteration, next, server_data) def test_value_error_on_request(self): assert_raises(ValueError, self.stream.get_data, [0, 1]) def test_close(self): self.stream.close() def test_next_epoch(self): self.stream.next_epoch() def test_reset(self): self.stream.reset()
class TestServer(object): def setUp(self): self.server_process = Process(target=start_server, args=(get_stream(), )) self.server_process.start() self.stream = ServerDataStream(('f', 't'), False) def tearDown(self): self.server_process.terminate() self.stream = None def test_server(self): server_data = self.stream.get_epoch_iterator() expected_data = get_stream().get_epoch_iterator() for _, s, e in zip(range(3), server_data, expected_data): for data in zip(s, e): assert_allclose(*data) assert_raises(StopIteration, next, server_data) def test_pickling(self): self.stream = cPickle.loads(cPickle.dumps(self.stream)) server_data = self.stream.get_epoch_iterator() expected_data = get_stream().get_epoch_iterator() for _, s, e in zip(range(3), server_data, expected_data): for data in zip(s, e): assert_allclose(*data, rtol=1e-5) assert_raises(StopIteration, next, server_data) def test_value_error_on_request(self): assert_raises(ValueError, self.stream.get_data, [0, 1]) def test_close(self): self.stream.close() def test_next_epoch(self): self.stream.next_epoch() def test_reset(self): self.stream.reset()
param_values = [f['arr_%d' % i] for i in range(len(f.files))] set_all_param_values(net[layers_names[len(layers_names)-1]], param_values[0]) # create predict function prediction_test = lasagne.layers.get_output(net[layers_names[len(layers_names)-1]], deterministic=True) eval_fn = function([X], [prediction_test]) # Classes classes = {0: 'haze', 1: 'primary', 2: 'agriculture', 3: 'clear', 4: 'water', 5: 'habitation', 6: 'road', 7: 'cultivation', 8: 'slash_burn', 9: 'cloudy', 10: 'partly_cloudy', 11: 'conventional_mine', 12: 'bare_ground', 13: 'artisinal_mine', 14: 'blooming', 15: 'selective_logging', 16: 'blow_down'} # submit file import csv csvfile = open('submits/submit_complexvgg.csv', 'wb') spamwriter = csv.writer(csvfile, delimiter=' ', quotechar='|', quoting=csv.QUOTE_MINIMAL) spamwriter.writerow(['image_name,tags']) # prediction import time print('writing submit file') for imgs, imgs_name in submit_stream.get_epoch_iterator(): pred_targets = eval_fn(imgs) for pred_idx in range(len(pred_targets[0])): spamwriter.writerow([imgs_name[pred_idx].split('.')[0] + ','] + [classes[k] for k in numpy.arange(len(classes))[(pred_targets[0][pred_idx] > .5)]]) time.sleep(.5) csvfile.close()
# build model and load weights input_var = tensor.tensor4('X') _, test_prediction, _ = model_resnet(input_var) # create prediction function val_fn = theano.function([input_var], [test_prediction]) # Try for a few data points n_datapoints = 2 from fuel.streams import ServerDataStream import numpy as np train_stream = ServerDataStream(('features', 'labels'), produces_examples=False) labels_count = np.zeros((17,)) mb_count = 0 iterator = train_stream.get_epoch_iterator() data = iterator.next() labels_count += data[1].sum(axis=0) mb_count += 1 feat = np.asarray(data[0][:n_datapoints], dtype=np.float32) pred = val_fn(feat) print('Prediction for the {0} datapoints is : '.format(n_datapoints)) print(pred)
predict = convnet.apply(x) cost = CategoricalCrossEntropy().apply(y.flatten(), predict).copy(name='cost') cg = ComputationGraph(cost) #Load the parameters of the model params = load_parameter_values('catsVsDogs256.pkl') mo = Model(predict) mo.set_parameter_values(params) print mo.inputs print dir(mo.inputs) print mo.outputs f = theano.function(mo.inputs, mo.outputs, allow_input_downcast=True) predictions = [] k = 0 for batch in stream_test.get_epoch_iterator(): example = numpy.array([batch[0]]) batch_predictions = f(example) #batch_predictions=batch_predictions[0] #get the array for result in batch_predictions: res = numpy.argmax(result) predictions.append(res) k += 1 print "example", k, "predicted", '\n' print predictions #save file f = open('predictions', 'wb') pickle.dump(predictions, f) #construct CSV file to submit to Kaggle with open('submission.csv', 'w') as f1:
updates = lasagne.updates.nesterov_momentum(loss, params, learning_rate=1e-4, momentum=0.9) train_fn = function([X, y], loss, updates=updates) valid_fn = function([X, y], f2) best_valid_score = 0 patience = 0 all_train_loss = [] for epoch in range(num_epochs): f2_valid_loss = [] f2_train_loss = [] for imgs, targets in train_stream.get_epoch_iterator(): f2_train_loss.append(train_fn(imgs, targets)) all_train_loss.append(f2_train_loss) train_score = -numpy.mean(numpy.asarray(f2_train_loss)) print('train score : {0}'.format(train_score)) for imgs, targets in valid_stream.get_epoch_iterator(): f2_valid_loss.append(valid_fn(imgs, targets)) valid_score = numpy.mean(numpy.asarray(f2_valid_loss)) print('valid score : {0}'.format(valid_score)) if best_valid_score < valid_score: best_valid_score = valid_score patience = 0 param_values = [p.get_value() for p in params] numpy.savez_compressed('simple_vgg_valid_lr4.npz', param_values) pickle.dump(all_train_loss, open('simple_vgg_valid_lr4.pkl', 'wb')) else:
def predict(port, l2_penalty_scale, save_model_file='./params_file.npz', batchsize=500, load_in_memory=False, be_verbose=False): print("Loading data for prediction...") # extract timestamp from model file - assume it is the first set of numbers # otherwise just use "now" import re import time tstamp = str(time.time()).split('.')[0] m = re.search(r"[0-9]+", save_model_file) if m: tstamp = m.group(0) # Prepare Theano variables for inputs and targets input_var_x = T.tensor4('inputs') input_var_u = T.tensor4('inputs') input_var_v = T.tensor4('inputs') target_var = T.ivector('targets') # Build the model network = build_cnn(input_var_x, input_var_u, input_var_v) with np.load(save_model_file) as f: param_values = [f['arr_%d' % i] for i in range(len(f.files))] lasagne.layers.set_all_param_values(network, param_values) # Create a loss expression for testing. test_prediction = lasagne.layers.get_output(network, deterministic=True) l2_penalty = lasagne.regularization.regularize_layer_params( lasagne.layers.get_all_layers(network), lasagne.regularization.l2) * l2_penalty_scale test_loss = categorical_crossentropy(test_prediction, target_var) + \ l2_penalty test_loss = test_loss.mean() # Also create an expression for the classification accuracy: test_acc = T.mean(T.eq(T.argmax(test_prediction, axis=1), target_var), dtype=theano.config.floatX) # Look at the classifications test_prediction_values = T.argmax(test_prediction, axis=1) # Compile a function computing the validation loss and accuracy: val_fn = theano.function([input_var_x, input_var_u, input_var_v, target_var], [test_loss, test_acc], allow_input_downcast=True) # Compute the actual predictions - also instructive is to look at # `test_prediction` as an output (array of softmax probabilities) # (but that prints a _lot_ of stuff to screen...) pred_fn = theano.function([input_var_x, input_var_u, input_var_v], [test_prediction_values], allow_input_downcast=True) # don't `produces_examples`, produce batches test_dstream = ServerDataStream(('test',), port=port, produces_examples=False) # look at some concrete predictions targ_numbers = [1, 2, 3, 4, 5] pred_target = np.array([0, 0, 0, 0, 0]) true_target = np.array([0, 0, 0, 0, 0]) targs_mat = np.zeros(11 * 11).reshape(11, 11) for data in test_dstream.get_epoch_iterator(): _, inputs, targets = data[0], data[1], data[2] inputx, inputu, inputv = split_inputs_xuv(inputs) pred = pred_fn(inputx, inputu, inputv) pred_targ = zip(pred[0], targets) if be_verbose: print("(prediction, true target):", pred_targ) print("----------------") for p, t in pred_targ: targs_mat[t][p] += 1 if t in targ_numbers: true_target[t-1] += 1 if p == t: pred_target[p-1] += 1 acc_target = 100.0 * pred_target / true_target.astype('float32') perf_file = 'perfmat' + tstamp + '.npy' np.save(perf_file, targs_mat) # compute and print the test error: test_err = 0 test_acc = 0 test_batches = 0 for data in test_dstream.get_epoch_iterator(): _, inputs, targets = data[0], data[1], data[2] inputx, inputu, inputv = split_inputs_xuv(inputs) err, acc = val_fn(inputx, inputu, inputv, targets) test_err += err test_acc += acc test_batches += 1 print("Final results:") print(" test loss:\t\t\t{:.6f}".format(test_err / test_batches)) print(" test accuracy:\t\t{:.2f} %".format( test_acc / test_batches * 100)) for i, v in enumerate(acc_target): print(" target {} accuracy:\t\t\t{:.3f} %".format( (i + 1), acc_target[i]))
def train(port=55557, num_epochs=500, learning_rate=0.01, momentum=0.9, l2_penalty_scale=1e-04, batchsize=500, save_model_file='./params_file.npz', start_with_saved_params=False): print("Loading data...") # Prepare Theano variables for inputs and targets input_var_x = T.tensor4('inputs') input_var_u = T.tensor4('inputs') input_var_v = T.tensor4('inputs') target_var = T.ivector('targets') # Build the model network = build_cnn(input_var_x, input_var_u, input_var_v) print(network_repr.get_network_str( lasagne.layers.get_all_layers(network), get_network=False, incomings=True, outgoings=True)) if start_with_saved_params and os.path.isfile(save_model_file): with np.load(save_model_file) as f: param_values = [f['arr_%d' % i] for i in range(len(f.files))] lasagne.layers.set_all_param_values(network, param_values) # Create a loss expression for training. prediction = lasagne.layers.get_output(network) l2_penalty = lasagne.regularization.regularize_layer_params( lasagne.layers.get_all_layers(network), lasagne.regularization.l2) * l2_penalty_scale loss = categorical_crossentropy(prediction, target_var) + l2_penalty loss = loss.mean() # Create update expressions for training. params = lasagne.layers.get_all_params(network, trainable=True) print( """ //// Use AdaGrad update schedule for learning rate, see Duchi, Hazan, and Singer (2011) "Adaptive subgradient methods for online learning and stochasitic optimization." JMLR, 12:2121-2159 //// """) updates_adagrad = lasagne.updates.adagrad( loss, params, learning_rate=learning_rate, epsilon=1e-06) print( """ //// Apply Nesterov momentum using Lisa Lab's modifications. //// """) updates = lasagne.updates.apply_nesterov_momentum( updates_adagrad, params, momentum=momentum) # Create a loss expression for validation/testing. Note we do a # deterministic forward pass through the network, disabling dropout. test_prediction = lasagne.layers.get_output(network, deterministic=True) test_loss = categorical_crossentropy(test_prediction, target_var) + \ l2_penalty test_loss = test_loss.mean() # Also create an expression for the classification accuracy: test_acc = T.mean(T.eq(T.argmax(test_prediction, axis=1), target_var), dtype=theano.config.floatX) # Compile a function performing a training step on a mini-batch (by giving # the updates dictionary) and returning the corresponding training loss: train_fn = theano.function([input_var_x, input_var_u, input_var_v, target_var], loss, updates=updates, allow_input_downcast=True) # Compile a second function computing the validation loss and accuracy: val_fn = theano.function([input_var_x, input_var_u, input_var_v, target_var], [test_loss, test_acc], allow_input_downcast=True) print("Starting training...") train_dstream = ServerDataStream(('train',), port=port, produces_examples=False) # # TODO: early stopping logic goes here... # for epoch in range(num_epochs): # In each epoch, we do a full pass over the training data: train_err = 0 train_batches = 0 start_time = time.time() for data in train_dstream.get_epoch_iterator(): _, inputs, targets = data[0], data[1], data[2] inputx, inputu, inputv = split_inputs_xuv(inputs) train_err += train_fn(inputx, inputu, inputv, targets) train_batches += 1 # And a full pass over the validation data: # val_err = 0 # val_acc = 0 # val_batches = 0 # for data in valid_dstream.get_epoch_iterator(): # _, inputs, targets = data[0], data[1], data[2] # inputx, inputu, inputv = split_inputs_xuv(inputs) # err, acc = val_fn(inputx, inputu, inputv, targets) # val_err += err # val_acc += acc # val_batches += 1 # Dump the current network weights to file np.savez(save_model_file, *lasagne.layers.get_all_param_values(network)) # Then we print the results for this epoch: print("Epoch {} of {} took {:.3f}s".format( epoch + 1, num_epochs, time.time() - start_time)) print(" training loss:\t\t{:.6f}".format(train_err / train_batches)) # print(" validation loss:\t\t{:.6f}".format(val_err / val_batches)) # print(" validation accuracy:\t\t{:.2f} %".format( # val_acc / val_batches * 100)) print("Finished {} epochs.".format(epoch + 1))
from resnet_152 import get_model as model_resnet # build model and load weights input_var = tensor.tensor4('X') _, test_prediction, _ = model_resnet(input_var) # create prediction function val_fn = theano.function([input_var], [test_prediction]) # Try for a few data points n_datapoints = 2 from fuel.streams import ServerDataStream import numpy as np train_stream = ServerDataStream(('features', 'labels'), produces_examples=False) labels_count = np.zeros((17, )) mb_count = 0 for data in train_stream.get_epoch_iterator(): labels_count += data[1].sum(axis=0) mb_count += 1 feat = np.asarray(data[0][:n_datapoints], dtype=np.float32) pred = val_fn(feat) print('Prediction for the {0} datapoints is : '.format(n_datapoints)) print(pred)
def timer(name): """Times a block of code and prints the result. Parameters ---------- name : str What this block of code represents. """ start_time = time.time() yield stop_time = time.time() print('{} took {} seconds'.format(name, stop_time - start_time)) if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument( '-p', '--parallel', action='store_true', help='run data preprocessing in a separate process') args = parser.parse_args() if args.parallel: data_stream = ServerDataStream(('features', ), True) else: data_stream = create_data_stream(0.005) with timer('Training'): for i in range(5): for data in data_stream.get_epoch_iterator(): time.sleep(0.01)