def main(name, num_epochs):
train_stream = ServerDataStream(('features', 'labels'),
produces_examples=False)
valid_stream = ServerDataStream(('features', 'labels'),
produces_examples=False,
port=5558)
X = tensor.ftensor4('images')
y = tensor.imatrix('targets')
prediction_train, prediction_test, params = get_model(X)
loss = lasagne.objectives.binary_crossentropy(prediction_train, y)
loss = loss.mean()
prediction_01 = tensor.ge(prediction_train, numpy.float32(.5))
f2 = f2_score(prediction_01, y)
f2_diff = f2_score(prediction_train, y)
loss = -f2_diff
updates = lasagne.updates.nesterov_momentum(loss,
params,
learning_rate=1e-3,
momentum=0.9)
train_fn = function([X, y], loss, updates=updates)
valid_fn = function([X, y], f2)
best_valid_score = 0
patience = 0
all_train_loss = []
iteration = 0
for epoch in range(num_epochs):
f2_valid_loss = []
f2_train_loss = []
for imgs, targets in train_stream.get_epoch_iterator():
f2_train_loss.append(train_fn(imgs, targets))
iteration += 1
all_train_loss.append(f2_train_loss)
train_score = -numpy.mean(numpy.asarray(f2_train_loss))
print('Iteration %d' % (iteration, ))
print('train score : {0}'.format(train_score))
for imgs, targets in valid_stream.get_epoch_iterator():
f2_valid_loss.append(valid_fn(imgs, targets))
valid_score = numpy.mean(numpy.asarray(f2_valid_loss))
print('valid score : {0}'.format(valid_score))
if best_valid_score < valid_score:
best_valid_score = valid_score
patience = 0
param_values = [p.get_value() for p in params]
numpy.savez_compressed('%s.npz' % (name, ), param_values)
pickle.dump(all_train_loss, open('%s.pkl' % (name, ), 'wb'))
else:
patience += 1
if patience == 5:
break
print('patience : {0}'.format(patience))
print('\n')
class TestServer(object):
def setUp(self):
self.server_process = Process(
target=start_server, args=(get_stream(),))
self.server_process.start()
self.stream = ServerDataStream(('f', 't'), False)
def tearDown(self):
self.server_process.terminate()
self.stream = None
def test_server(self):
server_data = self.stream.get_epoch_iterator()
expected_data = get_stream().get_epoch_iterator()
for _, s, e in zip(range(3), server_data, expected_data):
for data in zip(s, e):
assert_allclose(*data)
assert_raises(StopIteration, next, server_data)
def test_pickling(self):
try:
self.stream = cPickle.loads(cPickle.dumps(self.stream))
# regression test: pickling of an unpickled stream used it fail
cPickle.dumps(self.stream)
server_data = self.stream.get_epoch_iterator()
expected_data = get_stream().get_epoch_iterator()
for _, s, e in zip(range(3), server_data, expected_data):
for data in zip(s, e):
assert_allclose(*data, rtol=1e-3)
except AssertionError as e:
raise SkipTest("Skip test_that failed with: {}".format(e))
assert_raises(StopIteration, next, server_data)
def test_value_error_on_request(self):
assert_raises(ValueError, self.stream.get_data, [0, 1])
def test_close(self):
self.stream.close()
def test_next_epoch(self):
self.stream.next_epoch()
def test_reset(self):
self.stream.reset()
class TestServer(object):
def setUp(self):
self.server_process = Process(
target=start_server, args=(get_stream(),))
self.server_process.start()
self.stream = ServerDataStream(('f', 't'), False)
def tearDown(self):
self.server_process.terminate()
self.stream = None
def test_server(self):
server_data = self.stream.get_epoch_iterator()
expected_data = get_stream().get_epoch_iterator()
for _, s, e in zip(range(3), server_data, expected_data):
for data in zip(s, e):
assert_allclose(*data)
assert_raises(StopIteration, next, server_data)
def test_pickling(self):
self.stream = cPickle.loads(cPickle.dumps(self.stream))
server_data = self.stream.get_epoch_iterator()
expected_data = get_stream().get_epoch_iterator()
for _, s, e in zip(range(3), server_data, expected_data):
for data in zip(s, e):
#pass
assert_allclose(*data, rtol=1e-5)
assert_raises(StopIteration, next, server_data)
def test_value_error_on_request(self):
assert_raises(ValueError, self.stream.get_data, [0, 1])
def test_close(self):
self.stream.close()
def test_next_epoch(self):
self.stream.next_epoch()
def test_reset(self):
self.stream.reset()
class TestServer(object):
def setUp(self):
self.server_process = Process(target=start_server,
args=(get_stream(), ))
self.server_process.start()
self.stream = ServerDataStream(('f', 't'), False)
def tearDown(self):
self.server_process.terminate()
self.stream = None
def test_server(self):
server_data = self.stream.get_epoch_iterator()
expected_data = get_stream().get_epoch_iterator()
for _, s, e in zip(range(3), server_data, expected_data):
for data in zip(s, e):
assert_allclose(*data)
assert_raises(StopIteration, next, server_data)
def test_pickling(self):
self.stream = cPickle.loads(cPickle.dumps(self.stream))
server_data = self.stream.get_epoch_iterator()
expected_data = get_stream().get_epoch_iterator()
for _, s, e in zip(range(3), server_data, expected_data):
for data in zip(s, e):
assert_allclose(*data, rtol=1e-5)
assert_raises(StopIteration, next, server_data)
def test_value_error_on_request(self):
assert_raises(ValueError, self.stream.get_data, [0, 1])
def test_close(self):
self.stream.close()
def test_next_epoch(self):
self.stream.next_epoch()
def test_reset(self):
self.stream.reset()
param_values = [f['arr_%d' % i] for i in range(len(f.files))]
set_all_param_values(net[layers_names[len(layers_names)-1]], param_values[0])
# create predict function
prediction_test = lasagne.layers.get_output(net[layers_names[len(layers_names)-1]], deterministic=True)
eval_fn = function([X], [prediction_test])
# Classes
classes = {0: 'haze', 1: 'primary', 2: 'agriculture', 3: 'clear',
4: 'water', 5: 'habitation', 6: 'road', 7: 'cultivation', 8: 'slash_burn', 9: 'cloudy',
10: 'partly_cloudy', 11: 'conventional_mine', 12: 'bare_ground', 13: 'artisinal_mine', 14: 'blooming', 15: 'selective_logging', 16: 'blow_down'}
# submit file
import csv
csvfile = open('submits/submit_complexvgg.csv', 'wb')
spamwriter = csv.writer(csvfile, delimiter=' ', quotechar='|', quoting=csv.QUOTE_MINIMAL)
spamwriter.writerow(['image_name,tags'])
# prediction
import time
print('writing submit file')
for imgs, imgs_name in submit_stream.get_epoch_iterator():
pred_targets = eval_fn(imgs)
for pred_idx in range(len(pred_targets[0])):
spamwriter.writerow([imgs_name[pred_idx].split('.')[0] + ','] + [classes[k] for k in numpy.arange(len(classes))[(pred_targets[0][pred_idx] > .5)]])
time.sleep(.5)
csvfile.close()
# build model and load weights
input_var = tensor.tensor4('X')
_, test_prediction, _ = model_resnet(input_var)
# create prediction function
val_fn = theano.function([input_var], [test_prediction])
# Try for a few data points
n_datapoints = 2
from fuel.streams import ServerDataStream
import numpy as np
train_stream = ServerDataStream(('features', 'labels'),
produces_examples=False)
labels_count = np.zeros((17,))
mb_count = 0
iterator = train_stream.get_epoch_iterator()
data = iterator.next()
labels_count += data[1].sum(axis=0)
mb_count += 1
feat = np.asarray(data[0][:n_datapoints], dtype=np.float32)
pred = val_fn(feat)
print('Prediction for the {0} datapoints is : '.format(n_datapoints))
print(pred)
predict = convnet.apply(x)
cost = CategoricalCrossEntropy().apply(y.flatten(), predict).copy(name='cost')
cg = ComputationGraph(cost)
#Load the parameters of the model
params = load_parameter_values('catsVsDogs256.pkl')
mo = Model(predict)
mo.set_parameter_values(params)
print mo.inputs
print dir(mo.inputs)
print mo.outputs
f = theano.function(mo.inputs, mo.outputs, allow_input_downcast=True)
predictions = []
k = 0
for batch in stream_test.get_epoch_iterator():
example = numpy.array([batch[0]])
batch_predictions = f(example)
#batch_predictions=batch_predictions[0] #get the array
for result in batch_predictions:
res = numpy.argmax(result)
predictions.append(res)
k += 1
print "example", k, "predicted", '\n'
print predictions
#save file
f = open('predictions', 'wb')
pickle.dump(predictions, f)
#construct CSV file to submit to Kaggle
with open('submission.csv', 'w') as f1:
updates = lasagne.updates.nesterov_momentum(loss,
params,
learning_rate=1e-4,
momentum=0.9)
train_fn = function([X, y], loss, updates=updates)
valid_fn = function([X, y], f2)
best_valid_score = 0
patience = 0
all_train_loss = []
for epoch in range(num_epochs):
f2_valid_loss = []
f2_train_loss = []
for imgs, targets in train_stream.get_epoch_iterator():
f2_train_loss.append(train_fn(imgs, targets))
all_train_loss.append(f2_train_loss)
train_score = -numpy.mean(numpy.asarray(f2_train_loss))
print('train score : {0}'.format(train_score))
for imgs, targets in valid_stream.get_epoch_iterator():
f2_valid_loss.append(valid_fn(imgs, targets))
valid_score = numpy.mean(numpy.asarray(f2_valid_loss))
print('valid score : {0}'.format(valid_score))
if best_valid_score < valid_score:
best_valid_score = valid_score
patience = 0
param_values = [p.get_value() for p in params]
numpy.savez_compressed('simple_vgg_valid_lr4.npz', param_values)
pickle.dump(all_train_loss, open('simple_vgg_valid_lr4.pkl', 'wb'))
else:
def predict(port, l2_penalty_scale, save_model_file='./params_file.npz',
batchsize=500, load_in_memory=False, be_verbose=False):
print("Loading data for prediction...")
# extract timestamp from model file - assume it is the first set of numbers
# otherwise just use "now"
import re
import time
tstamp = str(time.time()).split('.')[0]
m = re.search(r"[0-9]+", save_model_file)
if m:
tstamp = m.group(0)
# Prepare Theano variables for inputs and targets
input_var_x = T.tensor4('inputs')
input_var_u = T.tensor4('inputs')
input_var_v = T.tensor4('inputs')
target_var = T.ivector('targets')
# Build the model
network = build_cnn(input_var_x, input_var_u, input_var_v)
with np.load(save_model_file) as f:
param_values = [f['arr_%d' % i] for i in range(len(f.files))]
lasagne.layers.set_all_param_values(network, param_values)
# Create a loss expression for testing.
test_prediction = lasagne.layers.get_output(network, deterministic=True)
l2_penalty = lasagne.regularization.regularize_layer_params(
lasagne.layers.get_all_layers(network),
lasagne.regularization.l2) * l2_penalty_scale
test_loss = categorical_crossentropy(test_prediction, target_var) + \
l2_penalty
test_loss = test_loss.mean()
# Also create an expression for the classification accuracy:
test_acc = T.mean(T.eq(T.argmax(test_prediction, axis=1), target_var),
dtype=theano.config.floatX)
# Look at the classifications
test_prediction_values = T.argmax(test_prediction, axis=1)
# Compile a function computing the validation loss and accuracy:
val_fn = theano.function([input_var_x, input_var_u, input_var_v,
target_var],
[test_loss, test_acc],
allow_input_downcast=True)
# Compute the actual predictions - also instructive is to look at
# `test_prediction` as an output (array of softmax probabilities)
# (but that prints a _lot_ of stuff to screen...)
pred_fn = theano.function([input_var_x, input_var_u, input_var_v],
[test_prediction_values],
allow_input_downcast=True)
# don't `produces_examples`, produce batches
test_dstream = ServerDataStream(('test',),
port=port,
produces_examples=False)
# look at some concrete predictions
targ_numbers = [1, 2, 3, 4, 5]
pred_target = np.array([0, 0, 0, 0, 0])
true_target = np.array([0, 0, 0, 0, 0])
targs_mat = np.zeros(11 * 11).reshape(11, 11)
for data in test_dstream.get_epoch_iterator():
_, inputs, targets = data[0], data[1], data[2]
inputx, inputu, inputv = split_inputs_xuv(inputs)
pred = pred_fn(inputx, inputu, inputv)
pred_targ = zip(pred[0], targets)
if be_verbose:
print("(prediction, true target):", pred_targ)
print("----------------")
for p, t in pred_targ:
targs_mat[t][p] += 1
if t in targ_numbers:
true_target[t-1] += 1
if p == t:
pred_target[p-1] += 1
acc_target = 100.0 * pred_target / true_target.astype('float32')
perf_file = 'perfmat' + tstamp + '.npy'
np.save(perf_file, targs_mat)
# compute and print the test error:
test_err = 0
test_acc = 0
test_batches = 0
for data in test_dstream.get_epoch_iterator():
_, inputs, targets = data[0], data[1], data[2]
inputx, inputu, inputv = split_inputs_xuv(inputs)
err, acc = val_fn(inputx, inputu, inputv, targets)
test_err += err
test_acc += acc
test_batches += 1
print("Final results:")
print(" test loss:\t\t\t{:.6f}".format(test_err / test_batches))
print(" test accuracy:\t\t{:.2f} %".format(
test_acc / test_batches * 100))
for i, v in enumerate(acc_target):
print(" target {} accuracy:\t\t\t{:.3f} %".format(
(i + 1), acc_target[i]))
def train(port=55557, num_epochs=500, learning_rate=0.01, momentum=0.9,
l2_penalty_scale=1e-04, batchsize=500,
save_model_file='./params_file.npz', start_with_saved_params=False):
print("Loading data...")
# Prepare Theano variables for inputs and targets
input_var_x = T.tensor4('inputs')
input_var_u = T.tensor4('inputs')
input_var_v = T.tensor4('inputs')
target_var = T.ivector('targets')
# Build the model
network = build_cnn(input_var_x, input_var_u, input_var_v)
print(network_repr.get_network_str(
lasagne.layers.get_all_layers(network),
get_network=False, incomings=True, outgoings=True))
if start_with_saved_params and os.path.isfile(save_model_file):
with np.load(save_model_file) as f:
param_values = [f['arr_%d' % i] for i in range(len(f.files))]
lasagne.layers.set_all_param_values(network, param_values)
# Create a loss expression for training.
prediction = lasagne.layers.get_output(network)
l2_penalty = lasagne.regularization.regularize_layer_params(
lasagne.layers.get_all_layers(network),
lasagne.regularization.l2) * l2_penalty_scale
loss = categorical_crossentropy(prediction, target_var) + l2_penalty
loss = loss.mean()
# Create update expressions for training.
params = lasagne.layers.get_all_params(network, trainable=True)
print(
"""
////
Use AdaGrad update schedule for learning rate, see Duchi, Hazan, and
Singer (2011) "Adaptive subgradient methods for online learning and
stochasitic optimization." JMLR, 12:2121-2159
////
""")
updates_adagrad = lasagne.updates.adagrad(
loss, params, learning_rate=learning_rate, epsilon=1e-06)
print(
"""
////
Apply Nesterov momentum using Lisa Lab's modifications.
////
""")
updates = lasagne.updates.apply_nesterov_momentum(
updates_adagrad, params, momentum=momentum)
# Create a loss expression for validation/testing. Note we do a
# deterministic forward pass through the network, disabling dropout.
test_prediction = lasagne.layers.get_output(network, deterministic=True)
test_loss = categorical_crossentropy(test_prediction, target_var) + \
l2_penalty
test_loss = test_loss.mean()
# Also create an expression for the classification accuracy:
test_acc = T.mean(T.eq(T.argmax(test_prediction, axis=1), target_var),
dtype=theano.config.floatX)
# Compile a function performing a training step on a mini-batch (by giving
# the updates dictionary) and returning the corresponding training loss:
train_fn = theano.function([input_var_x, input_var_u, input_var_v,
target_var],
loss, updates=updates,
allow_input_downcast=True)
# Compile a second function computing the validation loss and accuracy:
val_fn = theano.function([input_var_x, input_var_u, input_var_v,
target_var],
[test_loss, test_acc],
allow_input_downcast=True)
print("Starting training...")
train_dstream = ServerDataStream(('train',),
port=port,
produces_examples=False)
#
# TODO: early stopping logic goes here...
#
for epoch in range(num_epochs):
# In each epoch, we do a full pass over the training data:
train_err = 0
train_batches = 0
start_time = time.time()
for data in train_dstream.get_epoch_iterator():
_, inputs, targets = data[0], data[1], data[2]
inputx, inputu, inputv = split_inputs_xuv(inputs)
train_err += train_fn(inputx, inputu, inputv, targets)
train_batches += 1
# And a full pass over the validation data:
# val_err = 0
# val_acc = 0
# val_batches = 0
# for data in valid_dstream.get_epoch_iterator():
# _, inputs, targets = data[0], data[1], data[2]
# inputx, inputu, inputv = split_inputs_xuv(inputs)
# err, acc = val_fn(inputx, inputu, inputv, targets)
# val_err += err
# val_acc += acc
# val_batches += 1
# Dump the current network weights to file
np.savez(save_model_file,
*lasagne.layers.get_all_param_values(network))
# Then we print the results for this epoch:
print("Epoch {} of {} took {:.3f}s".format(
epoch + 1, num_epochs, time.time() - start_time))
print(" training loss:\t\t{:.6f}".format(train_err / train_batches))
# print(" validation loss:\t\t{:.6f}".format(val_err / val_batches))
# print(" validation accuracy:\t\t{:.2f} %".format(
# val_acc / val_batches * 100))
print("Finished {} epochs.".format(epoch + 1))
from resnet_152 import get_model as model_resnet
# build model and load weights
input_var = tensor.tensor4('X')
_, test_prediction, _ = model_resnet(input_var)
# create prediction function
val_fn = theano.function([input_var], [test_prediction])
# Try for a few data points
n_datapoints = 2
from fuel.streams import ServerDataStream
import numpy as np
train_stream = ServerDataStream(('features', 'labels'),
produces_examples=False)
labels_count = np.zeros((17, ))
mb_count = 0
for data in train_stream.get_epoch_iterator():
labels_count += data[1].sum(axis=0)
mb_count += 1
feat = np.asarray(data[0][:n_datapoints], dtype=np.float32)
pred = val_fn(feat)
print('Prediction for the {0} datapoints is : '.format(n_datapoints))
print(pred)
def timer(name):
"""Times a block of code and prints the result.
Parameters
----------
name : str
What this block of code represents.
"""
start_time = time.time()
yield
stop_time = time.time()
print('{} took {} seconds'.format(name, stop_time - start_time))
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument(
'-p', '--parallel', action='store_true',
help='run data preprocessing in a separate process')
args = parser.parse_args()
if args.parallel:
data_stream = ServerDataStream(('features', ), True)
else:
data_stream = create_data_stream(0.005)
with timer('Training'):
for i in range(5):
for data in data_stream.get_epoch_iterator(): time.sleep(0.01)
