import numpy as np from theano import function as fn from theano import In from theano import shared from theano.tensor.shared_randomstreams import RandomStreams import theano.tensor as T import matplotlib.pyplot as plt srng = RandomStreams(seed=504) # create random 2x2 matrices for #uniform and normal distributions respectively rv_u = srng.uniform((2, 2)) rv_n = srng.normal((2, 2)) f = fn([], rv_u) g = fn([], rv_n, no_default_updates=True) nearly_zeros = fn([], rv_u + rv_u - 2 * rv_u) print("\n\nnew values every time") for i in xrange(2): print(f()) print("\n\nsame numbers as thers no updates") for i in xrange(2): print(g()) print("\n\nrandom seed swap test") print(f()) a = rv_u.rng.get_value(borrow=True) a.seed(504) rv_u.rng.set_value(a, borrow=True) print(f())
import numpy as np from theano import function as fn from theano import In from theano import shared import theano.tensor as T import matplotlib.pyplot as plt state = shared(0) inc = T.iscalar('inc') acc = fn( [inc], state , updates = [(state, state+inc)] , on_unused_input = 'ignore') #overwrite the state variable with new state and copy new_state = shared(-1) new_acc = acc.copy( swap = {state:new_state} ) new_acc(100) print(new_state.get_value()) print(state.get_value()) #we could delete updates mechanism like this null_acc = acc.copy(delete_updates = True) null_acc(123) print(state.get_value())
import numpy as np from theano import function as fn import theano.tensor as T import matplotlib.pyplot as plt #create a vector and call it x x = T.vector('x') #build a return variable s = (1.0 * 1) / (1.0 + np.exp(-x)) #create a function based on association f = fn([x], s) #x line for the function l_start = -20.0 l_end = 20 l_n = 1000 d = np.linspace(l_start, l_end, l_n) #plot plt.plot(d, f(d)) plt.grid() plt.show() # we can show that # # 1 1+ tanh(x/2) # ----------- = ----------- # 1+ e(-x) 2 # s2 = (1.0 + np.tanh(x / 2.0)) / 2
from theano.tensor.shared_randomstreams import RandomStreams import theano.tensor as T import matplotlib.pyplot as plt # create two matrix objects in theano a, b = T.dmatrices('a', 'b') #create a function that perfroms multiple operations # 1 add op_add = a + b # 2 subtract op_sub = a - b # 3 picewise multiply op_mul = a * b fn_maths = fn([a, b], [op_add, op_sub, op_mul]) #test print(fn_maths([[1, 2], [3, 4]], [[1, 1], [2, 2]])) # theano uses scan for looping # defining the tensor variables # there is an example on the website to calculate # tanh( x(t).dot(W) + b ) X = T.matrix("X") W = T.matrix("W") b_sym = T.vector("b_sym") # lambda call means input v and return v*W + b_sym results, updates = scan(lambda v: T.tanh(T.dot(v, W) + b_sym), sequences=X)
def train_nnet(rng, data_set, n_examples, batch_size=5, learning_rate=0.1, init=False, params=None): print '#'*50 print 'Starting global fine tuning ...' x = T.tensor4('x') index = T.lscalar() model_ = model(rng, x, (img_h, img_w), batch_size=batch_size, init=init, params=params) cost = T.mean(T.sqr(model_.layer12.output-x)) params = model_.layer1.params + model_.layer2.params + model_.layer3.params + model_.layer4.params + model_.layer5.params + model_.layer6.params + model_.layer7.params + model_.layer8.params + model_.layer9.params + model_.layer10.params + model_.layer11.params + model_.layer12.params grads = T.grad(cost, params) updates = [ (param_i, param_i - learning_rate*grad_i) for param_i, grad_i in zip(params, grads) ] train_fn = fn([index], cost, updates=updates, givens ={ x:data_set[0][index*batch_size: (index+1)*batch_size] }) valid_fn = fn([index], cost, givens={ x:data_set[1][index*batch_size: (index+1)*batch_size] }) test_fn = fn([index], cost, givens={ x:data_set[2][index*batch_size: (index+1)*batch_size] }) n_train_batches = n_examples[0]/batch_size n_valid_batches = n_examples[1]/batch_size n_test_batches = n_examples[2]/batch_size epoch = 0 n_epochs = 200 done_looping = False patience = 10000 patience_increase = 2 improve_threshold = 0.995 validation_freq = min(n_train_batches, patience/2) best_validation_err = np.inf best_iter = 0 test_err = 0 start_time =timeit.default_timer() while((epoch < n_epochs) and (not done_looping)): epoch+=1 for mini_batch_index in xrange(n_train_batches): iter = (epoch-1)*n_train_batches+mini_batch_index if iter%100 ==0: print 'training @ iter = ', iter cost_ij = train_fn(mini_batch_index) if (iter+1)%validation_freq==0: validation_losses=[valid_fn(i) for i in xrange(n_valid_batches)] this_validation_loss = np.mean(validation_losses) print ('epoch %i, minibatch %i/%i, mean validation reconstruction error: %f ' %(epoch, mini_batch_index+1, n_train_batches, this_validation_loss)) if this_validation_loss<best_validation_err: if this_validation_loss<best_validation_err*improve_threshold: patience=max(patience, iter*patience_increase) best_validation_err=this_validation_loss best_iter=iter test_losses=[ test_fn(i) for i in xrange(n_test_batches) ] test_err=np.mean(test_losses) print '\tepoch %i, minibatch %i/%i, mean test reconstruction error: %f' %(epoch, mini_batch_index+1, n_train_batches, test_err) if patience<=iter: done_looping=True break # remove the following block made for visualisation # inp = T.tensor4() # train_ = model(rng, inp, (img_h, img_w), batch_size=batch_size, params=params) # f = fn([inp], train_.layer12.output) # f_img = f(data_set[0].eval()) # plt.gray() # for i in range(1, 2): # plt.subplot(1, 1, i); plt.axis('off'); plt.imshow(f_img[0, i-1, :, :]) # plt.show() # block ends here save_file = open('trained_params.pkl', 'wb') for i in range(len(params)): cPickle.dump(params[i].get_value(borrow=True), save_file, -1) save_file.close() print 'saved trained params @', save_file
import numpy as np from theano import function as fn from theano import In from theano import shared import theano.tensor as T import matplotlib.pyplot as plt st = shared(0) inc = T.iscalar('inc') acc = fn( [inc], st, updates = [ (st, st+inc)] ) print(st.get_value()) acc(400) print(st.get_value()) st.set_value(10) print(st.get_value()) dcc = fn( [inc], st, updates = [ (st, st-inc)] ) res = st * 2 + inc #something that matches shared var type v = T.scalar( dtype = st.dtype) # so v is of state (st) type # givens gets you state := v but dont modify the value of # st skip = fn( [inc,v], res, givens = [(st,v)])
from theano.sandbox.rng_mrg import MRG_RandomStreams from theano.tensor.shared_randomstreams import RandomStreams #class to be used in initialization of an object class Graph(): def __init__(self, seed=504): self.rng = RandomStreams(seed) self.y = self.rng.uniform(size=(1,)) def copy_random_state(g1, g2): if isinstance(g1.rng, MRG_RandomStreams): g2.rng.rstate = g1.rng.rstate for (su1, su2) in zip(g1.rng.state_updates, g2.rng.state_updates): su2[0].set_value(su1[0].get_value()) g1 = Graph(seed=123) f1 = fn([], g1.y) f2 = fn([], g1.y) print("\n\nFunctions taking on the same initialization graph are ouf of sync") print([ zip(f1(),f2()) for i in xrange(3)]) print("\n\nSame with two other graphs") g2 = Graph() f2 = fn([], g2.y) print("\n\nHowever, we could share the random state") copy_random_state(g1, g2) print([ zip(f1(),f2()) for i in xrange(3)])
#remove from auto_encoder import hidden_layer load_file = open('params/trained_params.pkl', 'r') params = () for i in range(12): # update the value, depends on which layer params are being saved W = theano.shared(cPickle.load(load_file), borrow=True) b = theano.shared(cPickle.load(load_file), borrow=True) params+=([W, b],) #if (i==5): # change for params introducing locally connected autoencoders # params+=([W,b], [W,b]) load_file.close() x = T.tensor4('x') vis_model = model(np.random.RandomState(23455), x, (96, 336), 1, True, params) visualize = fn([x], [vis_model.layer1.output, vis_model.layer2.output, vis_model.layer3.output, vis_model.layer4.output, vis_model.layer5.output, vis_model.layer6.output, vis_model.layer7.output, vis_model.layer8.output, vis_model.layer9.output, vis_model.layer10.output, vis_model.layer11.output, vis_model.layer12.output ]) image = '../data/scaled-0.25/I1_000001.png' image = Image.open(image).convert('L') image = np.array(image, dtype='float32') / 256. image = image.reshape(1, 1, 96, 336) outputs = visualize(image) '''print outputs.shape e_inp = T.reshape(outputs, (1, 5, 504)) layer6 = hidden_layer( np.random.RandomState(23455), input = e_inp, n_feature_maps=5, n_in=504, n_out=40, b_size=1,
import numpy as np from theano import function as fn from theano import In import theano.tensor as T import matplotlib.pyplot as plt a,b = T.dmatrices('a','b') diff = a-b abs_diff = abs(diff) diff_sr = diff ** 2 f = fn( [a,b], [diff, abs_diff, diff_sr]) n = [ [11,22], [22,11] ] m = [ [1,2], [3,4] ] print( f(m,n) ) x,y,z = f(n,m) # we coul try out a default value. f2 = fn( [a, In(b, value = [ [0,0], [0,0] ]) ], diff_sr)