def test_orthogonal_multi(): import numpy as np from lasagne.init import Orthogonal sample = Orthogonal().sample((100, 50, 80)) sample = sample.reshape(100, 50*80) assert np.allclose(np.dot(sample, sample.T), np.eye(100), atol=1e-6)
def test_orthogonal_multi(): import numpy as np from lasagne.init import Orthogonal sample = Orthogonal().sample((100, 50, 80)) sample = sample.reshape(100, 50 * 80) assert np.allclose(np.dot(sample, sample.T), np.eye(100), atol=1e-6)
def test_orthogonal(): import numpy as np from lasagne.init import Orthogonal sample = Orthogonal().sample((100, 200)) assert np.allclose(np.dot(sample, sample.T), np.eye(100), atol=1e-6) sample = Orthogonal().sample((200, 100)) assert np.allclose(np.dot(sample.T, sample), np.eye(100), atol=1e-6)
def highway_dense(incoming, Wh=Orthogonal(), bh=Constant(0.0), Wt=Orthogonal(), bt=Constant(-4.0), nonlinearity=rectify, **kwargs): num_inputs = int(np.prod(incoming.output_shape[1:])) l_h = DenseLayer(incoming, num_units=num_inputs, W=Wh, b=bh, nonlinearity=nonlinearity) l_t = DenseLayer(incoming, num_units=num_inputs, W=Wt, b=bt, nonlinearity=sigmoid) return MultiplicativeGatingLayer(gate=l_t, input1=l_h, input2=incoming)
def reset(): if any(np.isnan(scale.get_value()) for scale in scales): for scale in scales: scale.set_value(1.) for l in l_hiddens: l.b.set_value(Constant()(l.b.get_value().shape)) l.W.set_value(Orthogonal()(l.W.get_value().shape)) l_out.b.set_value(Constant()(l_out.b.get_value().shape)) l_out.W.set_value(Orthogonal()(l_out.W.get_value().shape)) for p in (p for u in (updates_ada, updates_other, updates_scal) for p in u if p not in get_all_params(l_out)): p.set_value(Constant()(p.get_value().shape))
def test_orthogonal_gain(): import numpy as np from lasagne.init import Orthogonal gain = 2 sample = Orthogonal(gain).sample((100, 200)) assert np.allclose(np.dot(sample, sample.T), gain * gain * np.eye(100), atol=1e-6) gain = np.sqrt(2) sample = Orthogonal('relu').sample((100, 200)) assert np.allclose(np.dot(sample, sample.T), gain * gain * np.eye(100), atol=1e-6)
def define_lat_pars_dict(d): """ Defines the parameters and neural networks pertaining the Latent Evolution Model Args: d: The object-dictionary Output: LatParsDict: A dictionary containing all these hyperparameters and NNs, to be fed to the Latent Evolution Model """ xDim = d.xDim nnodesEvlv = d.nnodesMevlv NNEvolve = InputLayer((None, xDim), name='Ev_IL') NNEvolve = DenseLayer(NNEvolve, nnodesEvlv, nonlinearity=softmax, W=Orthogonal(), num_leading_axes=1, name='Ev_HL1') NNEvolve = DenseLayer(NNEvolve, xDim**2, nonlinearity=linear, W=Uniform(0.9), num_leading_axes=1, name='Ev_OL') cmn_dct = dict([('NNEvolve', NNEvolve), ('alpha', options.alpha) ]) LatParsDict = npdict_to_theanodict(cmn_dct) return LatParsDict
def define_rec_pars_dict(d, MuX_layers_In=None, LX_layers_In=None): """ Defines the parameters and neural networks pertaining the Recognition Model. Args: d: The object-dictionary Output: RecParsDict: A dictionary containing all these hyperparameters and NNs, to be fed to the Recognition Model """ # outWrecscale = d.outWrecscale mux_dpth = d.mux_depth nnodesMrec = d.nnodesMrec act_func_dict = {'softplus' : softplus, 'tanh' : tanh} nl = act_func_dict[d.act_func_hls] NNMuX = InputLayer((None, None, d.yDim)) for i in range(mux_dpth): NNMuX = DenseLayer(NNMuX, nnodesMrec, nonlinearity=nl, W=Normal(0.5), num_leading_axes=2, name='MuX_HL' + str(i)) NNMuX = DenseLayer(NNMuX, d.xDim, nonlinearity=linear, W=Orthogonal(), num_leading_axes=2, name='MuX_OL') # if d.loadfit: # assert MuX_layers_In is not None, "List of layers not provided." # MuX_layers = lasagne.layers.get_all_layers(NNMuX)[1:] # init_layers(MuX_layers_In, MuX_layers) LambdaX_dpth = d.lbdaxdepth NNLambdaX = lasagne.layers.InputLayer((None, None, d.yDim)) for i in range(LambdaX_dpth): NNLambdaX = DenseLayer(NNLambdaX, nnodesMrec, nonlinearity=nl, W=Normal(0.5), num_leading_axes=2, name='LX_HL' + str(i)) NNLambdaX = DenseLayer(NNLambdaX, d.xDim**2, nonlinearity=linear, W=Orthogonal(), num_leading_axes=2, name='LX_OL') # if options.loadfit: # assert LX_layers_In is not None, "List of layers not provided." # LX_layers = lasagne.layers.get_all_layers(NNLambdaX)[1:] # init_layers(LX_layers_In, LX_layers) RecParsDict = {'LATCLASS' : LocallyLinearEvolution, 'NNMuX' : NNMuX, 'NNLambdaX' : NNLambdaX} return RecParsDict
def build_segmenter_simple(): inp = ll.InputLayer(shape=(None, 1, None, None), name='input') conv1 = ll.Conv2DLayer(inp, num_filters=32, filter_size=(7, 7), pad='same', W=Orthogonal(), nonlinearity=rectify, name='conv1') conv2 = ll.Conv2DLayer(conv1, num_filters=64, filter_size=(5, 5), pad='same', W=Orthogonal(), nonlinearity=rectify, name='conv2') conv3 = ll.Conv2DLayer(conv2, num_filters=128, filter_size=(5, 5), pad='same', W=Orthogonal(), nonlinearity=rectify, name='conv3') conv4 = ll.Conv2DLayer(conv3, num_filters=64, filter_size=(5, 5), pad='same', W=Orthogonal(), nonlinearity=rectify, name='conv4') conv5 = ll.Conv2DLayer(conv4, num_filters=32, filter_size=(3, 3), pad='same', W=Orthogonal(), nonlinearity=rectify, name='conv5') conv6 = ll.Conv2DLayer(conv5, num_filters=16, filter_size=(3, 3), pad='same', W=Orthogonal(), nonlinearity=rectify, name='conv6') # our output layer is also convolutional, remember that our Y is going to be the same exact size as the conv_final = ll.Conv2DLayer(conv6, num_filters=2, filter_size=(3, 3), pad='same', W=Orthogonal(), name='conv_final', nonlinearity=linear) # we need to reshape it to be a (batch*n*m x 3), i.e. unroll s.t. the feature dimension is preserved softmax = Softmax4D(conv_final, name='4dsoftmax') return [softmax]
def define_obs_pars_dict(d, MuY_layers_In=None): """ Defines the parameters and neural networks pertaining the Observation Model. Args: d: The object-dictionary Output: ObsParsDict: A dictionary containing all these hyperparameters and NNs, to be fed to the Observation Model """ act_func_dict = {'softplus' : softplus, 'tanh' : tanh} output_nl_dict = {'linear' : linear, 'softplus' : softplus} output_nl = output_nl_dict[d.output_nl] nl = act_func_dict[d.act_func_hls] nnodesMrec = d.nnodesMrec MuY_dpth = d.muydepth NNMuY = InputLayer((None, None, d.xDim), name='MuY_IL') for i in range(MuY_dpth): NNMuY = DenseLayer(NNMuY, nnodesMrec, nonlinearity=nl, W=Orthogonal(), num_leading_axes=2, name='MuY_HL' + str(i)) NNMuY = DenseLayer(NNMuY, d.yDim, nonlinearity=output_nl, W=Orthogonal(), num_leading_axes=2, name='MuY_OL') # Initialize layers weight if so desired. # if d.loadfit: # MuY_layers = lasagne.layers.get_all_layers(NNMuY)[1:] # init_layers(MuY_layers_In, MuY_layers) ObsParsDict = {'NNMuY' : NNMuY, 'NNMuY_W' : d.outWgenscale, 'NNMuY_b' : d.outbgenscale, 'LATCLASS' : LocallyLinearEvolution, 'is_out_positive' : d.is_out_positive} return ObsParsDict
def build_segmenter_simple_absurd_res(): sys.setrecursionlimit(1500) inp = ll.InputLayer(shape=(None, 1, None, None), name='input') n_layers = 64 # should get a 128 x 128 receptive field layers = [inp] for i in range(n_layers): # every 2 layers, add a skip connection layers.append( ll.Conv2DLayer(layers[-1], num_filters=8, filter_size=(3, 3), pad='same', W=Orthogonal(), nonlinearity=linear, name='conv%d' % (i + 1))) layers.append(ll.BatchNormLayer(layers[-1], name='bn%i' % (i + 1))) if (i % 2 == 0) and (i != 0): layers.append( ll.ElemwiseSumLayer([ layers[-1], # prev layer layers[-6], ] # 3 actual layers per block, skip the previous block )) layers.append(ll.NonlinearityLayer(layers[-1], nonlinearity=rectify)) # our output layer is also convolutional, remember that our Y is going to be the same exact size as the conv_final = ll.Conv2DLayer(layers[-1], num_filters=2, filter_size=(3, 3), pad='same', W=Orthogonal(), name='conv_final', nonlinearity=linear) # we need to reshape it to be a (batch*n*m x 3), i.e. unroll s.t. the feature dimension is preserved softmax = Softmax4D(conv_final, name='4dsoftmax') return [softmax]
def __init__(self, ObsPars, yDim, xDim, Y=None, X=None, lat_ev_model=None, LATCLASS=LocallyLinearEvolution): """ """ self.yDim = yDim self.xDim = xDim self.Y = Y = T.tensor3('Y') if Y is None else Y if lat_ev_model is None: self.common_lat = False self.X = X = T.tensor3('X') if X is None else X self.lat_ev_model = lat_ev_model = LATCLASS({}, xDim, X, nnname='GenEv') else: self.common_lat = True self.lat_ev_model = lat_ev_model self.X = lat_ev_model.get_X() if X is None else X for key in ObsPars.keys(): setattr(self, key, ObsPars[key]) if not hasattr(self, 'is_out_positive'): self.is_out_positive = True if not hasattr(self, 'NNMuY_W'): self.NNMuY_W = 1.0 if not hasattr(self, 'NNMuY_b'): self.NNMuY_b = 1.0 if not hasattr(self, 'NNMuY'): NNMuY = InputLayer((None, None, self.xDim), name='MuY_IL') self.NNMuY = DenseLayer(NNMuY, self.yDim, nonlinearity=softplus, W=Orthogonal(), num_leading_axes=2, name='MuY_OL') self.NNMuY.W.set_value(self.NNMuY_W * self.NNMuY.W.get_value()) self.NNMuY.b.set_value(self.NNMuY_b * np.ones(self.yDim) + 2.0 * (np.random.rand(yDim))) inv_tau = 0.2 self.Rate = (lasagne.layers.get_output( self.NNMuY, self.X) if self.is_out_positive else T.exp( inv_tau * lasagne.layers.get_output(self.NNMuY, self.X)))
def define_model(N_HIDDEN, depth, LEARNING_RATE=0.01, GRAD_CLIP=100, trans_vocab_size=0, vocab_size=0, is_train=False): ### Defines lasagne model ### Returns output layer and theano functions for training and computing the cost l_input = lasagne.layers.InputLayer(shape=(None, None, trans_vocab_size)) network = l_input symbolic_batch_size = lasagne.layers.get_output(network).shape[0] while depth > 0: l_forward = lasagne.layers.LSTMLayer( network, N_HIDDEN, grad_clipping=GRAD_CLIP, ingate=lasagne.layers.Gate(W_in=Orthogonal(gain=1.5), W_hid=Orthogonal(gain=1.5), W_cell=Normal(0.1)), forgetgate=lasagne.layers.Gate(W_in=Orthogonal(gain=1.5), W_hid=Orthogonal(gain=1.5), W_cell=Normal(0.1)), cell=lasagne.layers.Gate(W_cell=None, nonlinearity=lasagne.nonlinearities.tanh, W_in=Orthogonal(gain=1.5), W_hid=Orthogonal(gain=1.5)), outgate=lasagne.layers.Gate(W_in=Orthogonal(gain=1.5), W_hid=Orthogonal(gain=1.5), W_cell=Normal(0.1)), backwards=False) l_backward = lasagne.layers.LSTMLayer( network, N_HIDDEN, grad_clipping=GRAD_CLIP, ingate=lasagne.layers.Gate(W_in=Orthogonal(gain=1.5), W_hid=Orthogonal(gain=1.5), W_cell=Normal(0.1)), forgetgate=lasagne.layers.Gate(W_in=Orthogonal(gain=1.5), W_hid=Orthogonal(gain=1.5), W_cell=Normal(0.1)), cell=lasagne.layers.Gate(W_cell=None, nonlinearity=lasagne.nonlinearities.tanh, W_in=Orthogonal(gain=1.5), W_hid=Orthogonal(gain=1.5)), outgate=lasagne.layers.Gate(W_in=Orthogonal(gain=1.5), W_hid=Orthogonal(gain=1.5), W_cell=Normal(0.1)), backwards=True) if depth == 1: l_cell_forward = LSTMLayer( network, N_HIDDEN, grad_clipping=GRAD_CLIP, ingate=lasagne.layers.Gate( W_in=l_forward.W_in_to_ingate, W_hid=l_forward.W_hid_to_ingate, # W_cell=l_forward.W_cell_to_ingate, b=l_forward.b_ingate), forgetgate=lasagne.layers.Gate( W_in=l_forward.W_in_to_forgetgate, W_hid=l_forward.W_hid_to_forgetgate, # W_cell=l_forward.W_cell_to_forgetgate, b=l_forward.b_forgetgate), cell=lasagne.layers.Gate( W_cell=None, nonlinearity=lasagne.nonlinearities.tanh, W_in=l_forward.W_in_to_cell, W_hid=l_forward.W_hid_to_cell, b=l_forward.b_cell), outgate=lasagne.layers.Gate( W_in=l_forward.W_in_to_outgate, W_hid=l_forward.W_hid_to_outgate, # W_cell=l_forward.W_cell_to_outgate, b=l_forward.b_outgate), backwards=False, peepholes=False) l_cell_backwards = LSTMLayer( network, N_HIDDEN, grad_clipping=GRAD_CLIP, ingate=lasagne.layers.Gate( W_in=l_backward.W_in_to_ingate, W_hid=l_backward.W_hid_to_ingate, # W_cell=l_backward.W_cell_to_ingate, b=l_backward.b_ingate), forgetgate=lasagne.layers.Gate( W_in=l_backward.W_in_to_forgetgate, W_hid=l_backward.W_hid_to_forgetgate, # W_cell=l_backward.W_cell_to_forgetgate, b=l_backward.b_forgetgate), cell=lasagne.layers.Gate( W_cell=None, nonlinearity=lasagne.nonlinearities.tanh, W_in=l_backward.W_in_to_cell, W_hid=l_backward.W_hid_to_cell, b=l_backward.b_cell), outgate=lasagne.layers.Gate( W_in=l_backward.W_in_to_outgate, W_hid=l_backward.W_hid_to_outgate, # W_cell=l_backward.W_cell_to_outgate, b=l_backward.b_outgate), backwards=True, peepholes=False) concat_layer = lasagne.layers.ConcatLayer( incomings=[l_forward, l_backward], axis=2) concat_layer = lasagne.layers.ReshapeLayer(concat_layer, (-1, 2 * N_HIDDEN)) network = lasagne.layers.DenseLayer( concat_layer, num_units=N_HIDDEN, W=Orthogonal(), nonlinearity=lasagne.nonlinearities.tanh) network = lasagne.layers.ReshapeLayer( network, (symbolic_batch_size, -1, N_HIDDEN)) depth -= 1 network = lasagne.layers.ReshapeLayer(network, (-1, N_HIDDEN)) l_input_reshape = lasagne.layers.ReshapeLayer(l_input, (-1, trans_vocab_size)) network = lasagne.layers.ConcatLayer(incomings=[network, l_input_reshape], axis=1) l_out = lasagne.layers.DenseLayer( network, num_units=vocab_size, W=lasagne.init.Normal(), nonlinearity=lasagne.nonlinearities.softmax) target_values = T.dmatrix('target_output') network_output = lasagne.layers.get_output(l_out) network = lasagne.layers.get_output(network) concat_layer = lasagne.layers.get_output(concat_layer) last_lstm_cells_forward = lasagne.layers.get_output(l_cell_forward) last_lstm_cells_backwards = lasagne.layers.get_output(l_cell_backwards) #gates = l_cell_forward.get_gates() cost = T.nnet.categorical_crossentropy(network_output, target_values).mean() all_params = lasagne.layers.get_all_params(l_out, trainable=True) print("Compiling Functions ...") if is_train: print("Computing Updates ...") #updates = lasagne.updates.adagrad(cost, all_params, LEARNING_RATE) updates = lasagne.updates.adam( cost, all_params, beta1=0.5, learning_rate=LEARNING_RATE) # from DCGAN paper compute_cost = theano.function([l_input.input_var, target_values], cost, allow_input_downcast=True) train = theano.function([l_input.input_var, target_values], cost, updates=updates, allow_input_downcast=True) return (l_out, train, compute_cost) else: guess = theano.function( [l_input.input_var], [ network_output, network, concat_layer, last_lstm_cells_forward, last_lstm_cells_backwards #gates[0], gates[1], gates[2] ], allow_input_downcast=True) return (l_out, guess)
from lasagne.nonlinearities import softmax, tanh, sigmoid, rectify, LeakyRectify seed(SEED) print 'set random seed to {0} while loading NNet'.format(SEED) nonlinearities = { 'tanh': tanh, 'sigmoid': sigmoid, 'rectify': rectify, 'leaky2': LeakyRectify(leakiness=0.02), 'leaky20': LeakyRectify(leakiness=0.2), 'softmax': softmax, } initializers = { 'orthogonal': Orthogonal(), 'sparse': Sparse(), 'glorot_normal': GlorotNormal(), 'glorot_uniform': GlorotUniform(), 'he_normal': HeNormal(), 'he_uniform': HeUniform(), } class NNet(BaseEstimator, ClassifierMixin): def __init__( self, name='nameless_net', # used for saving, so maybe make it unique dense1_size=60, dense1_nonlinearity='tanh', dense1_init='orthogonal',
num_units=num_rnn_units, only_return_final=True, backwards=True, grad_clipping=GRAD_CLIP, nonlinearity=lasagne.nonlinearities.tanh, mask_input=net['mask']) net['lstm7'] = lasagne.layers.ConcatLayer( [net['lstm7_forward'], net['lstm7_backward']]) else: net['lstm7'] = LSTMLayer(net['fc6_resize'], num_units=num_rnn_units, unroll_scan=True, only_return_final=True, grad_clipping=GRAD_CLIP, nonlinearity=lasagne.nonlinearities.tanh, cell_init=Orthogonal(), hid_init=Orthogonal(), learn_init=True, mask_input=net['mask']) print 'dim_lstm7:', net['lstm7'].output_shape net['lstm7_dropout'] = DropoutLayer(net['lstm7'], p=0.5) # l_fc8 = DenseLayer(net['lstm7']_dropout, num_units=num_classes, W=HeNormal(), nonlinearity=softmax) net['fc8-1'] = DenseLayer(net['lstm7_dropout'], num_units=num_classes, nonlinearity=None, W=lasagne.init.Normal(std=0.01), b=lasagne.init.Constant(0.)) net['prob'] = NonlinearityLayer(net['fc8-1'], softmax) ########################################################################################################################
def __init__(self, train_raw, test_raw, dim, mode, l2, l1, batch_norm, dropout, batch_size, ihm_C, los_C, ph_C, decomp_C, partition, nbins, **kwargs): print "==> not used params in network class:", kwargs.keys() self.train_raw = train_raw self.test_raw = test_raw self.dim = dim self.mode = mode self.l2 = l2 self.l1 = l1 self.batch_norm = batch_norm self.dropout = dropout self.batch_size = batch_size self.ihm_C = ihm_C self.los_C = los_C self.ph_C = ph_C self.decomp_C = decomp_C self.nbins = nbins if (partition == 'log'): self.get_bin = metrics.get_bin_log self.get_estimate = metrics.get_estimate_log else: assert self.nbins == 10 self.get_bin = metrics.get_bin_custom self.get_estimate = metrics.get_estimate_custom self.train_batch_gen = self.get_batch_gen(self.train_raw) self.test_batch_gen = self.get_batch_gen(self.test_raw) self.input_var = T.tensor3('X') self.input_lens = T.ivector('L') self.ihm_pos = T.ivector('ihm_pos') self.ihm_mask = T.ivector('ihm_mask') self.ihm_label = T.ivector('ihm_label') self.los_mask = T.imatrix('los_mask') self.los_label = T.matrix('los_label') # for regression #self.los_label = T.imatrix('los_label') self.ph_label = T.imatrix('ph_label') self.decomp_mask = T.imatrix('decomp_mask') self.decomp_label = T.imatrix('decomp_label') print "==> Building neural network" # common network network = layers.InputLayer((None, None, self.train_raw[0][0].shape[1]), input_var=self.input_var) if (self.dropout > 0): network = layers.DropoutLayer(network, p=self.dropout) network = layers.LSTMLayer(incoming=network, num_units=dim, only_return_final=False, grad_clipping=10, ingate=lasagne.layers.Gate( W_in=Orthogonal(), W_hid=Orthogonal(), W_cell=Normal(0.1)), forgetgate=lasagne.layers.Gate( W_in=Orthogonal(), W_hid=Orthogonal(), W_cell=Normal(0.1)), cell=lasagne.layers.Gate(W_cell=None, nonlinearity=lasagne.nonlinearities.tanh, W_in=Orthogonal(), W_hid=Orthogonal()), outgate=lasagne.layers.Gate( W_in=Orthogonal(), W_hid=Orthogonal(), W_cell=Normal(0.1))) if (self.dropout > 0): network = layers.DropoutLayer(network, p=self.dropout) lstm_output = layers.get_output(network) self.params = layers.get_all_params(network, trainable=True) self.reg_params = layers.get_all_params(network, regularizable=True) # for each example in minibatch take the last output last_outputs = [] for index in range(self.batch_size): last_outputs.append(lstm_output[index, self.input_lens[index]-1, :]) last_outputs = T.stack(last_outputs) # take 48h outputs for fixed mortality task mid_outputs = [] for index in range(self.batch_size): mid_outputs.append(lstm_output[index, self.ihm_pos[index], :]) mid_outputs = T.stack(mid_outputs) # in-hospital mortality related network ihm_network = layers.InputLayer((None, dim), input_var=mid_outputs) ihm_network = layers.DenseLayer(incoming=ihm_network, num_units=2, nonlinearity=softmax) self.ihm_prediction = layers.get_output(ihm_network) self.ihm_det_prediction = layers.get_output(ihm_network, deterministic=True) self.params += layers.get_all_params(ihm_network, trainable=True) self.reg_params += layers.get_all_params(ihm_network, regularizable=True) self.ihm_loss = (self.ihm_mask * categorical_crossentropy(self.ihm_prediction, self.ihm_label)).mean() # length of stay related network # Regression los_network = layers.InputLayer((None, None, dim), input_var=lstm_output) los_network = layers.ReshapeLayer(los_network, (-1, dim)) los_network = layers.DenseLayer(incoming=los_network, num_units=1, nonlinearity=rectify) los_network = layers.ReshapeLayer(los_network, (lstm_output.shape[0], -1)) self.los_prediction = layers.get_output(los_network) self.los_det_prediction = layers.get_output(los_network, deterministic=True) self.params += layers.get_all_params(los_network, trainable=True) self.reg_params += layers.get_all_params(los_network, regularizable=True) self.los_loss = (self.los_mask * squared_error(self.los_prediction, self.los_label)).mean(axis=1).mean(axis=0) # phenotype related network ph_network = layers.InputLayer((None, dim), input_var=last_outputs) ph_network = layers.DenseLayer(incoming=ph_network, num_units=25, nonlinearity=sigmoid) self.ph_prediction = layers.get_output(ph_network) self.ph_det_prediction = layers.get_output(ph_network, deterministic=True) self.params += layers.get_all_params(ph_network, trainable=True) self.reg_params += layers.get_all_params(ph_network, regularizable=True) self.ph_loss = nn_utils.multilabel_loss(self.ph_prediction, self.ph_label) # decompensation related network decomp_network = layers.InputLayer((None, None, dim), input_var=lstm_output) decomp_network = layers.ReshapeLayer(decomp_network, (-1, dim)) decomp_network = layers.DenseLayer(incoming=decomp_network, num_units=2, nonlinearity=softmax) decomp_network = layers.ReshapeLayer(decomp_network, (lstm_output.shape[0], -1, 2)) self.decomp_prediction = layers.get_output(decomp_network)[:, :, 1] self.decomp_det_prediction = layers.get_output(decomp_network, deterministic=True)[:, :, 1] self.params += layers.get_all_params(decomp_network, trainable=True) self.reg_params += layers.get_all_params(decomp_network, regularizable=True) self.decomp_loss = nn_utils.multilabel_loss_with_mask(self.decomp_prediction, self.decomp_label, self.decomp_mask) """ data = next(self.train_batch_gen) print max(data[1]) print lstm_output.eval({self.input_var:data[0]}).shape exit() """ if self.l2 > 0: self.loss_l2 = self.l2 * nn_utils.l2_reg(self.reg_params) else: self.loss_l2 = T.constant(0) if self.l1 > 0: self.loss_l1 = self.l1 * nn_utils.l1_reg(self.reg_params) else: self.loss_l1 = T.constant(0) self.reg_loss = self.loss_l1 + self.loss_l2 self.loss = (ihm_C * self.ihm_loss + los_C * self.los_loss + ph_C * self.ph_loss + decomp_C * self.decomp_loss + self.reg_loss) #updates = lasagne.updates.adadelta(self.loss, self.params, # learning_rate=0.001) #updates = lasagne.updates.momentum(self.loss, self.params, # learning_rate=0.00003) #updates = lasagne.updates.adam(self.loss, self.params) updates = lasagne.updates.adam(self.loss, self.params, beta1=0.5, learning_rate=0.0001) # from DCGAN paper #updates = lasagne.updates.nesterov_momentum(loss, params, momentum=0.9, # learning_rate=0.001, all_inputs = [self.input_var, self.input_lens, self.ihm_pos, self.ihm_mask, self.ihm_label, self.los_mask, self.los_label, self.ph_label, self.decomp_mask, self.decomp_label] train_outputs = [self.ihm_prediction, self.los_prediction, self.ph_prediction, self.decomp_prediction, self.loss, self.ihm_loss, self.los_loss, self.ph_loss, self.decomp_loss, self.reg_loss] test_outputs = [self.ihm_det_prediction, self.los_det_prediction, self.ph_det_prediction, self.decomp_det_prediction, self.loss, self.ihm_loss, self.los_loss, self.ph_loss, self.decomp_loss, self.reg_loss] ## compiling theano functions if self.mode == 'train': print "==> compiling train_fn" self.train_fn = theano.function(inputs=all_inputs, outputs=train_outputs, updates=updates) print "==> compiling test_fn" self.test_fn = theano.function(inputs=all_inputs, outputs=test_outputs)
import numpy as np import re import sys import theano from common import ENCODING, EPSILON, FMAX, FMIN, MAX_EPOCHS, MIN_EPOCHS, \ NONMATCH_RE, NEGATIVE_IDX, NEUTRAL_IDX, POSITIVE_IDX, \ floatX, sgd_updates_adadelta from common import POSITIVE as POSITIVE_LBL from common import NEGATIVE as NEGATIVE_LBL from germanet import normalize ################################################################## # Constants SPACE_RE = re.compile(r"\s+") ORTHOGONAL = Orthogonal() HE_UNIFORM = HeUniform() ################################################################## # Methods def digitize_trainset(w2i, a_pos, a_neg, a_neut, a_pos_re, a_neg_re): """Method for generating sentiment lexicons using Velikovich's approach. @param a_N - number of terms to extract @param a_emb_fname - files of the original corpus @param a_pos - initial set of positive terms to be expanded @param a_neg - initial set of negative terms to be expanded @param a_neut - initial set of neutral terms to be expanded @param a_pos_re - regular expression for matching positive terms @param a_neg_re - regular expression for matching negative terms
('pool2', layers.MaxPool2DLayer), ('conv31', layers.Conv2DLayer), ('conv32', layers.Conv2DLayer), ('conv33', layers.Conv2DLayer), ('pool3', layers.MaxPool2DLayer), ('dropout4', layers.DropoutLayer), ('hidden4', layers.DenseLayer), ('dropout5', layers.DropoutLayer), ('hidden5', layers.DenseLayer), ('output', layers.DenseLayer), ], input_shape=(None, 5, 44, 44), conv11_num_filters=32, conv11_filter_size=(5, 5), conv11_nonlinearity=leaky_rectify, conv11_W=Orthogonal(np.sqrt(2 / (1 + 0.01**2))), conv11_b=Constant(0.1), conv12_num_filters=32, conv12_filter_size=(3, 3), conv12_pad=1, conv12_nonlinearity=leaky_rectify, conv12_W=Orthogonal(np.sqrt(2 / (1 + 0.01**2))), conv12_b=Constant(0.1), pool1_pool_size=(2, 2), conv21_num_filters=64, conv21_filter_size=(3, 3), conv21_pad=1, conv21_nonlinearity=leaky_rectify, conv21_W=Orthogonal(np.sqrt(2 / (1 + 0.01**2))), conv21_b=Constant(0.1), conv22_num_filters=64,
def build_kpextractor128(): inp = ll.InputLayer(shape=(None, 1, 128, 128), name='input') # alternate pooling and conv layers to minimize parameters filter_pad = lambda x, y: (x // 2, y // 2) filter3 = (3, 3) same_pad3 = filter_pad(*filter3) conv1 = ll.Conv2DLayer(inp, num_filters=16, filter_size=filter3, pad=same_pad3, W=Orthogonal(), nonlinearity=rectify, name='conv1') mp1 = ll.MaxPool2DLayer(conv1, 2, stride=2) # now down to 64 x 64 bn1 = ll.BatchNormLayer(mp1) conv2 = ll.Conv2DLayer(bn1, num_filters=32, filter_size=filter3, pad=same_pad3, W=Orthogonal(), nonlinearity=rectify, name='conv2') mp2 = ll.MaxPool2DLayer(conv2, 2, stride=2) # now down to 32 x 32 bn2 = ll.BatchNormLayer(mp2) conv3 = ll.Conv2DLayer(bn2, num_filters=64, filter_size=filter3, pad=same_pad3, W=Orthogonal(), nonlinearity=rectify, name='conv3') mp3 = ll.MaxPool2DLayer(conv3, 2, stride=2) # now down to 16 x 16 bn3 = ll.BatchNormLayer(mp3) conv4 = ll.Conv2DLayer(bn3, num_filters=128, filter_size=filter3, pad=same_pad3, W=Orthogonal(), nonlinearity=rectify, name='conv4') mp4 = ll.MaxPool2DLayer(conv4, 2, stride=2) # now down to 8 x 8 bn4 = ll.BatchNormLayer(mp4) conv5 = ll.Conv2DLayer(bn4, num_filters=256, filter_size=filter3, pad=same_pad3, W=Orthogonal(), nonlinearity=rectify, name='conv5') mp5 = ll.MaxPool2DLayer(conv5, 2, stride=2) # down to 4 x 4 bn5 = ll.BatchNormLayer(mp5) conv6 = ll.Conv2DLayer(bn5, num_filters=512, filter_size=filter3, pad=same_pad3, W=Orthogonal(), nonlinearity=rectify, name='conv6') mp6 = ll.MaxPool2DLayer(conv6, 2, stride=2) # down to 4 x 4 bn6 = ll.BatchNormLayer(mp6) # now let's bring it down to a FC layer that takes in the 2x2x64 mp4 output fc1 = ll.DenseLayer(bn6, num_units=256, nonlinearity=rectify) bn1_fc = ll.BatchNormLayer(fc1) #dp1 = ll.DropoutLayer(bn1, p=0.5) fc2 = ll.DenseLayer(bn1_fc, num_units=64, nonlinearity=rectify) #dp2 = ll.DropoutLayer(fc2, p=0.5) bn2_fc = ll.BatchNormLayer(fc2) out = ll.DenseLayer(bn2_fc, num_units=6, nonlinearity=linear) out_rs = ll.ReshapeLayer(out, ([0], 3, 2)) return out_rs
def build_kpextractor256_decoupled(): inp = ll.InputLayer(shape=(None, 1, 256, 256), name='input') # alternate pooling and conv layers to minimize parameters filter_pad = lambda x, y: (x // 2, y // 2) filter3 = (3, 3) same_pad3 = filter_pad(*filter3) conv1 = ll.Conv2DLayer(inp, num_filters=8, filter_size=filter3, pad=same_pad3, W=Orthogonal(), nonlinearity=rectify, name='conv1') mp1 = ll.MaxPool2DLayer(conv1, 2, stride=2) # now down to 128 x 128 bn1 = ll.BatchNormLayer(mp1) conv2 = ll.Conv2DLayer(bn1, num_filters=16, filter_size=filter3, pad=same_pad3, W=Orthogonal(), nonlinearity=rectify, name='conv2') mp2 = ll.MaxPool2DLayer(conv2, 2, stride=2) # now down to 128 x 128 bn2 = ll.BatchNormLayer(mp2) conv3 = ll.Conv2DLayer(bn2, num_filters=32, filter_size=filter3, pad=same_pad3, W=Orthogonal(), nonlinearity=rectify, name='conv3') mp3 = ll.MaxPool2DLayer(conv3, 2, stride=2) # now down to 32 x 32 bn3 = ll.BatchNormLayer(mp3) conv4 = ll.Conv2DLayer(bn3, num_filters=64, filter_size=filter3, pad=same_pad3, W=Orthogonal(), nonlinearity=rectify, name='conv4') mp4 = ll.MaxPool2DLayer(conv4, 2, stride=2) # now down to 16 x 16 bn4 = ll.BatchNormLayer(mp4) conv5 = ll.Conv2DLayer(bn4, num_filters=128, filter_size=filter3, pad=same_pad3, W=Orthogonal(), nonlinearity=rectify, name='conv5') mp5 = ll.MaxPool2DLayer(conv5, 2, stride=2) # now down to 8 x 8 bn5 = ll.BatchNormLayer(mp5) conv6 = ll.Conv2DLayer(bn5, num_filters=256, filter_size=filter3, pad=same_pad3, W=Orthogonal(), nonlinearity=rectify, name='conv6') mp6 = ll.MaxPool2DLayer(conv6, 2, stride=2) # down to 4 x 4 bn6 = ll.BatchNormLayer(mp6) conv7 = ll.Conv2DLayer(bn6, num_filters=512, filter_size=filter3, pad=same_pad3, W=Orthogonal(), nonlinearity=rectify, name='conv7') mp7 = ll.MaxPool2DLayer(conv7, 2, stride=2) # down to 4 x 4 bn7 = ll.BatchNormLayer(mp7) dp0 = ll.DropoutLayer(bn7, p=0.5) # now let's bring it down to a FC layer that takes in the 2x2x64 mp4 output fc1 = ll.DenseLayer(dp0, num_units=256, nonlinearity=rectify) bn1_fc = ll.BatchNormLayer(fc1) dp1 = ll.DropoutLayer(bn1_fc, p=0.5) # so what we're going to do here instead is break this into three separate layers (each 32 units) # then each of these layers goes into a separate out, and out_rs will be a merge and then reshape fc2_left = ll.DenseLayer(dp1, num_units=32, nonlinearity=rectify) fc2_right = ll.DenseLayer(dp1, num_units=32, nonlinearity=rectify) fc2_notch = ll.DenseLayer(dp1, num_units=32, nonlinearity=rectify) out_left = ll.DenseLayer(fc2_left, num_units=2, nonlinearity=linear) out_right = ll.DenseLayer(fc2_right, num_units=2, nonlinearity=linear) out_notch = ll.DenseLayer(fc2_notch, num_units=2, nonlinearity=linear) out = ll.ConcatLayer([out_left, out_right, out_notch], axis=1) out_rs = ll.ReshapeLayer(out, ([0], 3, 2)) return out_rs
def test_orthogonal_1d_not_supported(): from lasagne.init import Orthogonal with pytest.raises(RuntimeError): Orthogonal().sample((100, ))
def config_4c_1233_3d(batch_iterator="BatchIterator", max_epochs=30): custom_batch_iterator = globals()[batch_iterator] net1 = NeuralNet( layers=[ ('input', layers.InputLayer), ('conv1', layers.Conv2DLayer), ('pool1', layers.MaxPool2DLayer), ('conv2_1', layers.Conv2DLayer), ('conv2_2', layers.Conv2DLayer), ('pool2', layers.MaxPool2DLayer), ('conv3_1', layers.Conv2DLayer), ('conv3_2', layers.Conv2DLayer), ('conv3_3', layers.Conv2DLayer), ('pool3', layers.MaxPool2DLayer), ('conv4_1', layers.Conv2DLayer), ('conv4_2', layers.Conv2DLayer), ('conv4_3', layers.Conv2DLayer), ('pool4', layers.MaxPool2DLayer), ('dense1', layers.DenseLayer), ('dense2', layers.DenseLayer), ('dense3', layers.DenseLayer), ('output', layers.DenseLayer), ], # layer parameters: input_shape=(None, 3, 256, 256), conv1_num_filters=86, conv1_filter_size=(5, 5), conv1_stride=(2, 2), conv1_pad=(1, 1), pool1_pool_size=(2, 2), conv2_1_num_filters=128, conv2_1_filter_size=(3, 3), conv2_1_pad=(1, 1), conv2_2_num_filters=128, conv2_2_filter_size=(3, 3), conv2_2_pad=(1, 1), pool2_pool_size=(2, 2), conv3_1_num_filters=256, conv3_1_filter_size=(3, 3), conv3_1_pad=(1, 1), conv3_2_num_filters=256, conv3_2_filter_size=(3, 3), conv3_2_pad=(1, 1), conv3_3_num_filters=256, conv3_3_filter_size=(3, 3), conv3_3_pad=(1, 1), pool3_pool_size=(2, 2), conv4_1_num_filters=196, conv4_1_filter_size=(3, 3), conv4_1_pad=(1, 1), conv4_2_num_filters=196, conv4_2_filter_size=(3, 3), conv4_2_pad=(1, 1), conv4_3_num_filters=196, conv4_3_filter_size=(3, 3), conv4_3_pad=(1, 1), pool4_pool_size=(2, 2), conv1_W=Orthogonal(gain=1.0), conv2_1_W=Orthogonal(gain=1.0), conv2_2_W=Orthogonal(gain=1.0), conv3_1_W=Orthogonal(gain=1.0), conv3_2_W=Orthogonal(gain=1.0), conv3_3_W=Orthogonal(gain=1.0), conv4_1_W=Orthogonal(gain=1.0), conv4_2_W=Orthogonal(gain=1.0), conv4_3_W=Orthogonal(gain=1.0), dense1_num_units=2048, dense2_num_units=1024, dense3_num_units=512, # dense1_nonlinearity=lasagne.nonlinearities.rectify, # dense2_nonlinearity=lasagne.nonlinearities.rectify, dense3_nonlinearity=lasagne.nonlinearities.sigmoid, dense1_W=Orthogonal(gain=1.0), dense2_W=Orthogonal(gain=1.0), dense3_W=Orthogonal(gain=1.0), # output layer uses identity function output_nonlinearity=None, output_num_units=4, # optimization method: update=nesterov_momentum, update_learning_rate=0.01, update_momentum=0.975, batch_iterator_train=custom_batch_iterator(batch_size=64), batch_iterator_test=custom_batch_iterator(batch_size=64), regression=True, max_epochs=max_epochs, verbose=1, ) return net1
def config_4c_1234_3d_smoothl1_lr_step(batch_iterator="BatchIterator", max_epochs=30): custom_batch_iterator = globals()[batch_iterator] net1 = NeuralNet( layers=[ ('input', layers.InputLayer), ('conv1_1', layers.Conv2DLayer), ('conv1_2', layers.Conv2DLayer), ('pool1', layers.MaxPool2DLayer), ('conv2_1', layers.Conv2DLayer), ('conv2_2', layers.Conv2DLayer), ('pool2', layers.MaxPool2DLayer), ('conv3_1', layers.Conv2DLayer), ('conv3_2', layers.Conv2DLayer), ('conv3_3', layers.Conv2DLayer), ('pool3', layers.MaxPool2DLayer), ('conv4_1', layers.Conv2DLayer), ('conv4_2', layers.Conv2DLayer), ('conv4_3', layers.Conv2DLayer), ('conv4_4', layers.Conv2DLayer), ('pool4', layers.MaxPool2DLayer), ('dense1', layers.DenseLayer), # ('drop1', layers.DropoutLayer), ('dense2', layers.DenseLayer), # ('drop2', layers.DropoutLayer), ('dense3', layers.DenseLayer), # ('drop3', layers.DropoutLayer), ('output', layers.DenseLayer), ], # layer parameters: input_shape=(None, 3, 256, 256), conv1_1_num_filters=86, conv1_1_filter_size=(5, 5), conv1_1_stride=(2, 2), conv1_2_num_filters=104, conv1_2_filter_size=(3, 3), conv1_2_pad=(1, 1), pool1_pool_size=(2, 2), pool1_stride=(2, 2), conv2_1_num_filters=128, conv2_1_filter_size=(3, 3), conv2_1_pad=(1, 1), conv2_2_num_filters=128, conv2_2_filter_size=(3, 3), conv2_2_pad=(1, 1), pool2_pool_size=(3, 3), pool2_stride=(2, 2), conv3_1_num_filters=256, conv3_1_filter_size=(3, 3), conv3_1_pad=(1, 1), conv3_2_num_filters=256, conv3_2_filter_size=(3, 3), conv3_2_pad=(1, 1), conv3_3_num_filters=256, conv3_3_filter_size=(3, 3), conv3_3_pad=(1, 1), pool3_pool_size=(3, 3), pool3_stride=(2, 2), conv4_1_num_filters=196, conv4_1_filter_size=(3, 3), conv4_1_pad=(1, 1), conv4_2_num_filters=196, conv4_2_filter_size=(3, 3), conv4_2_pad=(1, 1), conv4_3_num_filters=196, conv4_3_filter_size=(3, 3), conv4_3_pad=(1, 1), conv4_4_num_filters=196, conv4_4_filter_size=(3, 3), conv4_4_pad=(1, 1), pool4_pool_size=(2, 2), pool4_stride=(2, 2), conv1_1_W=Orthogonal(gain=1.0), conv1_2_W=Orthogonal(gain=1.0), conv2_1_W=Orthogonal(gain=1.0), conv2_2_W=Orthogonal(gain=1.0), conv3_1_W=Orthogonal(gain=1.0), conv3_2_W=Orthogonal(gain=1.0), conv3_3_W=Orthogonal(gain=1.0), conv4_1_W=Orthogonal(gain=1.0), conv4_2_W=Orthogonal(gain=1.0), conv4_3_W=Orthogonal(gain=1.0), conv4_4_W=Orthogonal(gain=1.0), dense1_num_units=4096, # drop1_p=0.5, dense2_num_units=2048, # drop2_p=0.5, dense3_num_units=512, # drop3_p=0.5, dense3_nonlinearity=lasagne.nonlinearities.sigmoid, # output layer uses identity function output_nonlinearity=None, output_num_units=4, # optimization method: update=nesterov_momentum, update_learning_rate=theano.shared(np.float32(0.0001)), update_momentum=theano.shared(np.float32(0.9)), batch_iterator_train=custom_batch_iterator(batch_size=72), batch_iterator_test=custom_batch_iterator(batch_size=48), on_epoch_finished=[ StepVariableUpdate('update_learning_rate', changes={30: 0.00005}), AdjustVariable('update_momentum', start=0.9, stop=0.98) ], objective_loss_function=smooth_l1_loss, # objective_loss_function=iou_loss, custom_scores=[ # ('smoothl1', smooth_l1_loss_val), ('iou_loss', iou_loss_val), ('squared_error', mean_squared_error) ], regression=True, max_epochs=max_epochs, verbose=1, ) return net1
def build_segmenter_jet_preconv(): # downsample down to a small region, then upsample all the way back up, using jet architecture # recreate basic FCN-8s structure (though more aptly 1s here since we upsample back to the original input size) # this jet will have another conv layer in the final upsample # difference here is that instead of combining softmax layers in the jet, we'll upsample before the conv_f* layer # this will certainly make the model slower, but should give us better predictions... # The awkward part here is combining the intermediate conv layers when they have different filter shapes # We could: # concat them # have intermediate conv layers that bring them to the shape needed then merge them # in the interests of speed we'll just concat them, though we'll have a ton of filters at the end inp = ll.InputLayer(shape=(None, 1, None, None), name='input') conv1 = ll.Conv2DLayer(inp, num_filters=32, filter_size=(3, 3), pad='same', W=Orthogonal(), nonlinearity=rectify, name='conv1_1') bn1 = ll.BatchNormLayer(conv1, name='bn1') conv2 = ll.Conv2DLayer(conv1, num_filters=64, filter_size=(3, 3), pad='same', W=Orthogonal(), nonlinearity=rectify, name='conv1_2') bn2 = ll.BatchNormLayer(conv2, name='bn2') mp1 = ll.MaxPool2DLayer(conv2, 2, stride=2, name='mp1') # 2x downsample conv3 = ll.Conv2DLayer(mp1, num_filters=128, filter_size=(3, 3), pad='same', W=Orthogonal(), nonlinearity=rectify, name='conv2_1') bn3 = ll.BatchNormLayer(conv3, name='bn3') conv4 = ll.Conv2DLayer(conv3, num_filters=128, filter_size=(3, 3), pad='same', W=Orthogonal(), nonlinearity=rectify, name='conv2_2') bn4 = ll.BatchNormLayer(conv4, name='bn4') mp2 = ll.MaxPool2DLayer(conv4, 2, stride=2, name='mp2') # 4x downsample conv5 = ll.Conv2DLayer(mp2, num_filters=128, filter_size=(3, 3), pad='same', W=Orthogonal(), nonlinearity=rectify, name='conv3_1') bn5 = ll.BatchNormLayer(conv5, name='bn5') conv6 = ll.Conv2DLayer(conv5, num_filters=128, filter_size=(3, 3), pad='same', W=Orthogonal(), nonlinearity=rectify, name='conv3_2') bn6 = ll.BatchNormLayer(conv6, name='bn6') mp3 = ll.MaxPool2DLayer(conv6, 2, stride=2, name='mp3') # 8x downsample conv7 = ll.Conv2DLayer(mp3, num_filters=128, filter_size=(3, 3), pad='same', W=Orthogonal(), nonlinearity=rectify, name='conv4_1') bn7 = ll.BatchNormLayer(conv7, name='bn7') conv8 = ll.Conv2DLayer(conv7, num_filters=128, filter_size=(3, 3), pad='same', W=Orthogonal(), nonlinearity=rectify, name='conv4_2') bn8 = ll.BatchNormLayer(conv8, name='bn8') # f 68 s 8 # now start the upsample ## FIRST UPSAMPLE PREDICTION (akin to FCN-32s) up8 = ll.Upscale2DLayer( bn8, 8, name='upsample_8x') # take loss here, 8x upsample from 8x downsample conv_f8 = ll.Conv2DLayer(up8, num_filters=2, filter_size=(3, 3), pad='same', W=Orthogonal(), nonlinearity=linear, name='conv_8xpred') softmax_8 = Softmax4D(conv_f8, name='4dsoftmax_8x') ## COMBINE BY UPSAMPLING CONV 8 AND CONV 6 conv_8_up2 = ll.Upscale2DLayer(bn8, 2, name='upsample_c8_2') # 4x downsample concat_c8_c6 = ll.ConcatLayer([conv_8_up2, bn6], axis=1, name='concat_c8_c6') up4 = ll.Upscale2DLayer( concat_c8_c6, 4, name='upsample_4x') # take loss here, 4x upsample from 4x downsample conv_f4 = ll.Conv2DLayer(up4, num_filters=2, filter_size=(3, 3), pad='same', W=Orthogonal(), nonlinearity=linear, name='conv_4xpred') softmax_4 = Softmax4D(conv_f4, name='4dsoftmax_4x') # 4x downsample ## COMBINE BY UPSAMPLING CONCAT_86 AND CONV 4 concat_86_up2 = ll.Upscale2DLayer( concat_c8_c6, 2, name='upsample_concat_86_2') # 2x downsample concat_ct86_c4 = ll.ConcatLayer([concat_86_up2, bn4], axis=1, name='concat_ct86_c4') up2 = ll.Upscale2DLayer( concat_ct86_c4, 2, name='upsample_2x' ) # final loss here, 2x upsample from a 2x downsample conv_f2 = ll.Conv2DLayer(up2, num_filters=2, filter_size=(3, 3), pad='same', W=Orthogonal(), nonlinearity=linear, name='conv_2xpred') softmax_2 = Softmax4D(conv_f2, name='4dsoftmax_2x') ## COMBINE BY UPSAMPLING CONCAT_864 AND CONV 2 concat_864_up2 = ll.Upscale2DLayer( concat_ct86_c4, 2, name='upsample_concat_86_2') # no downsample concat_864_c2 = ll.ConcatLayer([concat_864_up2, bn2], axis=1, name='concat_ct864_c2') conv_f1 = ll.Conv2DLayer(concat_864_c2, num_filters=2, filter_size=(3, 3), pad='same', W=Orthogonal(), nonlinearity=linear, name='conv_1xpred') softmax_1 = Softmax4D(conv_f1, name='4dsoftmax_1x') # this is where up1 would go but that doesn't make any sense return [softmax_8, softmax_4, softmax_2, softmax_1]
def __init__(self, train_raw, test_raw, dim, mode, l2, l1, batch_norm, dropout, batch_size, **kwargs): print "==> not used params in network class:", kwargs.keys() self.train_raw = train_raw self.test_raw = test_raw self.dim = dim self.mode = mode self.l2 = l2 self.l1 = l1 self.batch_norm = batch_norm self.dropout = dropout self.batch_size = batch_size self.train_batch_gen = self.get_batch_gen(self.train_raw) self.test_batch_gen = self.get_batch_gen(self.test_raw) self.input_var = T.tensor3('X') self.input_lens = T.ivector('L') self.target_var = T.imatrix('y') """ for i in range(700//self.batch_size): ret=next(self.train_batch_gen) print len(ret[0]) print ret[0][0].shape print len(ret[1]) print type(ret[1][0]) print "---" exit() """ print "==> Building neural network" network = layers.InputLayer((None, None, self.train_raw[0][0].shape[1]), input_var=self.input_var) #print "!!!!!!!!!!! WARNING: dropout on input is disabled !!!!!!!!!!!!!!!!" if (self.dropout > 0): network = layers.DropoutLayer(network, p=self.dropout) network = layers.LSTMLayer(incoming=network, num_units=dim, grad_clipping=10, ingate=lasagne.layers.Gate( W_in=Orthogonal(), W_hid=Orthogonal(), W_cell=Normal(0.1)), forgetgate=lasagne.layers.Gate( W_in=Orthogonal(), W_hid=Orthogonal(), W_cell=Normal(0.1)), cell=lasagne.layers.Gate(W_cell=None, nonlinearity=lasagne.nonlinearities.tanh, W_in=Orthogonal(), W_hid=Orthogonal()), outgate=lasagne.layers.Gate( W_in=Orthogonal(), W_hid=Orthogonal(), W_cell=Normal(0.1))) if (self.dropout > 0): network = layers.DropoutLayer(network, p=self.dropout) network = layers.LSTMLayer(incoming=network, num_units=dim, only_return_final=False, grad_clipping=10, ingate=lasagne.layers.Gate( W_in=Orthogonal(), W_hid=Orthogonal(), W_cell=Normal(0.1)), forgetgate=lasagne.layers.Gate( W_in=Orthogonal(), W_hid=Orthogonal(), W_cell=Normal(0.1)), cell=lasagne.layers.Gate(W_cell=None, nonlinearity=lasagne.nonlinearities.tanh, W_in=Orthogonal(), W_hid=Orthogonal()), outgate=lasagne.layers.Gate( W_in=Orthogonal(), W_hid=Orthogonal(), W_cell=Normal(0.1))) lstm_output = layers.get_output(network) self.params = layers.get_all_params(network, trainable=True) self.reg_params = layers.get_all_params(network, regularizable=True) """ data = next(self.train_batch_gen) print max(data[1]) print lstm_output.eval({self.input_var:data[0]}).shape exit() """ # for each example in minibatch take the last output last_outputs = [] for index in range(self.batch_size): last_outputs.append(lstm_output[index, self.input_lens[index]-1, :]) last_outputs = T.stack(last_outputs) """ data = next(self.train_batch_gen) print max(data[1]) print last_outputs.eval({self.input_var:data[0], self.input_lens:data[1], }).shape exit() """ network = layers.InputLayer(shape=(self.batch_size, self.dim), input_var=last_outputs) if (self.dropout > 0): network = layers.DropoutLayer(network, p=self.dropout) network = layers.DenseLayer(incoming=network, num_units=train_raw[1][0].shape[0], nonlinearity=sigmoid) self.prediction = layers.get_output(network) self.det_prediction = layers.get_output(network, deterministic=True) self.params += layers.get_all_params(network, trainable=True) self.reg_params += layers.get_all_params(network, regularizable=True) self.loss_multilabel = -(self.target_var * T.log(self.prediction) + \ (1 - self.target_var) * T.log(1 - self.prediction)).mean(axis=1)\ .mean(axis=0) if self.l2 > 0: self.loss_l2 = self.l2 * nn_utils.l2_reg(self.reg_params) else: self.loss_l2 = 0 if self.l1 > 0: self.loss_l1 = self.l1 * nn_utils.l1_reg(self.reg_params) else: self.loss_l1 = 0 self.loss = self.loss_multilabel + self.loss_l2 + self.loss_l1 #updates = lasagne.updates.adadelta(self.loss, self.params, # learning_rate=0.001) #updates = lasagne.updates.momentum(self.loss, self.params, # learning_rate=0.00003) #updates = lasagne.updates.adam(self.loss, self.params) updates = lasagne.updates.adam(self.loss, self.params, beta1=0.5, learning_rate=0.0001) # from DCGAN paper #updates = lasagne.updates.nesterov_momentum(loss, params, momentum=0.9, # learning_rate=0.001, ## compiling theano functions if self.mode == 'train': print "==> compiling train_fn" self.train_fn = theano.function(inputs=[self.input_var, self.input_lens, self.target_var], outputs=[self.prediction, self.loss], updates=updates) print "==> compiling test_fn" self.test_fn = theano.function(inputs=[self.input_var, self.input_lens, self.target_var], outputs=[self.det_prediction, self.loss])
def build_segmenter_upsample(): # downsample down to a small region, then upsample all the way back up # Note: w/o any learning on the upsampler, we're limited in how far we can downsample # there will always be an error signal unless the loss fn is run on downsampled targets... inp = ll.InputLayer(shape=(None, 1, None, None), name='input') conv1 = ll.Conv2DLayer(inp, num_filters=32, filter_size=(3, 3), pad='same', W=Orthogonal(), nonlinearity=rectify, name='conv1_1') bn1 = ll.BatchNormLayer(conv1, name='bn1') conv2 = ll.Conv2DLayer(bn1, num_filters=64, filter_size=(3, 3), pad='same', W=Orthogonal(), nonlinearity=rectify, name='conv1_2') bn2 = ll.BatchNormLayer(conv2, name='bn2') mp1 = ll.MaxPool2DLayer(bn2, 2, stride=2, name='mp1') # 2x downsample conv3 = ll.Conv2DLayer(mp1, num_filters=128, filter_size=(3, 3), pad='same', W=Orthogonal(), nonlinearity=rectify, name='conv2_1') bn3 = ll.BatchNormLayer(conv3, name='bn3') conv4 = ll.Conv2DLayer(bn3, num_filters=128, filter_size=(3, 3), pad='same', W=Orthogonal(), nonlinearity=rectify, name='conv2_2') bn4 = ll.BatchNormLayer(conv4, name='bn4') mp2 = ll.MaxPool2DLayer(bn4, 2, stride=2, name='mp2') # 4x downsample conv5 = ll.Conv2DLayer(mp2, num_filters=128, filter_size=(3, 3), pad='same', W=Orthogonal(), nonlinearity=rectify, name='conv3_1') bn5 = ll.BatchNormLayer(conv5, name='bn5') conv6 = ll.Conv2DLayer(bn5, num_filters=128, filter_size=(3, 3), pad='same', W=Orthogonal(), nonlinearity=rectify, name='conv3_2') bn6 = ll.BatchNormLayer(conv6, name='bn6') mp3 = ll.MaxPool2DLayer(bn6, 2, stride=2, name='mp3') # 8x downsample conv7 = ll.Conv2DLayer(mp3, num_filters=128, filter_size=(3, 3), pad='same', W=Orthogonal(), nonlinearity=rectify, name='conv4_1') bn7 = ll.BatchNormLayer(conv7, name='bn7') conv8 = ll.Conv2DLayer(bn7, num_filters=128, filter_size=(3, 3), pad='same', W=Orthogonal(), nonlinearity=rectify, name='conv4_2') bn8 = ll.BatchNormLayer(conv8, name='bn8') # f 68 s 8 # now start the upsample up = ll.Upscale2DLayer(bn8, 8, name='upsample_8x') conv_f = ll.Conv2DLayer(up, num_filters=2, filter_size=(3, 3), pad='same', W=Orthogonal(), nonlinearity=linear, name='conv_final') softmax = Softmax4D(conv_f, name='4dsoftmax') return [softmax]
params = [zipped_grads, running_grads, running_grads2, updir] return (f_grad_shared, f_update, params) ################################################################## # Variables and Constants MAX_ITERS = 150 # 450 CONV_EPS = 1e-5 DFLT_VDIM = 100 _HE_NORMAL = HeNormal() HE_NORMAL = lambda x: floatX(_HE_NORMAL.sample(x)) _HE_UNIFORM = HeUniform() HE_UNIFORM = lambda x: floatX(_HE_UNIFORM.sample(x)) _HE_UNIFORM_RELU = HeUniform(gain=np.sqrt(2)) HE_UNIFORM_RELU = lambda x: floatX(_HE_UNIFORM_RELU.sample(x)) _RELU_ALPHA = 0. _HE_UNIFORM_LEAKY_RELU = HeUniform( gain=np.sqrt(2. / (1 + (_RELU_ALPHA or 1e-6)**2))) HE_UNIFORM_LEAKY_RELU = lambda x: \ floatX(_HE_UNIFORM_LEAKY_RELU.sample(x)) _ORTHOGONAL = Orthogonal() ORTHOGONAL = lambda x: floatX(_ORTHOGONAL.sample(x)) TRNG = RandomStreams()
def build_segmenter_jet_2(): # downsample down to a small region, then upsample all the way back up, using jet architecture # recreate basic FCN-8s structure (though more aptly 1s here since we upsample back to the original input size) # this jet will have another conv layer in the final upsample inp = ll.InputLayer(shape=(None, 1, None, None), name='input') conv1 = ll.Conv2DLayer(inp, num_filters=32, filter_size=(3, 3), pad='same', W=Orthogonal(), nonlinearity=rectify, name='conv1_1') bn1 = ll.BatchNormLayer(conv1, name='bn1') conv2 = ll.Conv2DLayer(bn1, num_filters=64, filter_size=(3, 3), pad='same', W=Orthogonal(), nonlinearity=rectify, name='conv1_2') bn2 = ll.BatchNormLayer(conv2, name='bn2') mp1 = ll.MaxPool2DLayer(bn2, 2, stride=2, name='mp1') # 2x downsample conv3 = ll.Conv2DLayer(mp1, num_filters=128, filter_size=(3, 3), pad='same', W=Orthogonal(), nonlinearity=rectify, name='conv2_1') bn3 = ll.BatchNormLayer(conv3, name='bn3') conv4 = ll.Conv2DLayer(bn3, num_filters=128, filter_size=(3, 3), pad='same', W=Orthogonal(), nonlinearity=rectify, name='conv2_2') bn4 = ll.BatchNormLayer(conv4, name='bn4') mp2 = ll.MaxPool2DLayer(bn4, 2, stride=2, name='mp2') # 4x downsample conv5 = ll.Conv2DLayer(mp2, num_filters=128, filter_size=(3, 3), pad='same', W=Orthogonal(), nonlinearity=rectify, name='conv3_1') bn5 = ll.BatchNormLayer(conv5, name='bn5') conv6 = ll.Conv2DLayer(bn5, num_filters=128, filter_size=(3, 3), pad='same', W=Orthogonal(), nonlinearity=rectify, name='conv3_2') bn6 = ll.BatchNormLayer(conv6, name='bn6') mp3 = ll.MaxPool2DLayer(bn6, 2, stride=2, name='mp3') # 8x downsample conv7 = ll.Conv2DLayer(mp3, num_filters=128, filter_size=(3, 3), pad='same', W=Orthogonal(), nonlinearity=rectify, name='conv4_1') bn7 = ll.BatchNormLayer(conv7, name='bn7') conv8 = ll.Conv2DLayer(bn7, num_filters=128, filter_size=(3, 3), pad='same', W=Orthogonal(), nonlinearity=rectify, name='conv4_2') bn8 = ll.BatchNormLayer(conv8, name='bn8') # f 68 s 8 # now start the upsample ## FIRST UPSAMPLE PREDICTION (akin to FCN-32s) conv_f8 = ll.Conv2DLayer(bn8, num_filters=2, filter_size=(3, 3), pad='same', W=Orthogonal(), nonlinearity=linear, name='conv_8xpred') softmax_8 = Softmax4D(conv_f8, name='4dsoftmax_8x') up8 = ll.Upscale2DLayer( softmax_8, 8, name='upsample_8x') # take loss here, 8x upsample from 8x downsample ## COMBINE BY UPSAMPLING SOFTMAX 8 AND PRED ON CONV 6 softmax_4up = ll.Upscale2DLayer(softmax_8, 2, name='upsample_4x_pre') # 4x downsample conv_f6 = ll.Conv2DLayer(bn6, num_filters=2, filter_size=(3, 3), pad='same', W=Orthogonal(), nonlinearity=linear, name='conv_4xpred') softmax_4 = Softmax4D(conv_f6, name='4dsoftmax_4x') # 4x downsample softmax_4_merge = ll.ElemwiseSumLayer([softmax_4, softmax_4up], coeffs=0.5, name='softmax_4_merge') up4 = ll.Upscale2DLayer( softmax_4_merge, 4, name='upsample_4x') # take loss here, 4x upsample from 4x downsample ## COMBINE BY UPSAMPLING SOFTMAX_4_MERGE AND CONV 4 softmax_2up = ll.Upscale2DLayer(softmax_4_merge, 2, name='upsample_2x_pre') # 2x downsample conv_f4 = ll.Conv2DLayer(bn4, num_filters=2, filter_size=(3, 3), pad='same', W=Orthogonal(), nonlinearity=linear, name='conv_2xpred') softmax_2 = Softmax4D(conv_f4, name='4dsoftmax_2x') softmax_2_merge = ll.ElemwiseSumLayer([softmax_2, softmax_2up], coeffs=0.5, name='softmax_2_merge') up2 = ll.Upscale2DLayer( softmax_2_merge, 2, name='upsample_2x' ) # final loss here, 2x upsample from a 2x downsample ## COMBINE BY UPSAMPLING SOFTMAX_2_MERGE AND CONV 2 softmax_1up = ll.Upscale2DLayer( softmax_2_merge, 2, name='upsample_1x_pre') # 1x downsample (i.e. no downsample) conv_f2 = ll.Conv2DLayer(bn2, num_filters=2, filter_size=(3, 3), pad='same', W=Orthogonal(), nonlinearity=linear, name='conv_1xpred') softmax_1 = Softmax4D(conv_f2, name='4dsoftmax_1x') softmax_1_merge = ll.ElemwiseSumLayer([softmax_1, softmax_1up], coeffs=0.5, name='softmax_1_merge') # this is where up1 would go but that doesn't make any sense return [up8, up4, up2, softmax_1_merge]
test_frac=0.3) for train, classes, test in validator.yield_cross_validation_sets(): # create tensor objects trainT = train.astype(config.floatX) testT = train.astype(config.floatX) classT = classes.astype('int32') # First, construct an input layer. # The shape parameter defines the expected input shape, which is just the shape of our data matrix X. l_in = InputLayer(shape=trainT.shape, W=Constant()) # A dense layer implements a linear mix (xW + b) followed by a nonlinearity. l_hidden = DenseLayer( l_in, # The first argument is the input to this layer num_units=25, # This defines the layer's output dimensionality nonlinearity=tanh, W=Orthogonal(), ) # Various nonlinearities are available # For our output layer, we'll use a dense layer with a softmax nonlinearity. l_output = DenseLayer(l_hidden, num_units=len(classes), nonlinearity=softmax, W=Constant()) # Now, we can generate the symbolic expression of the network's output given an input variable. net_input = T.matrix('net_input') net_output = l_output.get_output(net_input) # As a loss function, we'll use Theano's categorical_crossentropy function. # This allows for the network output to be class probabilities, # but the target output to be class labels. true_output = T.ivector('true_output') loss = T.mean(T.nnet.categorical_crossentropy(net_output, true_output))
def build_kpextractor64(): inp = ll.InputLayer(shape=(None, 1, 64, 64), name='input') # we're going to build something like what Daniel Nouri made for Facial Keypoint detection for a base reference # http://danielnouri.org/notes/2014/12/17/using-convolutional-neural-nets-to-detect-facial-keypoints-tutorial/ # alternate pooling and conv layers to minimize parameters filter_pad = lambda x, y: (x // 2, y // 2) filter3 = (3, 3) same_pad3 = filter_pad(*filter3) conv1 = ll.Conv2DLayer(inp, num_filters=16, filter_size=filter3, pad=same_pad3, W=Orthogonal(), nonlinearity=rectify, name='conv1') mp1 = ll.MaxPool2DLayer(conv1, 2, stride=2) # now down to 32 x 32 bn1 = ll.BatchNormLayer(mp1) conv2 = ll.Conv2DLayer(bn1, num_filters=32, filter_size=filter3, pad=same_pad3, W=Orthogonal(), nonlinearity=rectify, name='conv2') mp2 = ll.MaxPool2DLayer(conv2, 2, stride=2) # now down to 16 x 16 bn2 = ll.BatchNormLayer(mp2) conv3 = ll.Conv2DLayer(bn2, num_filters=64, filter_size=filter3, pad=same_pad3, W=Orthogonal(), nonlinearity=rectify, name='conv3') mp3 = ll.MaxPool2DLayer(conv3, 2, stride=2) # now down to 8 x 8 bn3 = ll.BatchNormLayer(mp3) conv4 = ll.Conv2DLayer(bn3, num_filters=128, filter_size=filter3, pad=same_pad3, W=Orthogonal(), nonlinearity=rectify, name='conv4') # larger max pool to reduce parameters in the FC layer mp4 = ll.MaxPool2DLayer(conv4, 2, stride=2) # now down to 4x4 bn4 = ll.BatchNormLayer(mp4) conv5 = ll.Conv2DLayer(bn4, num_filters=256, filter_size=filter3, pad=same_pad3, W=Orthogonal(), nonlinearity=rectify, name='conv5') mp5 = ll.MaxPool2DLayer(conv5, 2, stride=2) # down to 2x2 bn5 = ll.BatchNormLayer(mp5) # now let's bring it down to a FC layer that takes in the 2x2x64 mp4 output fc1 = ll.DenseLayer(bn5, num_units=256, nonlinearity=rectify) bn6 = ll.BatchNormLayer(fc1) #dp1 = ll.DropoutLayer(bn1, p=0.5) fc2 = ll.DenseLayer(bn6, num_units=64, nonlinearity=rectify) #dp2 = ll.DropoutLayer(fc2, p=0.5) bn7 = ll.BatchNormLayer(fc2) out = ll.DenseLayer(bn7, num_units=6, nonlinearity=linear) out_rs = ll.ReshapeLayer(out, ([0], 3, 2)) return out_rs
def __init__(self, dim, mode, l2, l1, batch_norm, dropout, batch_size, input_dim=76, **kwargs): print "==> not used params in network class:", kwargs.keys() self.dim = dim self.mode = mode self.l2 = l2 self.l1 = l1 self.batch_norm = batch_norm self.dropout = dropout self.batch_size = batch_size self.input_var = T.tensor3('X') self.input_lens = T.ivector('L') self.target_var = T.ivector('y') self.weight = T.vector('w') print "==> Building neural network" network = layers.InputLayer((None, None, input_dim), input_var=self.input_var) network = layers.LSTMLayer(incoming=network, num_units=dim, only_return_final=False, grad_clipping=10, ingate=lasagne.layers.Gate( W_in=Orthogonal(), W_hid=Orthogonal(), W_cell=Normal(0.1)), forgetgate=lasagne.layers.Gate( W_in=Orthogonal(), W_hid=Orthogonal(), W_cell=Normal(0.1)), cell=lasagne.layers.Gate(W_cell=None, nonlinearity=lasagne.nonlinearities.tanh, W_in=Orthogonal(), W_hid=Orthogonal()), outgate=lasagne.layers.Gate( W_in=Orthogonal(), W_hid=Orthogonal(), W_cell=Normal(0.1))) lstm_output = layers.get_output(network) self.params = layers.get_all_params(network, trainable=True) self.reg_params = layers.get_all_params(network, regularizable=True) # for each example in minibatch take the last output last_outputs = [] for index in range(self.batch_size): last_outputs.append(lstm_output[index, self.input_lens[index]-1, :]) last_outputs = T.stack(last_outputs) network = layers.InputLayer(shape=(self.batch_size, self.dim), input_var=last_outputs) network = layers.DenseLayer(incoming=network, num_units=2, nonlinearity=softmax) self.prediction = layers.get_output(network) self.params += layers.get_all_params(network, trainable=True) self.reg_params += layers.get_all_params(network, regularizable=True) self.loss_ce = (self.weight * categorical_crossentropy(self.prediction, self.target_var)).mean() if self.l2 > 0: self.loss_l2 = self.l2 * nn_utils.l2_reg(self.reg_params) else: self.loss_l2 = 0 if self.l1 > 0: self.loss_l1 = self.l1 * nn_utils.l1_reg(self.reg_params) else: self.loss_l1 = 0 self.loss = self.loss_ce + self.loss_l2 + self.loss_l1 #updates = lasagne.updates.adadelta(self.loss, self.params, # learning_rate=0.001) #updates = lasagne.updates.momentum(self.loss, self.params, # learning_rate=0.00003) #updates = lasagne.updates.adam(self.loss, self.params) updates = lasagne.updates.adam(self.loss, self.params, beta1=0.5, learning_rate=0.0001) # from DCGAN paper #updates = lasagne.updates.nesterov_momentum(loss, params, momentum=0.9, # learning_rate=0.001, ## compiling theano functions if self.mode == 'train': print "==> compiling train_fn" self.train_fn = theano.function(inputs=[self.input_var, self.input_lens, self.target_var, self.weight], outputs=[self.prediction, self.loss], updates=updates) print "==> compiling test_fn" self.test_fn = theano.function(inputs=[self.input_var, self.input_lens, self.target_var, self.weight], outputs=[self.prediction, self.loss])
def get_model(): dtensor4 = T.TensorType('float32', (False,)*4) input_var = dtensor4('inputs') dtensor2 = T.TensorType('float32', (False,)*2) target_var = dtensor2('targets') # input layer with unspecified batch size layer_input = InputLayer(shape=(None, 30, 64, 64), input_var=input_var) #InputLayer(shape=(None, 1, 30, 64, 64), input_var=input_var) layer_0 = DimshuffleLayer(layer_input, (0, 'x', 1, 2, 3)) # Z-score? # Convolution then batchNormalisation then activation layer, then zero padding layer followed by a dropout layer layer_1 = batch_norm(Conv3DDNNLayer(incoming=layer_0, num_filters=64, filter_size=(3,3,3), stride=(1,3,3), pad='same', nonlinearity=leaky_rectify, W=Orthogonal())) layer_2 = MaxPool3DDNNLayer(layer_1, pool_size=(1, 2, 2), stride=(1, 2, 2), pad=(0, 1, 1)) layer_3 = DropoutLayer(layer_2, p=0.25) # Convolution then batchNormalisation then activation layer, then zero padding layer followed by a dropout layer layer_4 = batch_norm(Conv3DDNNLayer(incoming=layer_3, num_filters=128, filter_size=(3,3,3), stride=(1,3,3), pad='same', nonlinearity=leaky_rectify, W=Orthogonal())) layer_5 = MaxPool3DDNNLayer(layer_4, pool_size=(1, 2, 2), stride=(1, 2, 2), pad=(0, 1, 1)) layer_6 = DropoutLayer(layer_5, p=0.25) # Convolution then batchNormalisation then activation layer, then zero padding layer followed by a dropout layer layer_7 = batch_norm(Conv3DDNNLayer(incoming=layer_6, num_filters=256, filter_size=(3,3,3), stride=(1,3,3), pad='same', nonlinearity=leaky_rectify, W=Orthogonal())) layer_8 = MaxPool3DDNNLayer(layer_7, pool_size=(1, 2, 2), stride=(1, 2, 2), pad=(0, 1, 1)) layer_9 = DropoutLayer(layer_8, p=0.25) # Recurrent layer layer_10 = DimshuffleLayer(layer_9, (0,2,1,3,4)) layer_11 = LSTMLayer(layer_10, num_units=612, hid_init=Orthogonal(), only_return_final=False) # Output Layer layer_systole = DenseLayer(layer_11, 600, nonlinearity=leaky_rectify, W=Orthogonal()) layer_diastole = DenseLayer(layer_11, 600, nonlinearity=leaky_rectify, W=Orthogonal()) layer_systole_1 = DropoutLayer(layer_systole, p=0.3) layer_diastole_1 = DropoutLayer(layer_diastole, p=0.3) layer_systole_2 = DenseLayer(layer_systole_1, 1, nonlinearity=None, W=Orthogonal()) layer_diastole_2 = DenseLayer(layer_diastole_1, 1, nonlinearity=None, W=Orthogonal()) layer_output = ConcatLayer([layer_systole_2, layer_diastole_2]) # Loss prediction = get_output(layer_output) loss = squared_error(prediction, target_var) loss = loss.mean() #Updates : Stochastic Gradient Descent (SGD) with Nesterov momentum Or Adam params = get_all_params(layer_output, trainable=True) updates = adam(loss, params) #updates_0 = rmsprop(loss, params) #updates = apply_nesterov_momentum(updates_0, params) # Create a loss expression for validation/testing. The crucial difference # here is that we do a deterministic forward pass through the network, disabling dropout layers. test_prediction = get_output(layer_output, deterministic=True) test_loss = squared_error(test_prediction, target_var) test_loss = test_loss.mean() # Compile a function performing a training step on a mini-batch (by giving # the updates dictionary) and returning the corresponding training loss: train_fn = theano.function([input_var, target_var], loss, updates=updates, allow_input_downcast=True) # Compile a second function computing the validation loss and accuracy val_fn = theano.function([input_var, target_var], test_loss, allow_input_downcast=True) # Compule a third function computing the prediction predict_fn = theano.function([input_var], test_prediction, allow_input_downcast=True) return [layer_output, train_fn, val_fn, predict_fn]