def test_transform_identity():
from lasagne.layers import InputLayer, TransformerLayer
from lasagne.utils import floatX
from theano.tensor import constant
batchsize = 10
l_in = InputLayer((batchsize, 3, 28, 28))
l_loc = InputLayer((batchsize, 6))
layer = TransformerLayer(l_in, l_loc)
inputs = floatX(np.arange(np.prod(l_in.shape)).reshape(l_in.shape))
thetas = floatX(np.tile([1, 0, 0, 0, 1, 0], (batchsize, 1)))
outputs = layer.get_output_for([constant(inputs), constant(thetas)]).eval()
np.testing.assert_allclose(inputs, outputs, rtol=1e-6)
def test_transform_identity():
from lasagne.layers import InputLayer, TransformerLayer
from lasagne.utils import floatX
from theano.tensor import constant
batchsize = 10
l_in = InputLayer((batchsize, 3, 28, 28))
l_loc = InputLayer((batchsize, 6))
layer = TransformerLayer(l_in, l_loc)
inputs = floatX(np.arange(np.prod(l_in.shape)).reshape(l_in.shape))
thetas = floatX(np.tile([1, 0, 0, 0, 1, 0], (batchsize, 1)))
outputs = layer.get_output_for([constant(inputs), constant(thetas)]).eval()
np.testing.assert_allclose(inputs, outputs, rtol=1e-6)
def test_model(input_shape):
b = np.zeros((2, 3), dtype=theano.config.floatX)
b[0, 0] = 1
b[1, 1] = 1
b = b.flatten()
l_in = InputLayer(shape=(None, input_shape[1], input_shape[2],
input_shape[3]))
l_dense = DenseLayer(l_in, num_units=6, W=lasagne.init.Constant(0.0), b=b)
l_transform = TransformerLayer(l_in, l_dense, downsample_factor=1)
return l_transform
def construct_gen(noise_1, noise_2, batch_size=10):
# There are two time steps considered for this model, so two LSTMs
# Reshape noises
noise1_rshp = noise_1.dimshuffle(0, 'x', 1)
noise2_rshp = noise_2.dimshuffle(0, 'x', 1)
lstm1_inp = InputLayer((None, 1, 100), input_var=noise1_rshp)
lstm2_inp = InputLayer((None, 1, 100), input_var=noise2_rshp)
lstm2 = ExposedLSTMLayer(lstm2_inp, 100)
lstm2_h = SliceLayer(lstm2, indices=slice(num_units, None), axis=-1)
lstm2_reshape = ReshapeLayer(lstm2_h, (batch_size, 100))
print("LSTM2's output is " + str(lstm2_reshape.output_shape))
build_bg = gen_bg(lstm1_inp)
build_gfc = gen_fc(lstm2_reshape)
build_gif = gen_fi(build_gfc)
build_gfmask = gen_fmask(build_gfc)
# Affine transformation and pasting with bg
a_t = DenseLayer(lstm2_reshape, num_units=6, W=w1) #6 dim output
m_t_hat = NonlinearityLayer(PadLayer(
TransformerLayer(build_gfmask, a_t, downsample_factor=2), 8),
nonlinearity=tanh)
f_t_hat = NonlinearityLayer(PadLayer(
TransformerLayer(build_gif, a_t, downsample_factor=2), 8),
nonlinearity=tanh)
prior = ElemwiseMergeLayer([m_t_hat, f_t_hat],
merge_function=tensor.mul,
broadcastable=1)
posterior = ElemwiseMergeLayer([ComplimentLayer(m_t_hat), build_bg],
merge_function=tensor.mul,
broadcastable=1)
gen_image = ElemwiseSumLayer([prior, posterior])
return gen_image
def build_st_network(b_size, input_shape, withdisc=True):
# General Params
num_filters = 64
filter_size = (3, 3)
pool_size = (2, 2)
# SP Param
b = np.zeros((2, 3), dtype=theano.config.floatX)
b[0, 0] = 1
b[1, 1] = 1
b = b.flatten() # identity transform
# Localization Network
l_in = InputLayer(shape=(None, input_shape[1], input_shape[2],
input_shape[3]))
l_conv1 = Conv2DLayer(l_in,
num_filters=num_filters,
filter_size=filter_size)
l_pool1 = MaxPool2DLayer(l_conv1, pool_size=pool_size)
l_conv2 = Conv2DLayer(l_pool1,
num_filters=num_filters,
filter_size=filter_size)
l_pool2 = MaxPool2DLayer(l_conv2, pool_size=pool_size)
l_loc = DenseLayer(l_pool2, num_units=64, W=lasagne.init.HeUniform('relu'))
l_param_reg = DenseLayer(l_loc,
num_units=6,
b=b,
nonlinearity=lasagne.nonlinearities.linear,
W=lasagne.init.Constant(0.0),
name='param_regressor')
if withdisc:
l_dis = DiscreteLayer(l_param_reg,
start=Constant(-3.),
stop=Constant(3.),
linrange=Constant(50.))
else:
l_dis = l_param_reg
# Transformer Network
l_trans = TransformerLayer(l_in, l_dis, downsample_factor=1.0)
final = ReshapeLayer(l_trans, shape=([0], -1))
return final
def build_scaled_model(self, previous_layer):
from lasagne.layers import TransformerLayer
b = np.zeros((2, 3), dtype='float32')
b[0, 0] = 7.0
b[1, 1] = 7.0
b = b.flatten() # identity transform
W = lasagne.init.Constant(0.0)
scalenet = {}
scalenet['input'] = previous_layer
scalenet['scale_init'] = lasagne.layers.DenseLayer(scalenet['input'], num_units=6, W=W, b=b, nonlinearity=None)
scalenet['scale'] = TransformerLayer(scalenet['input'], scalenet['scale_init'], downsample_factor=1.0/7.0) # Output should be 3x224x224
return scalenet, scalenet['scale']
def test_transform_border_modes(self):
from lasagne.layers import InputLayer, TransformerLayer
from lasagne.utils import floatX
from theano.tensor import constant
l_in = InputLayer((1, 1, 16, 16))
l_loc = InputLayer((1, 6))
# border_mode='nearest'
layer = TransformerLayer(l_in, l_loc, border_mode='nearest')
image = np.hstack((np.zeros((16, 8)), np.ones((16, 8))))
inputs = floatX(image).reshape(l_in.shape)
thetas = floatX(np.array([[4, 0, 0, 0, 1, 0]]))
outputs = layer.get_output_for([constant(inputs),
constant(thetas)]).eval()
np.testing.assert_allclose(inputs, outputs, rtol=1e-6)
# border_mode='mirror'
layer = TransformerLayer(l_in, l_loc, border_mode='mirror')
outputs = layer.get_output_for([constant(inputs),
constant(thetas)]).eval()
expected = np.zeros_like(outputs)
expected[0, 0] = [.5, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 0, 0, 0, .5]
np.testing.assert_allclose(expected, outputs, rtol=1e-6)
# border_mode='wrap'
layer = TransformerLayer(l_in, l_loc, border_mode='wrap')
outputs = layer.get_output_for([constant(inputs),
constant(thetas)]).eval()
expected = np.zeros_like(outputs)
expected[0, 0] = [1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0, 1, 1, 0, 0]
np.testing.assert_allclose(expected, outputs, rtol=1e-6)
with pytest.raises(ValueError):
layer = TransformerLayer(l_in, l_loc, border_mode='invalid')
outputs = layer.get_output_for([constant(inputs),
constant(thetas)]).eval()
def build_mitosis_encoder(input_shape, encoding_size=32, withst=False):
# Parameters
filter_size = (3, 3)
num_filters = 32
pool_size = (2, 2)
# Localization Network
l_input = InputLayer(shape=(None, input_shape[1], input_shape[2],
input_shape[3]))
l_conv1 = Conv2DLayer(l_input,
num_filters=num_filters,
filter_size=filter_size)
l_conv2 = Conv2DLayer(l_conv1,
num_filters=num_filters,
filter_size=filter_size)
l_pool1 = MaxPool2DLayer(l_conv2, pool_size=pool_size)
l_pipe1_layer = l_pool1 # We need this
# ST Network
if withst:
# ST Params
b = np.zeros((2, 3), dtype=theano.config.floatX)
b[0, 0] = 1
b[1, 1] = 1
b = b.flatten()
# ST Layers
st_encode1 = DenseLayer(l_pool1,
num_units=50,
W=lasagne.init.HeUniform('relu'))
st_encode2 = DenseLayer(st_encode1,
num_units=6,
b=b,
W=lasagne.init.Constant(0.0))
l_trans1 = TransformerLayer(l_input, st_encode2, downsample_factor=1.0)
# Localization Network
st_conv1 = Conv2DLayer(l_trans1,
num_filters=num_filters,
filter_size=filter_size)
st_covn2 = Conv2DLayer(st_conv1,
num_filters=num_filters,
filter_size=filter_size)
st_pool1 = MaxPool2DLayer(st_covn2, pool_size=pool_size)
l_pipe1_layer = st_pool1
# Encoding Step
l_reshape1 = ReshapeLayer(l_pipe1_layer, shape=([0], -1))
l_encode = DenseLayer(l_reshape1,
num_units=encoding_size,
W=lasagne.init.HeUniform('relu'),
name='encoder')
# Decoding Step
l_decode = DenseLayer(l_encode,
W=l_encode.W.T,
num_units=l_reshape1.output_shape[1])
l_reshape2 = ReshapeLayer(
l_decode,
shape=([0], num_filters,
int(np.sqrt(l_reshape1.output_shape[1] / num_filters)),
int(np.sqrt(l_reshape1.output_shape[1] / num_filters))))
# Deconv Network
l_unpool1 = Upscale2DLayer(l_reshape2, scale_factor=pool_size)
l_deconv2 = TransposedConv2DLayer(l_unpool1,
num_filters=l_conv2.input_shape[1],
W=l_conv2.W,
filter_size=l_conv2.filter_size,
stride=l_conv2.stride,
crop=l_conv2.pad,
flip_filters=not l_conv2.flip_filters)
l_deconv1 = TransposedConv2DLayer(l_deconv2,
num_filters=l_conv1.input_shape[1],
W=l_conv1.W,
filter_size=l_conv1.filter_size,
stride=l_conv1.stride,
crop=l_conv1.pad,
flip_filters=not l_conv1.flip_filters)
return l_deconv1
def build_st_network_MNIST(input_shape, mins, maxs, ranges, withdisc=True):
# General Params
num_filters = 64
filter_size = (3, 3)
pool_size = (2, 2)
# SP Param
b = np.zeros((2, 3), dtype=theano.config.floatX)
b[0, 0] = 1
b[1, 1] = 1
b = b.flatten() # identity transform
# Localization Network
l_in = InputLayer(shape=(None, input_shape[1], input_shape[2],
input_shape[3]))
l_conv1 = Conv2DLayer(l_in,
num_filters=num_filters,
filter_size=filter_size)
l_pool1 = MaxPool2DLayer(l_conv1, pool_size=pool_size)
l_conv2 = Conv2DLayer(l_pool1,
num_filters=num_filters,
filter_size=filter_size)
l_pool2 = MaxPool2DLayer(l_conv2, pool_size=pool_size)
l_loc = DenseLayer(l_pool2, num_units=64, W=lasagne.init.HeUniform('relu'))
l_param_reg = DenseLayer(l_loc,
num_units=6,
b=b,
nonlinearity=lasagne.nonlinearities.linear,
W=lasagne.init.Constant(0.0),
name='param_regressor')
if withdisc:
l_dis = DiscreteLayer(l_param_reg, mins, maxs, ranges)
else:
l_dis = l_param_reg
# Transformer Network
l_trans = TransformerLayer(l_in, l_dis, downsample_factor=1.0)
# Classification Network
network = lasagne.layers.Conv2DLayer(
l_trans,
num_filters=32,
filter_size=(5, 5),
nonlinearity=lasagne.nonlinearities.rectify,
W=lasagne.init.GlorotUniform())
# Expert note: Lasagne provides alternative convolutional layers that
# override Theano's choice of which implementation to use; for details
# please see http://lasagne.readthedocs.org/en/latest/user/tutorial.html.
# Max-pooling layer of factor 2 in both dimensions:
network = lasagne.layers.MaxPool2DLayer(network, pool_size=(2, 2))
# Another convolution with 32 5x5 kernels, and another 2x2 pooling:
network = lasagne.layers.Conv2DLayer(
network,
num_filters=32,
filter_size=(5, 5),
nonlinearity=lasagne.nonlinearities.rectify)
network = lasagne.layers.MaxPool2DLayer(network, pool_size=(2, 2))
# A fully-connected layer of 256 units with 50% dropout on its inputs:
network = lasagne.layers.DenseLayer(
lasagne.layers.dropout(network, p=.5),
num_units=256,
nonlinearity=lasagne.nonlinearities.rectify)
# And, finally, the 10-unit output layer with 50% dropout on its inputs:
network = lasagne.layers.DenseLayer(
lasagne.layers.dropout(network, p=.5),
num_units=10,
nonlinearity=lasagne.nonlinearities.softmax)
return network
def test_transform_border_modes(self):
from lasagne.layers import InputLayer, TransformerLayer
from lasagne.utils import floatX
from theano.tensor import constant
l_in = InputLayer((1, 1, 16, 16))
l_loc = InputLayer((1, 6))
# border_mode='nearest'
layer = TransformerLayer(l_in, l_loc, border_mode='nearest')
image = np.hstack((np.zeros((16, 8)), np.ones((16, 8))))
inputs = floatX(image).reshape(l_in.shape)
thetas = floatX(np.array([[4, 0, 0, 0, 1, 0]]))
outputs = layer.get_output_for([constant(inputs),
constant(thetas)]).eval()
np.testing.assert_allclose(inputs, outputs, rtol=1e-6)
# border_mode='mirror'
layer = TransformerLayer(l_in, l_loc, border_mode='mirror')
outputs = layer.get_output_for([constant(inputs),
constant(thetas)]).eval()
expected = np.zeros_like(outputs)
expected[0, 0, :, (0, 1, 2, 3, 8, 9, 10, 11, 15)] = 1.
np.testing.assert_allclose(expected, np.ceil(outputs), rtol=1e-6)
# border_mode='wrap'
layer = TransformerLayer(l_in, l_loc, border_mode='wrap')
outputs = layer.get_output_for([constant(inputs),
constant(thetas)]).eval()
expected = np.zeros_like(outputs)
expected[0, 0, :, (0, 1, 4, 5, 8, 9, 12, 13, 15)] = 1.
np.testing.assert_allclose(expected, np.ceil(outputs), rtol=1e-6)
with pytest.raises(ValueError):
layer = TransformerLayer(l_in, l_loc, border_mode='invalid')
outputs = layer.get_output_for(
[constant(inputs), constant(thetas)]).eval()
def buildNetwork(CFG, params, vocab):
# {{{
"""
TODO document me
"""
# Use params to update CFG
CFG = get_CFG(CFG, params)
#-----------------------------------------------------------
# Setting up the image Embedding.
#-----------------------------------------------------------
l_input_sentence = InputLayer((CFG['BATCH_SIZE'], 1), name='l_input_sentence') # input (1 word)
l_sentence_embedding = lasagne.layers.EmbeddingLayer(l_input_sentence,
input_size=len(vocab),
output_size=CFG['EMBEDDING_SIZE'],
name='l_sentence_embedding')
# Setting up CNN in case of fine tuning.
if CFG['CNN_FINE_TUNE']:
cnn, l_input_cnn, l_input_img = build_CNN(CFG)
if CFG['CNN_MODEL'] == "vgg":
vgg16, resnet50 = cnn, None
elif CFG["CNN_MODEL"] == "resnet":
vgg16, resnet50 = None, cnn
if CFG['START_NORMALIZED'] == 1:
l_input_cnn = ExpressionLayer(l_input_cnn, lambda X: X / (T.sum(X, axis=1, keepdims=True) + 1e-8), output_shape='auto')
elif CFG['START_NORMALIZED'] == 2:
l_input_cnn = ExpressionLayer(l_input_cnn, lambda X: X / T.sqrt(
T.sum(X**2, axis=1, keepdims=True) + 1e-8), output_shape='auto')
else:
l_input_cnn = InputLayer((CFG['BATCH_SIZE'], CFG['CNN_FEATURE_SIZE']), name='l_input_cnn')
l_cnn_embedding = DenseLayer(l_input_cnn, num_units=CFG['EMBEDDING_SIZE'],
nonlinearity=lasagne.nonlinearities.identity, name='l_cnn_embedding')
l_cnn_embedding2 = ReshapeLayer(l_cnn_embedding, ([0], 1, [1]), name='l_cnn_embedding2')
l_rnn_input = InputLayer((CFG['BATCH_SIZE'], 1, CFG['EMBEDDING_SIZE']), name='l_rnn_input')
l_dropout_input = DropoutLayer(
l_rnn_input, p=0.5, name='l_dropout_input')
l_input_reg = None
l_out_reg = None
l_decoder = None
l_region_feedback = None
l_region = None
l_input_img2 = None
l_boxes = None
l_conv = None
l_loc = None
l_loc1 = None
l_input_loc = None
l_sel_region2 = None
l_weighted_region_prev = None
l_weighted_region = None
input_shape = (CFG['BATCH_SIZE'], CFG['EMBEDDING_SIZE'])
if CFG['MODE'] == 'normal':
# {{{1
l_cell_input = InputLayer(input_shape, name='l_cell_input')
l_prev_gru = InputLayer(input_shape, name="l_prev_gru")
l_gru = GRUMemoryLayer(CFG['EMBEDDING_SIZE'],
l_cell_input, l_prev_gru, name='l_gru')
l_dropout_output = DropoutLayer(
l_gru, p=0.5, name='l_dropout_output')
# decoder is a fully connected layer with one output unit for each word in the vocabulary
l_decoder = DenseLayer(l_dropout_output, num_units=len(
vocab), nonlinearity=lasagne.nonlinearities.softmax, name='l_decoder')
l_out = ReshapeLayer(
l_decoder, ([0], 1, [1]), name='l_out')
# }}}
elif CFG['MODE'] == 'tensor':
l_cell_input = InputLayer(input_shape, name='l_cell_input')
l_prev_gru = InputLayer(input_shape, name="l_prev_gru")
l_gru = GRUMemoryLayer(CFG['EMBEDDING_SIZE'], l_cell_input, l_prev_gru, name='l_gru')
l_dropout_output = DropoutLayer(l_gru, p=0.5, name='l_dropout_output')
l_dropout_output = ReshapeLayer(l_dropout_output, ([0], 1, [1]), name='l_dropout_output')
# TODO put me back
if CFG['CNN_FINE_TUNE']:
l_input_regions, _input_regions, l_out_reg, l_input_img2, l_boxes, l_conv = build_finetune_proposals(CFG, vgg16, resnet50)
else:
l_input_regions = InputLayer((CFG['BATCH_SIZE'], CFG['REGION_SIZE'], CFG['NUM_REGIONS']), name='l_input_regions')
# TODO a block.
#l_decoder = build_decoderLayer(l_dropout_output, l_input_regions, vocab, CFG)
if CFG.has_key('DISSECT') and CFG['DISSECT'] != 'No':
if CFG['DISSECT'] == 'wr':
l_decoder = TensorProdFactLayer((l_dropout_output, l_input_regions), dim_h=CFG['EMBEDDING_SIZE'], dim_r=CFG['REGION_SIZE'],
dim_w=len(vocab), nonlinearity=softmax, name='l_tensor', W_hr='skip', b_hr='skip')
elif CFG['DISSECT'] == 'rs':
l_decoder = TensorProdFactLayer((l_dropout_output, l_input_regions), dim_h=CFG['EMBEDDING_SIZE'], dim_r=CFG['REGION_SIZE'],
dim_w=len(vocab), nonlinearity=softmax, name='l_tensor', W_rw='skip', b_rw='skip')
if CFG['DISSECT'] == 'wr':
l_decoder = TensorProdFactLayer((l_dropout_output, l_input_regions), dim_h=CFG['EMBEDDING_SIZE'], dim_r=CFG['REGION_SIZE'],
dim_w=len(vocab), nonlinearity=softmax, name='l_tensor', W_hr='skip', b_hr='skip')
elif CFG['DISSECT'] == 'rs':
l_decoder = TensorProdFactLayer((l_dropout_output, l_input_regions), dim_h=CFG['EMBEDDING_SIZE'], dim_r=CFG['REGION_SIZE'],
dim_w=len(vocab), nonlinearity=softmax, name='l_tensor', W_rw='skip', b_rw='skip')
else:
l_decoder = TensorProdFactLayer((l_dropout_output, l_input_regions), dim_h=CFG['EMBEDDING_SIZE'], dim_r=CFG['REGION_SIZE'],
dim_w=len(vocab), nonlinearity=softmax, name='l_tensor')
l_out = ExpressionLayer(l_decoder, lambda X: X.sum(2), output_shape='auto', name='l_out') # sum over regions
elif CFG['MODE'] == 'transformer':
#{{{2
print(bcolors.OKGREEN + "Transformer mode." + bcolors.ENDC)
from TProd3 import TensorProdFactLayer, WeightedSumLayer, SubsampleLayer
# define a cell
l_cell_input = InputLayer((CFG['BATCH_SIZE'], CFG['EMBEDDING_SIZE']), name='l_cell_input')
from agentnet.memory import GRUMemoryLayer
l_prev_gru = InputLayer((CFG['BATCH_SIZE'], CFG['EMBEDDING_SIZE']), name="l_prev_gru")
if CFG['TRANS_FEEDBACK']:
l_weighted_region_prev = InputLayer((CFG['BATCH_SIZE'], CFG['REGION_SIZE']), name="l_weighted_region_prev")
if CFG['FEEDBACK'] == 2:
l_cell_concat = lasagne.layers.ConcatLayer(
[l_cell_input, l_weighted_region_prev], axis=1, name='l_cell_concat')
else:
print("Are you sure you don't want to use feedback=2? I think you should. Change your mind, then come to see me again.")
else:
l_cell_concat = l_cell_input
l_gru = GRUMemoryLayer(CFG['EMBEDDING_SIZE'],
l_cell_concat, l_prev_gru, name='l_gru')
l_dropout_output = DropoutLayer(l_gru, p=CFG['RNN_DROPOUT'], name='l_dropout_output')
l_dropout_output = ReshapeLayer(l_dropout_output, ([0], 1, [1]), name='l_dropout_output')
if CFG['TRANS_USE_PRETRAINED']:
l_out_reg = vgg16['conv5_2']
#l_out_reg2 = vgg16['conv5_3']
else:
l_out_reg = vgg16['conv5_3']
l_input_reg = InputLayer((CFG['BATCH_SIZE'], CFG['REGION_SIZE'], 14, 14), name='l_input_reg')
l_input_regions = l_input_reg
if CFG['TRANS_USE_PRETRAINED']:
l_input_regions = l_input_regions
else:
if CFG['CONV_NORMALIZED'] == 1:
l_input_regions = ExpressionLayer(l_input_regions, lambda X: X / (T.sum(X, axis=1, keepdims=True) + 1e-8), output_shape='auto')
elif CFG['CONV_NORMALIZED'] == 2:
l_input_regions = ExpressionLayer(l_input_regions, lambda X: X / T.sqrt(T.sum(X**2, axis=1, keepdims=True) + 1e-8), output_shape='auto')
else:
l_input_regions = ExpressionLayer(l_input_regions, lambda X: X * 0.01, output_shape='auto')
factor = 2.0
W = lasagne.init.Constant(0.0)
b = lasagne.init.Constant(0.0)
if CFG['TRANS_MULTIPLE_BOXES']:
num_prop, l_loc = build_loc_net(CFG, l_gru, l_input_regions, 1, (
14, 14), (3, 3), CFG['TRANS_STRIDE'], CFG['TRANS_ZOOM'], W, b, name='')
if CFG['TRANS_ADD_BIG_PROPOSALS']:
num_prop_big, l_loc_big = build_loc_net(CFG, l_gru, l_input_regions, 1, (
14, 14), (3, 3), CFG['TRANS_STRIDE'], CFG['TRANS_ZOOM'] * 2, W, b, name='_big')
l_loc = ConcatLayer((l_loc, l_loc_big), axis=0)
num_prop += num_prop_big
l_sel_region2 = MultiTransformerLayer(l_input_regions, l_loc, kernel_size=(
3, 3), zero_padding=CFG['TRANS_ZEROPAD']) # 3x3
if CFG['TRANS_USE_PRETRAINED']:
Wvgg = vgg16['conv5_3'].W.reshape(
(CFG['REGION_SIZE'], CFG['REGION_SIZE'] * 3 * 3)).swapaxes(0, 1)
bvgg = vgg16['conv5_3'].b
l_sel_region = DenseLayer(
l_sel_region2, num_units=CFG['REGION_SIZE'], name='l_sel_region', W=Wvgg, b=bvgg)
if CFG['CONV_NORMALIZED'] == 1:
l_sel_region = ExpressionLayer(l_sel_region, lambda X: X / (T.sum(X, axis=1, keepdims=True) + 1e-8), output_shape='auto')
elif CFG['CONV_NORMALIZED'] == 2:
l_sel_region = ExpressionLayer(l_sel_region, lambda X: X / T.sqrt(T.sum(X**2, axis=1, keepdims=True) + 1e-8), output_shape='auto')
else:
l_sel_region = l_sel_region
else:
l_sel_region = DenseLayer(
l_sel_region, num_units=CFG['REGION_SIZE'], name='l_sel_region')
l_sel_region = ReshapeLayer(
l_sel_region, (CFG['BATCH_SIZE'], num_prop, CFG['REGION_SIZE']))
l_sel_region = DimshuffleLayer(l_sel_region, (0, 2, 1))
l_sel_region = ReshapeLayer(
l_sel_region, (CFG['BATCH_SIZE'], CFG['REGION_SIZE'], num_prop))
else:
b = np.zeros((2, 3), dtype='float32')
b[0, 0] = 2
b[1, 1] = 2
b = b.flatten()
W = lasagne.init.Constant(0.0)
l_input_loc = l_gru
if CFG['TRANS_USE_STATE']:
l_input_im = ConvLayer(l_input_regions, num_filters=512, filter_size=(
3, 3), pad='same', name='l_reduce_im1')
l_input_im = lasagne.layers.MaxPool2DLayer(l_input_im, (2, 2))
l_input_im = ConvLayer(l_input_im, num_filters=512, filter_size=(
3, 3), pad='same', name='l_reduce_im2')
l_input_im = lasagne.layers.MaxPool2DLayer(l_input_im, (2, 2))
l_input_im = ReshapeLayer(l_input_im, (CFG['BATCH_SIZE'], 512))
l_input_loc = ConcatLayer((l_gru, l_input_im))
l_loc1 = DenseLayer(
l_input_loc, num_units=256, name='l_loc1')
l_loc = DenseLayer(
l_loc1, num_units=6, W=W, b=b, nonlinearity=None, name='l_loc2')
l_sel_region = TransformerLayer(
l_input_regions, l_loc, downsample_factor=2)
l_sel_region = DenseLayer(
l_sel_region, num_units=CFG['REGION_SIZE'], name='l_sel_region')
l_sel_region = ReshapeLayer(
l_sel_region, (CFG['BATCH_SIZE'], CFG['REGION_SIZE'], 1))
l_decoder = TensorProdFactLayer((l_dropout_output, l_sel_region), dim_h=CFG['EMBEDDING_SIZE'], dim_r=CFG['REGION_SIZE'], dim_w=len(
vocab), W=lasagne.init.Normal(std=0.001, mean=0.0), nonlinearity=lasagne.nonlinearities.softmax, name='l_tensor')
if CFG['TRANS_FEEDBACK']:
l_region = ExpressionLayer(l_decoder, lambda X: X.sum(3), output_shape='auto', name='l_region') # sum over regions
l_weighted_region = WeightedSumLayer([l_sel_region, l_region], name='l_weighted_region')
l_out = ExpressionLayer(l_decoder, lambda X: X.sum(
2), output_shape='auto', name='l_out') # sum over regions
#}}}
elif CFG['MODE'] == 'tensor-feedback':
# {{{2
# define a cell
l_cell_input = InputLayer((CFG['BATCH_SIZE'], CFG['EMBEDDING_SIZE']), name='l_cell_input')
l_region_feedback = InputLayer((CFG['BATCH_SIZE'], CFG['NUM_REGIONS']), name='l_region_feedback')
l_cell_concat = lasagne.layers.ConcatLayer(
[l_cell_input, l_region_feedback], axis=1, name='l_cell_concat')
from agentnet.memory import GRUMemoryLayer
l_prev_gru = InputLayer((CFG['BATCH_SIZE'], CFG['EMBEDDING_SIZE']), name="l_prev_gru")
l_gru = GRUMemoryLayer(CFG['EMBEDDING_SIZE'],
l_cell_concat, l_prev_gru, name='l_gru')
l_dropout_output = DropoutLayer(
l_gru, p=0.5, name='l_dropout_output')
l_dropout_output = ReshapeLayer(
l_dropout_output, ([0], 1, [1]), name='l_dropout_output')
from TProd3 import TensorProdFactLayer
l_input_regions = InputLayer((CFG['BATCH_SIZE'], CFG['REGION_SIZE'], CFG['NUM_REGIONS']), name='l_input_regions')
l_tensor = TensorProdFactLayer((l_dropout_output, l_input_regions), dim_h=CFG['EMBEDDING_SIZE'], dim_r=CFG['REGION_SIZE'], dim_w=len(
vocab), nonlinearity=lasagne.nonlinearities.softmax, name='l_tensor')
l_region = ExpressionLayer(l_tensor, lambda X: X.sum(
3), output_shape='auto', name='l_region') # sum over
l_region = ReshapeLayer(
l_region, ([0], [2]), name='l_region')
l_out = ExpressionLayer(l_tensor, lambda X: X.sum(
2), output_shape='auto', name='l_out') # sum over regions
#}}}
elif CFG['MODE'] == 'tensor-feedback2':
# {{{2
l_feedback = InputLayer((CFG['BATCH_SIZE'], CFG['EMBEDDING_SIZE']), name='l_feedback')
l_prev_gru = InputLayer((CFG['BATCH_SIZE'], CFG['EMBEDDING_SIZE']), name="l_prev_gru")
from TProd3 import TensorProdFactLayer, WeightedSumLayer
if CFG['PROPOSALS'] == 3: # use images at different resolution but without fully connected layers
import CNN
vgg16_det = CNN.build_model_RCNN(
CFG['NUM_REGIONS'], CFG['IM_SIZE'] * 1.5, pool_dims=3, dropout_value=CFG['RNN_DROPOUT'])
print "Loading pretrained VGG16 parameters for detection"
l_input_img2 = vgg16_det['input']
l_conv = vgg16_det['conv5_3']
l_boxes = vgg16_det['boxes']
l_input_regions = vgg16_det['reshape']
if CFG['CONV_NORMALIZED'] == 1:
l_input_regions = ExpressionLayer(
l_input_regions, lambda X: X / (T.sum(X, axis=1, keepdims=True) + 1e-8), output_shape='auto')
elif CFG['CONV_NORMALIZED'] == 2:
l_input_regions = ExpressionLayer(
l_input_regions, lambda X: X / T.sqrt(T.sum(X**2, axis=1, keepdims=True) + 1e-8), output_shape='auto')
else:
l_input_regions = ExpressionLayer(
l_input_regions, lambda X: X * 0.01, output_shape='auto')
l_cnn_embedding2 = DenseLayer(
l_input_regions, num_units=CFG['REGION_SIZE'], name='l_cnn_proposals')
l_input_regions = ReshapeLayer(
l_cnn_embedding2, (CFG['BATCH_SIZE'], CFG['NUM_REGIONS'], CFG['REGION_SIZE'], 1))
l_input_regions = lasagne.layers.DimshuffleLayer(
l_input_regions, (0, 2, 1, 3))
l_out_reg = l_input_regions
l_input_reg = InputLayer((CFG['BATCH_SIZE'], CFG['REGION_SIZE'], CFG['NUM_REGIONS'], 1), name='l_input_reg')
l_input_regions = ReshapeLayer(
l_input_reg, (CFG['BATCH_SIZE'], CFG['REGION_SIZE'], CFG['NUM_REGIONS']), name='l_input_regions')
# use images at different resolution but without fully connected layers
elif CFG['PROPOSALS'] == 4:
if CFG['CNN_MODEL'] == 'vgg':
import CNN
vgg16_det = CNN.build_model_RCNN(CFG['NUM_REGIONS'], int(
CFG['IM_SIZE'] * 1.5), pool_dims=1, dropout_value=CFG['RNN_DROPOUT'])
print "Loading pretrained VGG16 parameters for detection"
model_param_values = pickle.load(open('vgg16.pkl'))[
'param values']
lasagne.layers.set_all_param_values(
vgg16_det['conv5_3'], model_param_values[:-6])
l_input_img2 = vgg16_det['input']
l_conv = vgg16_det['conv5_3']
l_boxes = vgg16_det['boxes']
l_input_regions = vgg16_det['crop']
l_input_regions = ReshapeLayer(
l_input_regions, (CFG['BATCH_SIZE'] * CFG['NUM_REGIONS'], CFG['REGION_SIZE']))
else:
resnet50_det = resnet_CNN.build_model_RCNN(
CFG['NUM_REGIONS'], im_size=CFG['IM_SIZE'] * 1.5, pool_dims=1, dropout_value=CFG['RNN_DROPOUT'])
print "Loading pretrained resnet50 parameters for detection"
# You can use this format to store other things for best effort
model_param_values = pickle.load(open('resnet50.pkl'))[
'param values']
from save_layers import add_names_layers_and_params
add_names_layers_and_params(resnet50_det)
#lasagne.layers.set_all_param_values(resnet50['prob'], model_param_values)
set_param_dict(
resnet50_det['pool5'], model_param_values, prefix='', show_layers=False, relax=False)
l_input_img2 = resnet50_det['input']
l_conv = resnet50_det['res4f_relu']
l_boxes = resnet50_det['boxes']
l_input_regions = resnet50_det['crop']
l_input_regions = ReshapeLayer(
l_input_regions, (CFG['BATCH_SIZE'] * CFG['NUM_REGIONS'], CFG['REGION_SIZE']))
if CFG['CONV_NORMALIZED'] == 1:
l_input_regions = ExpressionLayer(
l_input_regions, lambda X: X / (T.sum(X, axis=1, keepdims=True) + 1e-8), output_shape='auto')
elif CFG['CONV_NORMALIZED'] == 2:
l_input_regions = ExpressionLayer(
l_input_regions, lambda X: X / T.sqrt(T.sum(X**2, axis=1, keepdims=True) + 1e-8), output_shape='auto')
else:
_input_regions = ExpressionLayer(
l_input_regions, lambda X: X * 0.01, output_shape='auto')
l_input_regions = ReshapeLayer(
l_input_regions, (CFG['BATCH_SIZE'], CFG['NUM_REGIONS'], CFG['REGION_SIZE'], 1))
l_input_regions = lasagne.layers.DimshuffleLayer(
l_input_regions, (0, 2, 1, 3))
l_out_reg = l_input_regions
l_input_reg = InputLayer((CFG['BATCH_SIZE'], CFG['REGION_SIZE'], CFG['NUM_REGIONS'], 1), name='l_input_reg')
l_input_regions = ReshapeLayer(
l_input_reg, (CFG['BATCH_SIZE'], CFG['REGION_SIZE'], CFG['NUM_REGIONS']), name='l_input_regions')
else:
if CFG['CNN_MODEL'] == 'vgg':
l_out_reg = vgg16['conv5_3']
elif CFG['CNN_MODEL'] == 'resnet':
l_out_reg = resnet50['res4f_relu']
else:
print(bcolors.FAIL + "Unrecognized network" + bcolors.ENDC)
l_input_reg = InputLayer((CFG['BATCH_SIZE'], CFG['REGION_SIZE'], 14, 14), name='l_input_reg')
if CFG['CONV_REDUCED'] > 1:
# added a scaling factor of 100 to avoid exploding gradients
l_input_regions = ExpressionLayer(
l_input_reg, lambda X: X[:, :, ::CFG['CONV_REDUCED'], ::CFG['CONV_REDUCED']], output_shape='auto')
else:
l_input_regions = l_input_reg
if CFG['CONV_NORMALIZED'] == 1:
l_input_regions = ExpressionLayer(
l_input_regions, lambda X: X / (T.sum(X, axis=1, keepdims=True) + 1e-8), output_shape='auto')
elif CFG['CONV_NORMALIZED'] == 2:
l_input_regions = ExpressionLayer(
l_input_regions, lambda X: X / T.sqrt(T.sum(X**2, axis=1, keepdims=True) + 1e-8), output_shape='auto')
else:
l_input_regions = ExpressionLayer(
l_input_regions, lambda X: X * 0.01, output_shape='auto')
if CFG['TENSOR_ADD_CONV']:
l_input_regions = ConvLayer(l_input_regions, num_filters=CFG['REGION_SIZE'], filter_size=(
3, 3), pad='same', name='l_add_con')
l_input_regions = ReshapeLayer(
l_input_regions, (CFG['BATCH_SIZE'], CFG['REGION_SIZE'], CFG['NUM_REGIONS']))
if CFG['TENSOR_TIED']:
l_region_feedback = InputLayer((CFG['BATCH_SIZE'], CFG['NUM_REGIONS']), name='l_region_feedback')
l_region_feedback2 = ReshapeLayer(
l_region_feedback, ([0], 1, [1]), name='l_region_feedback2')
else:
l_shp2 = ReshapeLayer(
l_prev_gru, (CFG['BATCH_SIZE'], 1, CFG['EMBEDDING_SIZE']))
l_shp2 = DropoutLayer(
l_shp2, p=CFG['RNN_DROPOUT'], name='l_shp2')
l_tensor2 = TensorProdFactLayer((l_shp2, l_input_regions), dim_h=CFG['EMBEDDING_SIZE'], dim_r=CFG['REGION_SIZE'], dim_w=len(
vocab), nonlinearity=lasagne.nonlinearities.softmax, name='l_tensor2')
l_region_feedback = ExpressionLayer(l_tensor2, lambda X: T.sum(
X, 3), output_shape='auto', name='l_region') # sum over
l_region_feedback2 = ReshapeLayer(
l_region_feedback, (CFG['BATCH_SIZE'], 1, CFG['NUM_REGIONS']))
l_weighted_region = WeightedSumLayer(
[l_input_regions, l_region_feedback2], name='l_weighted_region')
# define a cell
l_cell_input = InputLayer((CFG['BATCH_SIZE'], CFG['EMBEDDING_SIZE']), name='l_cell_input')
if CFG['FEEDBACK'] == 0: # none
l_cell_concat = l_cell_input
elif CFG['FEEDBACK'] == 1: # none
l_region2 = ReshapeLayer(
l_region_feedback2, ([0], [2]))
l_cell_concat = lasagne.layers.ConcatLayer(
[l_cell_input, l_region2], axis=1, name='l_cell_concat')
elif CFG['FEEDBACK'] == 2:
l_cell_concat = lasagne.layers.ConcatLayer(
[l_cell_input, l_weighted_region], axis=1, name='l_cell_concat')
elif CFG['FEEDBACK'] == 3:
l_region2 = ReshapeLayer(
l_region_feedback2, ([0], [2]))
l_cell_concat = lasagne.layers.ConcatLayer(
[l_cell_input, l_weighted_region, l_region2], axis=1, name='l_cell_concat')
elif CFG['FEEDBACK'] == 4:
# See RNNTraining.py for comments on this.
from TProd3 import WeightedImageLayer
l_weighted_image = WeightedImageLayer(
[l_input_regions, l_region_feedback2], name='l_weighted_image')
if CFG['IMGFEEDBACK_MECHANISM'] == 'highres':
l_weighted_image_reshaped = ReshapeLayer(
l_weighted_image, ([0], [1], 14, 14), name='l_weighted_image_reshaped')
l_weighted_image_conv_reduced = lasagne.layers.MaxPool2DLayer(
l_weighted_image_reshaped, (2, 2), name='l_weighted_image_conv_reduced')
l_feedback_co1 = lasagne.layers.Conv2DLayer(
incoming=l_weighted_image_conv_reduced, num_filters=512, filter_size=(3, 3), pad='same', name='l_feedback_co1')
else:
l_weighted_image_reshaped = ReshapeLayer(
l_weighted_image, ([0], [1], 7, 7), name='l_weighted_image_reshaped')
l_feedback_co1 = lasagne.layers.Conv2DLayer(
incoming=l_weighted_image_reshaped, num_filters=512, filter_size=(3, 3), pad='same', name='l_feedback_co1')
l_feedback_po1 = lasagne.layers.MaxPool2DLayer(
l_feedback_co1, (2, 2), name='l_feedback_po1')
l_feedback_co2 = lasagne.layers.Conv2DLayer(
incoming=l_feedback_po1, num_filters=512, filter_size=(3, 3), pad='same', name='l_feedback_co2')
l_feedback_po2 = lasagne.layers.MaxPool2DLayer(
l_feedback_co2, (2, 2), name='l_feedback_po2')
l_feedback_po2_reshaped = ReshapeLayer(
l_feedback_po2, ([0], [1]), name='l_feedback_po2_reshaped')
l_cell_concat = lasagne.layers.ConcatLayer(
[l_cell_input, l_feedback_po2_reshaped], axis=1, name='l_cell_concat')
from agentnet.memory import GRUMemoryLayer
l_gru = GRUMemoryLayer(CFG['EMBEDDING_SIZE'],
l_cell_concat, l_prev_gru, name='l_gru')
l_dropout_output = DropoutLayer(
l_gru, p=0.5, name='l_dropout_output')
l_shp1 = ReshapeLayer(
l_dropout_output, ([0], 1, [1]), name='l_shp1')
if CFG.has_key('DISSECT') and CFG['DISSECT'] != 'No':
import pdb
pdb.set_trace() # XXX BREAKPOINT
if CFG['DISSECT'] == 'wr':
l_decoder = TensorProdFactLayer((l_shp1, l_input_regions), dim_h=CFG['EMBEDDING_SIZE'], dim_r=CFG['REGION_SIZE'],
dim_w=len(vocab), nonlinearity=softmax, name='l_tensor', W_hr='skip', b_hr='skip')
elif CFG['DISSECT'] == 'rs':
import pdb
pdb.set_trace() # XXX BREAKPOINT
l_decoder = TensorProdFactLayer((l_shp1, l_input_regions), dim_h=CFG['EMBEDDING_SIZE'], dim_r=CFG['REGION_SIZE'],
dim_w=len(vocab), nonlinearity=softmax, name='l_tensor', W_rw='skip', b_rw='skip')
else:
if CFG.has_key('DENSITY_TEMPERING') and CFG['DENSITY_TEMPERING']:
print("TEMPERING")
l_gamma = DenseLayer(
l_shp1, num_units=1, name='l_gamma')
l_gamma_shp = ReshapeLayer(
l_gamma, ([0], [1], 1, 1))
from TProd3 import TensorTemperatureLayer
l_decoder = TensorTemperatureLayer((l_shp1, l_input_regions, l_gamma_shp), dim_h=CFG['EMBEDDING_SIZE'], dim_r=CFG['REGION_SIZE'], dim_w=len(
vocab), nonlinearity=lasagne.nonlinearities.softmax, name='l_tensor')
else:
l_decoder = TensorProdFactLayer((l_shp1, l_input_regions), dim_h=CFG['EMBEDDING_SIZE'], dim_r=CFG['REGION_SIZE'],
dim_w=len(vocab), nonlinearity=softmax, name='l_tensor')
if CFG['TENSOR_COND_WORD']:
from RNNTraining import get_Regions_cond_words
l_region = ExpressionLayer(
l_decoder, get_Regions_cond_words, output_shape='auto', name='l_region')
else:
l_region = ExpressionLayer(l_decoder, lambda X: X.sum(
3), output_shape='auto', name='l_region') # sum over
l_region = ReshapeLayer(
l_region, ([0], [2]), name='l_region')
l_out = ExpressionLayer(l_decoder, lambda X: X.sum(
2), output_shape='auto', name='l_out') # sum over regions
#}}}
elif CFG['MODE'] == 'tensor-reducedw':
# {{{2
from TProd3 import TensorProdFactLayer
# input: [h(batch,dimh),r(num_batch,r_dim,num_r)]
# output: [ h[0],r[2], dim_w ]
l_input_regions = InputLayer((CFG['BATCH_SIZE'], CFG['REGION_SIZE'], CFG['NUM_REGIONS']), name='l_input_regins')
if CFG.has_key('TENSOR_RECTIFY') and CFG['TENSOR_RECTIFY']:
l_tensor = TensorProdFactLayer((l_dropout_output, l_input_regions), dim_h=CFG['EMBEDDING_SIZE'], dim_r=CFG[
'REGION_SIZE'], dim_w=CFG['EMBEDDING_WORDS'], nonlinearity=lasagne.nonlinearities.rectify, name='l_tensor')
else:
l_tensor = TensorProdFactLayer((l_dropout_output, l_input_regions), dim_h=CFG['EMBEDDING_SIZE'], dim_r=CFG[
'REGION_SIZE'], dim_w=CFG['EMBEDDING_WORDS'], nonlinearity=lasagne.nonlinearities.identity, name='l_tensor')
# softmax does not accept non-flat layers, then flatten->softmax->reshape
l_flatten = ReshapeLayer(
l_decoder, (CFG['BATCH_SIZE'] * 1 * CFG['NUM_REGIONS'], CFG['EMBEDDING_WORDS']), name='l_flatten')
l_words = DenseLayer(l_flatten, num_units=len(vocab),
nonlinearity=lasagne.nonlinearities.identity, name='l_words')
l_reshape = ReshapeLayer(
l_words, (CFG['BATCH_SIZE'] * 1, CFG['NUM_REGIONS'] * len(vocab)), name='l_reshape')
l_softmax = lasagne.layers.NonlinearityLayer(
l_reshape, nonlinearity=lasagne.nonlinearities.softmax, name='l_softmax')
l_reshape1 = ReshapeLayer(
l_softmax, (CFG['BATCH_SIZE'], 1, CFG['NUM_REGIONS'], len(vocab)), name='l_reshape1')
l_out = ExpressionLayer(l_reshape1, lambda X: X.sum(
2), output_shape='auto', name='l_out') # sum over regions
# }}}
elif CFG['MODE'] == 'tensor-removedWrw':
from TProd3 import TensorProdFact2Layer
# input: [h(batch,dimh),r(num_batch,r_dim,num_r)]
# output: [ h[0],r[2], dim_w ]
l_input_regions = InputLayer((CFG['BATCH_SIZE'], CFG['REGION_SIZE'], CFG['NUM_REGIONS']), name='l_input_regins')
l_decoder = TensorProdFact2Layer((l_dropout_output, l_input_regions), dim_h=CFG['EMBEDDING_SIZE'], dim_r=CFG['REGION_SIZE'], dim_w=len(
vocab), nonlinearity=lasagne.nonlinearities.softmax, name='l_decoder')
l_out = ExpressionLayer(l_decoder, lambda X: X.sum( 2), output_shape='auto', name='l_out') # sum over regions
net_dictionnary = {'loc1': l_loc1, 'input_loc': l_input_loc, 'sel_region2': l_sel_region2,
'loc': l_loc, 'conv': l_conv, 'prev': l_prev_gru, 'input': l_cell_input,
'gru': l_gru, 'sent': l_input_sentence, 'img': l_input_img, 'img2': l_input_img2,
'reg_feedback2': l_region_feedback, 'reg_feedback': l_region, 'reg': l_input_reg,
'out_reg': l_out_reg, 'out': l_out, 'cnn': l_cnn_embedding, 'sent_emb': l_sentence_embedding,
'decoder': l_decoder, 'boxes': l_boxes, 'weighted_regions_prev': l_weighted_region_prev,
'weighted_regions': l_weighted_region}
return net_dictionnary