def alexnet_layers():
conv_weight_filler = layers.Filler(type="gaussian", std=0.01)
bias_filler0 = layers.Filler(type="constant", value=0.0)
bias_filler1 = layers.Filler(type="constant", value=1.0)
conv_lr_mults = [1.0, 2.0]
conv_decay_mults = [1.0, 0.0]
alexnet_layers = [
layers.Convolution(name="conv1", bottoms=["data"], param_lr_mults=conv_lr_mults,
param_decay_mults=conv_decay_mults, kernel_size=11,
stride=4, weight_filler=conv_weight_filler, bias_filler=bias_filler0, num_output=96),
layers.ReLU(name="relu1", bottoms=["conv1"], tops=["conv1"]),
layers.Pooling(name="pool1", bottoms=["conv1"], kernel_size=3, stride=2),
layers.LRN(name="norm1", bottoms=["pool1"], local_size=5, alpha=0.0001, beta=0.75),
layers.Convolution(name="conv2", bottoms=["norm1"], param_lr_mults=conv_lr_mults,
param_decay_mults=conv_decay_mults, kernel_size=5,
pad=2, group=2, weight_filler=conv_weight_filler, bias_filler=bias_filler1, num_output=256),
layers.ReLU(name="relu2", bottoms=["conv2"], tops=["conv2"]),
layers.Pooling(name="pool2", bottoms=["conv2"], kernel_size=3, stride=2),
layers.LRN(name="norm2", bottoms=["pool2"], local_size=5, alpha=0.0001, beta=0.75),
layers.Convolution(name="conv3", bottoms=["norm2"], param_lr_mults=conv_lr_mults,
param_decay_mults=conv_decay_mults, kernel_size=3,
pad=1, weight_filler=conv_weight_filler, bias_filler=bias_filler0, num_output=384),
layers.ReLU(name="relu3", bottoms=["conv3"], tops=["conv3"]),
layers.Convolution(name="conv4", bottoms=["conv3"], param_lr_mults=conv_lr_mults,
param_decay_mults=conv_decay_mults, kernel_size=3,
pad=1, group=2, weight_filler=conv_weight_filler, bias_filler=bias_filler1, num_output=384),
layers.ReLU(name="relu4", bottoms=["conv4"], tops=["conv4"]),
layers.Convolution(name="conv5", bottoms=["conv4"], param_lr_mults=conv_lr_mults,
param_decay_mults=conv_decay_mults, kernel_size=3,
pad=1, group=2, weight_filler=conv_weight_filler, bias_filler=bias_filler1, num_output=256),
layers.ReLU(name="relu5", bottoms=["conv5"], tops=["conv5"]),
layers.Pooling(name="pool5", bottoms=["conv5"], kernel_size=3, stride=2),
layers.InnerProduct(name="fc6", bottoms=["pool5"], param_lr_mults=conv_lr_mults,
param_decay_mults=conv_decay_mults,
weight_filler=layers.Filler(type="gaussian", std=0.005),
bias_filler=bias_filler1, num_output=4096),
layers.ReLU(name="relu6", bottoms=["fc6"], tops=["fc6"]),
layers.Dropout(name="drop6", bottoms=["fc6"], tops=["fc6"], dropout_ratio=0.5, phase='TRAIN'),
layers.InnerProduct(name="fc7", bottoms=["fc6"], param_lr_mults=conv_lr_mults,
param_decay_mults=conv_decay_mults,
weight_filler=layers.Filler(type="gaussian", std=0.005),
bias_filler=bias_filler1, num_output=4096),
layers.ReLU(name="relu7", bottoms=["fc7"], tops=["fc7"]),
layers.Dropout(name="drop7", bottoms=["fc7"], tops=["fc7"], dropout_ratio=0.5, phase='TRAIN'),
layers.InnerProduct(name="fc8", bottoms=["fc7"], param_lr_mults=[1.0, 2.0],
param_decay_mults=conv_decay_mults,
weight_filler=layers.Filler(type="gaussian", std=0.01),
bias_filler=bias_filler0, num_output=1000),
layers.SoftmaxWithLoss(name="loss", bottoms=["fc8", "label"]),
]
return alexnet_layers
def evaluate_forward(net):
length = 20
net.forward_layer(layers.NumpyData(name='prev_hidden',
data=np.zeros((1, hyper['mem_cells'], 1, 1))))
net.forward_layer(layers.NumpyData(name='prev_mem',
data=np.zeros((1, hyper['mem_cells'], 1, 1))))
filler = layers.Filler(type='uniform', min=-hyper['init_range'], max=hyper['init_range'])
accum = np.array([0.])
predictions = []
for step in range(length):
value = 0.5
net.forward_layer(layers.NumpyData(name='value',
data=np.array(value).reshape((1, 1, 1, 1))))
accum += value
prev_hidden = 'prev_hidden'
prev_mem = 'prev_mem'
net.forward_layer(layers.Concat(name='lstm_concat', bottoms=[prev_hidden, 'value']))
net.forward_layer(layers.Lstm(name='lstm', bottoms=['lstm_concat', prev_mem],
param_names=['input_value', 'input_gate', 'forget_gate', 'output_gate'],
weight_filler=filler,
tops=['next_hidden', 'next_mem'], num_cells=hyper['mem_cells']))
net.forward_layer(layers.InnerProduct(name='ip', bottoms=['next_hidden'],
num_output=1))
predictions.append(float(net.tops['ip'].data.flatten()[0]))
# set up for next prediction by copying LSTM outputs back to inputs
net.tops['prev_hidden'].data_tensor.copy_from(net.tops['next_hidden'].data_tensor)
net.tops['prev_mem'].data_tensor.copy_from(net.tops['next_mem'].data_tensor)
net.reset_forward()
return predictions
def forward(net, hyper, sentence_batches):
batch = next(sentence_batches)
#sentence_batch = np.array(pad_batch([x['body'] for x in batch], hyper))
sentence_batch = np.array(pad_batch(batch, hyper))
length = min(sentence_batch.shape[1], 100)
assert length > 0
filler = layers.Filler(type='uniform', max=hyper['init_range'],
min=(-hyper['init_range']))
net.forward_layer(layers.NumpyData(name='lstm_seed',
data=np.zeros((hyper['batch_size'], hyper['mem_cells'], 1, 1))))
net.forward_layer(layers.NumpyData(name='label',
data=np.zeros((hyper['batch_size'] * length, 1, 1, 1))))
loss = []
for step in range(length):
net.forward_layer(layers.DummyData(name=('word%d' % step),
shape=[hyper['batch_size'], 1, 1, 1]))
if step == 0:
prev_hidden = 'lstm_seed'
prev_mem = 'lstm_seed'
word = np.zeros(sentence_batch[:, 0].shape)
else:
prev_hidden = 'lstm%d_hidden' % (step - 1)
prev_mem = 'lstm%d_mem' % (step - 1)
word = sentence_batch[:, step - 1]
net.tops['word%d' % step].data[:,0,0,0] = word
net.forward_layer(layers.Wordvec(name=('wordvec%d' % step),
bottoms=['word%d' % step],
dimension=hyper['mem_cells'], vocab_size=hyper['vocab_size'],
param_names=['wordvec_param'], weight_filler=filler))
net.forward_layer(layers.Concat(name='lstm_concat%d' % step,
bottoms=[prev_hidden, 'wordvec%d' % step]))
net.forward_layer(layers.Lstm(name='lstm%d' % step,
bottoms=['lstm_concat%d' % step, prev_mem],
param_names=['lstm_input_value', 'lstm_input_gate',
'lstm_forget_gate', 'lstm_output_gate'],
tops=['lstm%d_hidden' % step, 'lstm%d_mem' % step],
num_cells=hyper['mem_cells'], weight_filler=filler))
net.forward_layer(layers.Dropout(name='dropout%d' % step,
bottoms=['lstm%d_hidden' % step], dropout_ratio=0.16))
label = np.reshape(sentence_batch[:, step], (hyper['batch_size'], 1, 1, 1))
net.forward_layer(layers.NumpyData(name='label%d' % step,
data=label))
net.forward_layer(layers.InnerProduct(name='ip%d' % step, bottoms=['dropout%d' % step],
param_names=['softmax_ip_weights', 'softmax_ip_bias'],
num_output=hyper['vocab_size'], weight_filler=filler))
loss.append(net.forward_layer(layers.SoftmaxWithLoss(name='softmax_loss%d' % step,
ignore_label=hyper['zero_symbol'], bottoms=['ip%d' % step, 'label%d' % step])))
return np.mean(loss)
def forward(net):
length = random.randrange(5, 15)
# initialize all weights in [-0.1, 0.1]
filler = layers.Filler(type='uniform', min=-hyper['init_range'],
max=hyper['init_range'])
# initialize the LSTM memory with all 0's
net.forward_layer(layers.NumpyData(name='lstm_seed',
data=np.zeros((hyper['batch_size'], hyper['mem_cells'], 1, 1))))
accum = np.zeros((hyper['batch_size'],))
# Begin recurrence through 5 - 15 inputs
for step in range(length):
# Set up the value blob
net.forward_layer(layers.DummyData(name='value%d' % step,
shape=[hyper['batch_size'], 1, 1, 1]))
value = np.array([random.random() for _ in range(hyper['batch_size'])])
accum += value
# Set data of value blob to contain a batch of random numbers
net.tops['value%d' % step].data[:, 0, 0, 0] = value
if step == 0:
prev_hidden = 'lstm_seed'
prev_mem = 'lstm_seed'
else:
prev_hidden = 'lstm%d_hidden' % (step - 1)
prev_mem = 'lstm%d_mem' % (step - 1)
# Concatenate the hidden output with the next input value
net.forward_layer(layers.Concat(name='lstm_concat%d' % step,
bottoms=[prev_hidden, 'value%d' % step]))
# Run the LSTM for one more step
net.forward_layer(layers.Lstm(name='lstm%d' % step,
bottoms=['lstm_concat%d' % step, prev_mem],
param_names=['input_value', 'input_gate', 'forget_gate', 'output_gate'],
tops=['lstm%d_hidden' % step, 'lstm%d_mem' % step],
num_cells=hyper['mem_cells'], weight_filler=filler))
# Add a fully connected layer with a bottom blob set to be the last used LSTM cell
# Note that the network structure is now a function of the data
net.forward_layer(layers.InnerProduct(name='ip',
bottoms=['lstm%d_hidden' % (length - 1)],
num_output=1, weight_filler=filler))
# Add a label for the sum of the inputs
net.forward_layer(layers.NumpyData(name='label',
data=np.reshape(accum, (hyper['batch_size'], 1, 1, 1))))
# Compute the Euclidean loss between the preiction and label, used for backprop
loss = net.forward_layer(layers.EuclideanLoss(name='euclidean',
bottoms=['ip', 'label']))
return loss
def eval_forward(net, hyper):
output_words = []
filler = layers.Filler(type='uniform', max=hyper['init_range'],
min=(-hyper['init_range']))
net.forward_layer(layers.NumpyData(name='lstm_hidden_prev',
data=np.zeros((1, hyper['mem_cells'], 1, 1))))
net.forward_layer(layers.NumpyData(name='lstm_mem_prev',
data=np.zeros((1, hyper['mem_cells'], 1, 1))))
length = hyper['length']
for step in range(length):
net.forward_layer(layers.NumpyData(name=('word'),
data=np.zeros((1, 1, 1, 1))))
prev_hidden = 'lstm_hidden_prev'
prev_mem = 'lstm_mem_prev'
word = np.zeros((1, 1, 1, 1))
if step == 0:
#output = ord('.')
output = vocab[' ']
else:
output = softmax_choice(net.tops['softmax'].data)
output_words.append(output)
net.tops['word'].data[0,0,0,0] = output
net.forward_layer(layers.Wordvec(name=('wordvec'),
bottoms=['word'],
dimension=hyper['mem_cells'], vocab_size=hyper['vocab_size'],
param_names=['wordvec_param'], weight_filler=filler))
net.forward_layer(layers.Concat(name='lstm_concat',
bottoms=[prev_hidden, 'wordvec']))
net.forward_layer(layers.Lstm(name='lstm',
bottoms=['lstm_concat', prev_mem],
param_names=['lstm_input_value', 'lstm_input_gate',
'lstm_forget_gate', 'lstm_output_gate'],
tops=['lstm_hidden_next', 'lstm_mem_next'],
num_cells=hyper['mem_cells'], weight_filler=filler))
net.forward_layer(layers.Dropout(name='dropout',
bottoms=['lstm_hidden_next'], dropout_ratio=0.16))
net.forward_layer(layers.InnerProduct(name='ip', bottoms=['dropout'],
param_names=['softmax_ip_weights', 'softmax_ip_bias'],
num_output=hyper['vocab_size'], weight_filler=filler))
net.tops['ip'].data[:] *= hyper['i_temperature']
net.forward_layer(layers.Softmax(name='softmax',
ignore_label=hyper['zero_symbol'], bottoms=['ip']))
net.tops['lstm_hidden_prev'].data_tensor.copy_from(net.tops['lstm_hidden_next'].data_tensor)
net.tops['lstm_mem_prev'].data_tensor.copy_from(net.tops['lstm_mem_next'].data_tensor)
net.reset_forward()
print ''.join([ivocab[x].encode('utf8') for x in output_words])
return 0.
def forward(net, sentence_batches):
source_batch, target_batch = next(sentence_batches)
filler = layers.Filler(type='uniform',
max=hyper['init_range'],
min=(-hyper['init_range']))
net.forward_layer(
layers.NumpyData(name='source_lstm_seed',
data=np.zeros(
(hyper['batch_size'], hyper['mem_cells'], 1, 1))))
hidden_bottoms = ['source_lstm_seed']
mem_bottoms = ['source_lstm_seed']
lengths = [
min(len([1 for token in x
if token != hyper['pad_symbol']]), hyper['max_len'])
for x in source_batch
]
for step in range(source_batch.shape[1]):
net.forward_layer(
layers.DummyData(name=('source_word%d' % step),
shape=[hyper['batch_size'], 1, 1, 1]))
if step == 0:
prev_hidden = 'source_lstm_seed'
prev_mem = 'source_lstm_seed'
else:
prev_hidden = 'source_lstm%d_hidden' % (step - 1)
prev_mem = 'source_lstm%d_mem' % (step - 1)
next_hidden = 'source_lstm%d_hidden' % (step)
next_mem = 'source_lstm%d_mem' % (step)
hidden_bottoms.append(next_hidden)
mem_bottoms.append(next_mem)
word = source_batch[:, step]
net.tops['source_word%d' % step].data[:, 0, 0, 0] = word
wordvec = layers.Wordvec(name=('source_wordvec%d' % step),
bottoms=['source_word%d' % step],
dimension=hyper['mem_cells'],
vocab_size=hyper['vocab_size'],
param_names=['source_wordvec_param'],
weight_filler=filler)
concat = layers.Concat(
name='source_lstm_concat%d' % step,
bottoms=[prev_hidden, 'source_wordvec%d' % step])
lstm = layers.Lstm(
name='source_lstm%d' % step,
bottoms=['source_lstm_concat%d' % step, prev_mem],
param_names=[
'source_lstm_input_value', 'source_lstm_input_gate',
'source_lstm_forget_gate', 'source_lstm_output_gate'
],
tops=['source_lstm%d_hidden' % step,
'source_lstm%d_mem' % step],
num_cells=hyper['mem_cells'],
weight_filler=filler)
net.forward_layer(wordvec)
net.forward_layer(concat)
net.forward_layer(lstm)
net.forward_layer(
layers.CapSequence(name='hidden_seed',
sequence_lengths=lengths,
bottoms=hidden_bottoms))
net.forward_layer(
layers.CapSequence(name='mem_seed',
sequence_lengths=lengths,
bottoms=mem_bottoms))
loss = []
for step in range(target_batch.shape[1]):
if step == 0:
prev_hidden = 'hidden_seed'
prev_mem = 'mem_seed'
word = np.zeros(target_batch[:, 0].shape)
else:
prev_hidden = 'lstm%d_hidden' % (step - 1)
prev_mem = 'lstm%d_mem' % (step - 1)
word = target_batch[:, step - 1]
word = layers.NumpyData(name=('word%d' % step),
data=np.reshape(
word, (hyper['batch_size'], 1, 1, 1)))
wordvec = layers.Wordvec(name=('wordvec%d' % step),
bottoms=['word%d' % step],
dimension=hyper['mem_cells'],
vocab_size=hyper['vocab_size'],
param_names=['source_wordvec_param'],
weight_filler=filler)
concat = layers.Concat(name='lstm_concat%d' % step,
bottoms=[prev_hidden,
'wordvec%d' % step])
lstm = layers.Lstm(name='lstm%d' % step,
bottoms=['lstm_concat%d' % step, prev_mem],
param_names=[
'lstm_input_value', 'lstm_input_gate',
'lstm_forget_gate', 'lstm_output_gate'
],
tops=['lstm%d_hidden' % step,
'lstm%d_mem' % step],
num_cells=hyper['mem_cells'],
weight_filler=filler)
dropout = layers.Dropout(name='dropout%d' % step,
bottoms=['lstm%d_hidden' % step],
dropout_ratio=0.16)
net.forward_layer(word)
net.forward_layer(wordvec)
net.forward_layer(concat)
net.forward_layer(lstm)
net.forward_layer(dropout)
net.forward_layer(
layers.NumpyData(name='label%d' % step,
data=np.reshape(target_batch[:, step],
(hyper['batch_size'], 1, 1, 1))))
net.forward_layer(
layers.InnerProduct(name='ip%d' % step,
bottoms=['dropout%d' % step],
param_names=['ip_weight', 'ip_bias'],
num_output=hyper['vocab_size'],
weight_filler=filler))
loss.append(
net.forward_layer(
layers.SoftmaxWithLoss(
name='softmax_loss%d' % step,
ignore_label=hyper['pad_symbol'],
bottoms=['ip%d' % step, 'label%d' % step])))
loss.append(
net.forward_layer(
layers.Softmax(name='softmax%d' % step,
ignore_label=hyper['pad_symbol'],
bottoms=['ip%d' % step])))
return np.mean(loss)
def googlenet_layers():
weight_filler = layers.Filler(type="xavier")
bias_filler = layers.Filler(type="constant", value=0.2)
conv_lr_mults = [1.0, 2.0]
conv_decay_mults = [1.0, 0.0]
googlenet_layers = [
layers.Convolution(name="conv1/7x7_s2", bottoms=["data"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=7, stride=2, pad=3, weight_filler=weight_filler, bias_filler=bias_filler, num_output=64),
layers.ReLU(name="conv1/relu_7x7", bottoms=["conv1/7x7_s2"], tops=["conv1/7x7_s2"]),
layers.Pooling(name="pool1/3x3_s2", bottoms=["conv1/7x7_s2"], kernel_size=3, stride=2),
layers.LRN(name="pool1/norm1", bottoms=["pool1/3x3_s2"], local_size=5, alpha=0.0001, beta=0.75),
layers.Convolution(name="conv2/3x3_reduce", bottoms=["pool1/norm1"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=1, weight_filler=weight_filler, bias_filler=bias_filler, num_output=64),
layers.ReLU(name="conv2/relu_3x3_reduce", bottoms=["conv2/3x3_reduce"], tops=["conv2/3x3_reduce"]),
layers.Convolution(name="conv2/3x3", bottoms=["conv2/3x3_reduce"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=3, pad=1, weight_filler=weight_filler, bias_filler=bias_filler, num_output=192),
layers.ReLU(name="conv2/relu_3x3", bottoms=["conv2/3x3"], tops=["conv2/3x3"]),
layers.LRN(name="conv2/norm2", bottoms=["conv2/3x3"], local_size=5, alpha=0.0001, beta=0.75),
layers.Pooling(name="pool2/3x3_s2", bottoms=["conv2/norm2"], kernel_size=3, stride=2),
layers.Convolution(name="inception_3a/1x1", bottoms=["pool2/3x3_s2"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=1, weight_filler=weight_filler, bias_filler=bias_filler, num_output=64),
layers.ReLU(name="inception_3a/relu_1x1", bottoms=["inception_3a/1x1"], tops=["inception_3a/1x1"]),
layers.Convolution(name="inception_3a/3x3_reduce", bottoms=["pool2/3x3_s2"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=1, weight_filler=weight_filler, bias_filler=bias_filler, num_output=96),
layers.ReLU(name="inception_3a/relu_3x3_reduce", bottoms=["inception_3a/3x3_reduce"], tops=["inception_3a/3x3_reduce"]),
layers.Convolution(name="inception_3a/3x3", bottoms=["inception_3a/3x3_reduce"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=3, pad=1, weight_filler=weight_filler, bias_filler=bias_filler, num_output=128),
layers.ReLU(name="inception_3a/relu_3x3", bottoms=["inception_3a/3x3"], tops=["inception_3a/3x3"]),
layers.Convolution(name="inception_3a/5x5_reduce", bottoms=["pool2/3x3_s2"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=1, weight_filler=weight_filler, bias_filler=bias_filler, num_output=16),
layers.ReLU(name="inception_3a/relu_5x5_reduce", bottoms=["inception_3a/5x5_reduce"], tops=["inception_3a/5x5_reduce"]),
layers.Convolution(name="inception_3a/5x5", bottoms=["inception_3a/5x5_reduce"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=5, pad=2, weight_filler=weight_filler, bias_filler=bias_filler, num_output=32),
layers.ReLU(name="inception_3a/relu_5x5", bottoms=["inception_3a/5x5"], tops=["inception_3a/5x5"]),
layers.Pooling(name="inception_3a/pool", bottoms=["pool2/3x3_s2"], kernel_size=3, stride=1, pad=1),
layers.Convolution(name="inception_3a/pool_proj", bottoms=["inception_3a/pool"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=1, weight_filler=weight_filler, bias_filler=bias_filler, num_output=32),
layers.ReLU(name="inception_3a/relu_pool_proj", bottoms=["inception_3a/pool_proj"], tops=["inception_3a/pool_proj"]),
layers.Concat(name="inception_3a/output", bottoms=["inception_3a/1x1", "inception_3a/3x3", "inception_3a/5x5", "inception_3a/pool_proj"]),
layers.Convolution(name="inception_3b/1x1", bottoms=["inception_3a/output"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=1, weight_filler=weight_filler, bias_filler=bias_filler, num_output=128),
layers.ReLU(name="inception_3b/relu_1x1", bottoms=["inception_3b/1x1"], tops=["inception_3b/1x1"]),
layers.Convolution(name="inception_3b/3x3_reduce", bottoms=["inception_3a/output"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=1, weight_filler=weight_filler, bias_filler=bias_filler, num_output=128),
layers.ReLU(name="inception_3b/relu_3x3_reduce", bottoms=["inception_3b/3x3_reduce"], tops=["inception_3b/3x3_reduce"]),
layers.Convolution(name="inception_3b/3x3", bottoms=["inception_3b/3x3_reduce"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=3, pad=1, weight_filler=weight_filler, bias_filler=bias_filler, num_output=192),
layers.ReLU(name="inception_3b/relu_3x3", bottoms=["inception_3b/3x3"], tops=["inception_3b/3x3"]),
layers.Convolution(name="inception_3b/5x5_reduce", bottoms=["inception_3a/output"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=1, weight_filler=weight_filler, bias_filler=bias_filler, num_output=32),
layers.ReLU(name="inception_3b/relu_5x5_reduce", bottoms=["inception_3b/5x5_reduce"], tops=["inception_3b/5x5_reduce"]),
layers.Convolution(name="inception_3b/5x5", bottoms=["inception_3b/5x5_reduce"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=5, pad=2, weight_filler=weight_filler, bias_filler=bias_filler, num_output=96),
layers.ReLU(name="inception_3b/relu_5x5", bottoms=["inception_3b/5x5"], tops=["inception_3b/5x5"]),
layers.Pooling(name="inception_3b/pool", bottoms=["inception_3a/output"], kernel_size=3, stride=1, pad=1),
layers.Convolution(name="inception_3b/pool_proj", bottoms=["inception_3b/pool"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=1, weight_filler=weight_filler, bias_filler=bias_filler, num_output=64),
layers.ReLU(name="inception_3b/relu_pool_proj", bottoms=["inception_3b/pool_proj"], tops=["inception_3b/pool_proj"]),
layers.Concat(name="inception_3b/output", bottoms=["inception_3b/1x1", "inception_3b/3x3", "inception_3b/5x5", "inception_3b/pool_proj"]),
layers.Pooling(name="pool3/3x3_s2", bottoms=["inception_3b/output"], kernel_size=3, stride=2),
layers.Convolution(name="inception_4a/1x1", bottoms=["pool3/3x3_s2"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=1, weight_filler=weight_filler, bias_filler=bias_filler, num_output=192),
layers.ReLU(name="inception_4a/relu_1x1", bottoms=["inception_4a/1x1"], tops=["inception_4a/1x1"]),
layers.Convolution(name="inception_4a/3x3_reduce", bottoms=["pool3/3x3_s2"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=1, weight_filler=weight_filler, bias_filler=bias_filler, num_output=96),
layers.ReLU(name="inception_4a/relu_3x3_reduce", bottoms=["inception_4a/3x3_reduce"], tops=["inception_4a/3x3_reduce"]),
layers.Convolution(name="inception_4a/3x3", bottoms=["inception_4a/3x3_reduce"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=3, pad=1, weight_filler=weight_filler, bias_filler=bias_filler, num_output=208),
layers.ReLU(name="inception_4a/relu_3x3", bottoms=["inception_4a/3x3"], tops=["inception_4a/3x3"]),
layers.Convolution(name="inception_4a/5x5_reduce", bottoms=["pool3/3x3_s2"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=1, weight_filler=weight_filler, bias_filler=bias_filler, num_output=16),
layers.ReLU(name="inception_4a/relu_5x5_reduce", bottoms=["inception_4a/5x5_reduce"], tops=["inception_4a/5x5_reduce"]),
layers.Convolution(name="inception_4a/5x5", bottoms=["inception_4a/5x5_reduce"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=5, pad=2, weight_filler=weight_filler, bias_filler=bias_filler, num_output=48),
layers.ReLU(name="inception_4a/relu_5x5", bottoms=["inception_4a/5x5"], tops=["inception_4a/5x5"]),
layers.Pooling(name="inception_4a/pool", bottoms=["pool3/3x3_s2"], kernel_size=3, stride=1, pad=1),
layers.Convolution(name="inception_4a/pool_proj", bottoms=["inception_4a/pool"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=1, weight_filler=weight_filler, bias_filler=bias_filler, num_output=64),
layers.ReLU(name="inception_4a/relu_pool_proj", bottoms=["inception_4a/pool_proj"], tops=["inception_4a/pool_proj"]),
layers.Concat(name="inception_4a/output", bottoms=["inception_4a/1x1", "inception_4a/3x3", "inception_4a/5x5", "inception_4a/pool_proj"]),
layers.Pooling(name="loss1/ave_pool", bottoms=["inception_4a/output"], kernel_size=5, stride=3),
layers.Convolution(name="loss1/conv", bottoms=["loss1/ave_pool"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=1, weight_filler=weight_filler, bias_filler=bias_filler, num_output=128),
layers.ReLU(name="loss1/relu_conv", bottoms=["loss1/conv"], tops=["loss1/conv"]),
layers.InnerProduct(name="loss1/fc", bottoms=["loss1/conv"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, weight_filler=weight_filler, bias_filler=bias_filler, num_output=1024),
layers.ReLU(name="loss1/relu_fc", bottoms=["loss1/fc"], tops=["loss1/fc"]),
layers.Dropout(name="loss1/drop_fc", bottoms=["loss1/fc"], tops=["loss1/fc"], dropout_ratio=0.7, phase='TRAIN'),
layers.InnerProduct(name="loss1/classifier", bottoms=["loss1/fc"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, weight_filler=weight_filler, bias_filler=layers.Filler(type="constant", value=0.0), num_output=1000),
layers.SoftmaxWithLoss(name="loss1/loss", bottoms=["loss1/classifier", "label"], tops=["loss1/loss1"], loss_weight=0.3),
layers.Convolution(name="inception_4b/1x1", bottoms=["inception_4a/output"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=1, weight_filler=weight_filler, bias_filler=bias_filler, num_output=160),
layers.ReLU(name="inception_4b/relu_1x1", bottoms=["inception_4b/1x1"], tops=["inception_4b/1x1"]),
layers.Convolution(name="inception_4b/3x3_reduce", bottoms=["inception_4a/output"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=1, weight_filler=weight_filler, bias_filler=bias_filler, num_output=112),
layers.ReLU(name="inception_4b/relu_3x3_reduce", bottoms=["inception_4b/3x3_reduce"], tops=["inception_4b/3x3_reduce"]),
layers.Convolution(name="inception_4b/3x3", bottoms=["inception_4b/3x3_reduce"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=3, pad=1, weight_filler=weight_filler, bias_filler=bias_filler, num_output=224),
layers.ReLU(name="inception_4b/relu_3x3", bottoms=["inception_4b/3x3"], tops=["inception_4b/3x3"]),
layers.Convolution(name="inception_4b/5x5_reduce", bottoms=["inception_4a/output"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=1, weight_filler=weight_filler, bias_filler=bias_filler, num_output=24),
layers.ReLU(name="inception_4b/relu_5x5_reduce", bottoms=["inception_4b/5x5_reduce"], tops=["inception_4b/5x5_reduce"]),
layers.Convolution(name="inception_4b/5x5", bottoms=["inception_4b/5x5_reduce"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=5, pad=2, weight_filler=weight_filler, bias_filler=bias_filler, num_output=64),
layers.ReLU(name="inception_4b/relu_5x5", bottoms=["inception_4b/5x5"], tops=["inception_4b/5x5"]),
layers.Pooling(name="inception_4b/pool", bottoms=["inception_4a/output"], kernel_size=3, stride=1, pad=1),
layers.Convolution(name="inception_4b/pool_proj", bottoms=["inception_4b/pool"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=1, weight_filler=weight_filler, bias_filler=bias_filler, num_output=64),
layers.ReLU(name="inception_4b/relu_pool_proj", bottoms=["inception_4b/pool_proj"], tops=["inception_4b/pool_proj"]),
layers.Concat(name="inception_4b/output", bottoms=["inception_4b/1x1", "inception_4b/3x3", "inception_4b/5x5", "inception_4b/pool_proj"]),
layers.Convolution(name="inception_4c/1x1", bottoms=["inception_4b/output"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=1, weight_filler=weight_filler, bias_filler=bias_filler, num_output=128),
layers.ReLU(name="inception_4c/relu_1x1", bottoms=["inception_4c/1x1"], tops=["inception_4c/1x1"]),
layers.Convolution(name="inception_4c/3x3_reduce", bottoms=["inception_4b/output"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=1, weight_filler=weight_filler, bias_filler=bias_filler, num_output=128),
layers.ReLU(name="inception_4c/relu_3x3_reduce", bottoms=["inception_4c/3x3_reduce"], tops=["inception_4c/3x3_reduce"]),
layers.Convolution(name="inception_4c/3x3", bottoms=["inception_4c/3x3_reduce"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=3, pad=1, weight_filler=weight_filler, bias_filler=bias_filler, num_output=256),
layers.ReLU(name="inception_4c/relu_3x3", bottoms=["inception_4c/3x3"], tops=["inception_4c/3x3"]),
layers.Convolution(name="inception_4c/5x5_reduce", bottoms=["inception_4b/output"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=1, weight_filler=weight_filler, bias_filler=bias_filler, num_output=24),
layers.ReLU(name="inception_4c/relu_5x5_reduce", bottoms=["inception_4c/5x5_reduce"], tops=["inception_4c/5x5_reduce"]),
layers.Convolution(name="inception_4c/5x5", bottoms=["inception_4c/5x5_reduce"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=5, pad=2, weight_filler=weight_filler, bias_filler=bias_filler, num_output=64),
layers.ReLU(name="inception_4c/relu_5x5", bottoms=["inception_4c/5x5"], tops=["inception_4c/5x5"]),
layers.Pooling(name="inception_4c/pool", bottoms=["inception_4b/output"], kernel_size=3, stride=1, pad=1),
layers.Convolution(name="inception_4c/pool_proj", bottoms=["inception_4c/pool"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=1, weight_filler=weight_filler, bias_filler=bias_filler, num_output=64),
layers.ReLU(name="inception_4c/relu_pool_proj", bottoms=["inception_4c/pool_proj"], tops=["inception_4c/pool_proj"]),
layers.Concat(name="inception_4c/output", bottoms=["inception_4c/1x1", "inception_4c/3x3", "inception_4c/5x5", "inception_4c/pool_proj"]),
layers.Convolution(name="inception_4d/1x1", bottoms=["inception_4c/output"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=1, weight_filler=weight_filler, bias_filler=bias_filler, num_output=112),
layers.ReLU(name="inception_4d/relu_1x1", bottoms=["inception_4d/1x1"], tops=["inception_4d/1x1"]),
layers.Convolution(name="inception_4d/3x3_reduce", bottoms=["inception_4c/output"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=1, weight_filler=weight_filler, bias_filler=bias_filler, num_output=144),
layers.ReLU(name="inception_4d/relu_3x3_reduce", bottoms=["inception_4d/3x3_reduce"], tops=["inception_4d/3x3_reduce"]),
layers.Convolution(name="inception_4d/3x3", bottoms=["inception_4d/3x3_reduce"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=3, pad=1, weight_filler=weight_filler, bias_filler=bias_filler, num_output=288),
layers.ReLU(name="inception_4d/relu_3x3", bottoms=["inception_4d/3x3"], tops=["inception_4d/3x3"]),
layers.Convolution(name="inception_4d/5x5_reduce", bottoms=["inception_4c/output"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=1, weight_filler=weight_filler, bias_filler=bias_filler, num_output=32),
layers.ReLU(name="inception_4d/relu_5x5_reduce", bottoms=["inception_4d/5x5_reduce"], tops=["inception_4d/5x5_reduce"]),
layers.Convolution(name="inception_4d/5x5", bottoms=["inception_4d/5x5_reduce"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=5, pad=2, weight_filler=weight_filler, bias_filler=bias_filler, num_output=64),
layers.ReLU(name="inception_4d/relu_5x5", bottoms=["inception_4d/5x5"], tops=["inception_4d/5x5"]),
layers.Pooling(name="inception_4d/pool", bottoms=["inception_4c/output"], kernel_size=3, stride=1, pad=1),
layers.Convolution(name="inception_4d/pool_proj", bottoms=["inception_4d/pool"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=1, weight_filler=weight_filler, bias_filler=bias_filler, num_output=64),
layers.ReLU(name="inception_4d/relu_pool_proj", bottoms=["inception_4d/pool_proj"], tops=["inception_4d/pool_proj"]),
layers.Concat(name="inception_4d/output", bottoms=["inception_4d/1x1", "inception_4d/3x3", "inception_4d/5x5", "inception_4d/pool_proj"]),
layers.Pooling(name="loss2/ave_pool", bottoms=["inception_4d/output"], kernel_size=5, stride=3),
layers.Convolution(name="loss2/conv", bottoms=["loss2/ave_pool"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=1, weight_filler=weight_filler, bias_filler=bias_filler, num_output=128),
layers.ReLU(name="loss2/relu_conv", bottoms=["loss2/conv"], tops=["loss2/conv"]),
layers.InnerProduct(name="loss2/fc", bottoms=["loss2/conv"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, weight_filler=weight_filler, bias_filler=bias_filler, num_output=1024),
layers.ReLU(name="loss2/relu_fc", bottoms=["loss2/fc"], tops=["loss2/fc"]),
layers.Dropout(name="loss2/drop_fc", bottoms=["loss2/fc"], tops=["loss2/fc"], dropout_ratio=0.7, phase='TRAIN'),
layers.InnerProduct(name="loss2/classifier", bottoms=["loss2/fc"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, weight_filler=weight_filler, bias_filler=layers.Filler(type="constant", value=0.0), num_output=1000),
layers.SoftmaxWithLoss(name="loss2/loss", bottoms=["loss2/classifier", "label"], tops=["loss2/loss1"], loss_weight=0.3),
layers.Convolution(name="inception_4e/1x1", bottoms=["inception_4d/output"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=1, weight_filler=weight_filler, bias_filler=bias_filler, num_output=256),
layers.ReLU(name="inception_4e/relu_1x1", bottoms=["inception_4e/1x1"], tops=["inception_4e/1x1"]),
layers.Convolution(name="inception_4e/3x3_reduce", bottoms=["inception_4d/output"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=1, weight_filler=weight_filler, bias_filler=bias_filler, num_output=160),
layers.ReLU(name="inception_4e/relu_3x3_reduce", bottoms=["inception_4e/3x3_reduce"], tops=["inception_4e/3x3_reduce"]),
layers.Convolution(name="inception_4e/3x3", bottoms=["inception_4e/3x3_reduce"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=3, pad=1, weight_filler=weight_filler, bias_filler=bias_filler, num_output=320),
layers.ReLU(name="inception_4e/relu_3x3", bottoms=["inception_4e/3x3"], tops=["inception_4e/3x3"]),
layers.Convolution(name="inception_4e/5x5_reduce", bottoms=["inception_4d/output"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=1, weight_filler=weight_filler, bias_filler=bias_filler, num_output=32),
layers.ReLU(name="inception_4e/relu_5x5_reduce", bottoms=["inception_4e/5x5_reduce"], tops=["inception_4e/5x5_reduce"]),
layers.Convolution(name="inception_4e/5x5", bottoms=["inception_4e/5x5_reduce"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=5, pad=2, weight_filler=weight_filler, bias_filler=bias_filler, num_output=128),
layers.ReLU(name="inception_4e/relu_5x5", bottoms=["inception_4e/5x5"], tops=["inception_4e/5x5"]),
layers.Pooling(name="inception_4e/pool", bottoms=["inception_4d/output"], kernel_size=3, stride=1, pad=1),
layers.Convolution(name="inception_4e/pool_proj", bottoms=["inception_4e/pool"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=1, weight_filler=weight_filler, bias_filler=bias_filler, num_output=128),
layers.ReLU(name="inception_4e/relu_pool_proj", bottoms=["inception_4e/pool_proj"], tops=["inception_4e/pool_proj"]),
layers.Concat(name="inception_4e/output", bottoms=["inception_4e/1x1", "inception_4e/3x3", "inception_4e/5x5", "inception_4e/pool_proj"]),
layers.Pooling(name="pool4/3x3_s2", bottoms=["inception_4e/output"], kernel_size=3, stride=2),
layers.Convolution(name="inception_5a/1x1", bottoms=["pool4/3x3_s2"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=1, weight_filler=weight_filler, bias_filler=bias_filler, num_output=256),
layers.ReLU(name="inception_5a/relu_1x1", bottoms=["inception_5a/1x1"], tops=["inception_5a/1x1"]),
layers.Convolution(name="inception_5a/3x3_reduce", bottoms=["pool4/3x3_s2"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=1, weight_filler=weight_filler, bias_filler=bias_filler, num_output=160),
layers.ReLU(name="inception_5a/relu_3x3_reduce", bottoms=["inception_5a/3x3_reduce"], tops=["inception_5a/3x3_reduce"]),
layers.Convolution(name="inception_5a/3x3", bottoms=["inception_5a/3x3_reduce"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=3, pad=1, weight_filler=weight_filler, bias_filler=bias_filler, num_output=320),
layers.ReLU(name="inception_5a/relu_3x3", bottoms=["inception_5a/3x3"], tops=["inception_5a/3x3"]),
layers.Convolution(name="inception_5a/5x5_reduce", bottoms=["pool4/3x3_s2"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=1, weight_filler=weight_filler, bias_filler=bias_filler, num_output=32),
layers.ReLU(name="inception_5a/relu_5x5_reduce", bottoms=["inception_5a/5x5_reduce"], tops=["inception_5a/5x5_reduce"]),
layers.Convolution(name="inception_5a/5x5", bottoms=["inception_5a/5x5_reduce"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=5, pad=2, weight_filler=weight_filler, bias_filler=bias_filler, num_output=128),
layers.ReLU(name="inception_5a/relu_5x5", bottoms=["inception_5a/5x5"], tops=["inception_5a/5x5"]),
layers.Pooling(name="inception_5a/pool", bottoms=["pool4/3x3_s2"], kernel_size=3, stride=1, pad=1),
layers.Convolution(name="inception_5a/pool_proj", bottoms=["inception_5a/pool"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=1, weight_filler=weight_filler, bias_filler=bias_filler, num_output=128),
layers.ReLU(name="inception_5a/relu_pool_proj", bottoms=["inception_5a/pool_proj"], tops=["inception_5a/pool_proj"]),
layers.Concat(name="inception_5a/output", bottoms=["inception_5a/1x1", "inception_5a/3x3", "inception_5a/5x5", "inception_5a/pool_proj"]),
layers.Convolution(name="inception_5b/1x1", bottoms=["inception_5a/output"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=1, weight_filler=weight_filler, bias_filler=bias_filler, num_output=384),
layers.ReLU(name="inception_5b/relu_1x1", bottoms=["inception_5b/1x1"], tops=["inception_5b/1x1"]),
layers.Convolution(name="inception_5b/3x3_reduce", bottoms=["inception_5a/output"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=1, weight_filler=weight_filler, bias_filler=bias_filler, num_output=192),
layers.ReLU(name="inception_5b/relu_3x3_reduce", bottoms=["inception_5b/3x3_reduce"], tops=["inception_5b/3x3_reduce"]),
layers.Convolution(name="inception_5b/3x3", bottoms=["inception_5b/3x3_reduce"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=3, pad=1, weight_filler=weight_filler, bias_filler=bias_filler, num_output=384),
layers.ReLU(name="inception_5b/relu_3x3", bottoms=["inception_5b/3x3"], tops=["inception_5b/3x3"]),
layers.Convolution(name="inception_5b/5x5_reduce", bottoms=["inception_5a/output"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=1, weight_filler=weight_filler, bias_filler=bias_filler, num_output=48),
layers.ReLU(name="inception_5b/relu_5x5_reduce", bottoms=["inception_5b/5x5_reduce"], tops=["inception_5b/5x5_reduce"]),
layers.Convolution(name="inception_5b/5x5", bottoms=["inception_5b/5x5_reduce"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=5, pad=2, weight_filler=weight_filler, bias_filler=bias_filler, num_output=128),
layers.ReLU(name="inception_5b/relu_5x5", bottoms=["inception_5b/5x5"], tops=["inception_5b/5x5"]),
layers.Pooling(name="inception_5b/pool", bottoms=["inception_5a/output"], kernel_size=3, stride=1, pad=1),
layers.Convolution(name="inception_5b/pool_proj", bottoms=["inception_5b/pool"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, kernel_size=1, weight_filler=weight_filler, bias_filler=bias_filler, num_output=128),
layers.ReLU(name="inception_5b/relu_pool_proj", bottoms=["inception_5b/pool_proj"], tops=["inception_5b/pool_proj"]),
layers.Concat(name="inception_5b/output", bottoms=["inception_5b/1x1", "inception_5b/3x3", "inception_5b/5x5", "inception_5b/pool_proj"]),
layers.Pooling(name="pool5/7x7_s1", bottoms=["inception_5b/output"], kernel_size=7, stride=1),
layers.Dropout(name="pool5/drop_7x7_s1", bottoms=["pool5/7x7_s1"], tops=["pool5/7x7_s1"], dropout_ratio=0.4, phase='TRAIN'),
layers.InnerProduct(name="loss3/classifier", bottoms=["pool5/7x7_s1"], param_lr_mults=conv_lr_mults, param_decay_mults=conv_decay_mults, weight_filler=weight_filler, bias_filler=layers.Filler(type="constant", value=0.0), num_output=1000),
layers.SoftmaxWithLoss(name="loss3/loss3", bottoms=["loss3/classifier", "label"], loss_weight=1.0),
]
return googlenet_layers