def cnn(X, y):
nb_classes = y.shape.as_list()[-1]
with N.args_scope(['Conv', dict(b_init=None, activation=K.linear)],
['BatchNorm', dict(activation=K.relu)]):
f = N.Sequence([
N.Dimshuffle(pattern=(0, 2, 3, 1)),
N.Conv(32, (3, 3), pad='same', stride=(1, 1)),
N.BatchNorm(),
N.Conv(32, (3, 3), pad='same', stride=(1, 1),
b_init=0, activation=K.relu),
N.Pool(pool_size=(2, 2), strides=None, mode='max'),
N.Dropout(level=0.25),
#
N.Conv(64, (3, 3), pad='same', stride=(1, 1)),
N.BatchNorm(),
N.Conv(64, (3, 3), pad='same', stride=(1, 1),
b_init=0., activation=K.relu),
N.Pool(pool_size=(2, 2), strides=None, mode='max'),
N.Dropout(level=0.25),
#
N.Flatten(outdim=2),
N.Dense(512, activation=K.relu),
N.Dropout(level=0.5),
N.Dense(nb_classes, activation=K.linear)
], debug=1)
logit = f(X)
prob = tf.nn.softmax(logit)
return {'logit': logit, 'prob': prob}
def gender(X, f, **kwargs):
nb_gender = kwargs.get('nb_gender', 4)
if f is None:
f = N.Sequence([
N.Dimshuffle(pattern=(0, 1, 2, 'x')),
N.Conv(num_filters=32,
filter_size=3,
strides=1,
b_init=None,
pad='valid'),
N.BatchNorm(activation=K.relu),
N.Pool(pool_size=2, mode='avg'),
N.Conv(num_filters=64,
filter_size=3,
strides=1,
b_init=None,
pad='valid'),
N.BatchNorm(activation=K.relu),
N.Pool(pool_size=2, mode='avg'),
N.Flatten(outdim=3),
N.Dense(num_units=512, b_init=None),
N.BatchNorm(axes=(0, 1)),
N.AutoRNN(num_units=128,
rnn_mode='gru',
num_layers=2,
input_mode='linear',
direction_mode='unidirectional'),
N.Flatten(outdim=2),
N.Dense(num_units=nb_gender, activation=K.softmax)
],
debug=True)
return f(X), f
def test_pool_depool(self):
X1 = K.placeholder(shape=(None, 12, 8, 25), name='X1')
X2 = K.placeholder(shape=(None, 12, 8, 25, 18), name='X2')
x1 = np.random.rand(13, 12, 8, 25)
x2 = np.random.rand(13, 12, 8, 25, 18)
prog = Progbar(target=2 * 2 * 2 * 3, print_report=True)
def check_shape(s1, s2):
self.assertEqual(tuple(s1),
tuple(s2),
msg="%s != %s" % (str(s1), str(s2)))
for pool_size in (2, 3):
for strides in (2, 3):
# strides > window_shape not supported due to inconsistency
# between CPU and GPU implementations
if pool_size < strides:
prog.add(1)
continue
for pad in ('valid', 'same'):
for transpose_mode in ('nn', 'pad_margin', 'repeat'):
# ====== print prog ====== #
prog['test'] = "Size:%d,Stride:%d,Pad:%s,T:%s" % \
(pool_size, strides, pad, transpose_mode)
prog.add(1)
# ====== check ops 4D ====== #
down = N.Pool(pool_size=pool_size,
strides=strides,
pad=pad,
mode='max',
transpose_mode=transpose_mode)
up = down.T
y1 = down(X1)
check_shape(
K.eval(y1, {
X1: x1
}).shape[1:],
y1.shape.as_list()[1:])
y2 = up(y1)
check_shape(K.eval(y2, {X1: x1}).shape, x1.shape)
# ====== check ops 5D ====== #
down = N.Pool(pool_size=pool_size,
strides=strides,
pad=pad,
mode='max',
transpose_mode=transpose_mode)
up = down.T
y1 = down(X2)
check_shape(
K.eval(y1, {
X2: x2
}).shape[1:], y1.shape[1:])
y2 = up(y1)
check_shape(K.eval(y2, {X2: x2}).shape, x2.shape)
def odin_net2():
"CNN"
f = N.Sequence([
N.Dimshuffle((0, 1, 2, 'x')),
N.Conv(12, (3, 3), strides=(1, 1), pad='same',
untie_biases=False,
W_init=random(3, 3, 1, 12),
activation=K.relu),
N.Pool(pool_size=(2, 2), strides=None, mode='max',
ignore_border=True),
N.Conv(16, (3, 3), strides=(1, 1), pad='same',
untie_biases=False,
W_init=random(3, 3, 12, 16),
activation=K.sigmoid),
N.Dimshuffle((0, 3, 1, 2))
])
return X1, f(X1)
X = K.placeholder(shape=(None,) + input_shape[1:], name='X_input')
y = K.placeholder(shape=(None,), name='y_input')
# ===========================================================================
# Create the network
# ===========================================================================
LATENT_DROPOUT = 0.3
if args.cnn:
with N.args_scope(([N.Conv, N.Dense], dict(b_init=None, activation=K.linear)),
(N.BatchNorm, dict(activation=tf.nn.elu)),
(N.Pool, dict(mode='max', pool_size=2))):
f_encoder = N.Sequence([
N.Dropout(level=0.5),
N.Dimshuffle((0, 2, 3, 1)) if is_cifar10 else N.Dimshuffle((0, 1, 2, 'x')),
N.Conv(num_filters=32, filter_size=3, pad='valid'),
N.Pool(),
N.BatchNorm(),
N.Conv(num_filters=64, filter_size=3, pad='same'),
N.BatchNorm(),
N.Conv(num_filters=64, filter_size=3, pad='valid'),
N.BatchNorm(activation=tf.nn.elu),
N.Pool(),
N.Flatten(outdim=2),
N.Dense(num_units=args.dim)
], debug=True, name='EncoderNetwork')
f_decoder = N.Sequence([
N.Dropout(level=LATENT_DROPOUT, noise_type='uniform'),
dtype='float32',
name='input%d' % i)
for i, shape in enumerate(as_tuple_of_shape(train.shape))
]
X = inputs[0]
y = inputs[1]
print("Inputs:", ctext(inputs, 'cyan'))
# ====== create the networks ====== #
with N.args_scope([('Conv', 'Dense'),
dict(b_init=None, activation=K.linear, pad='same')],
['BatchNorm', dict(activation=K.relu)]):
f = N.Sequence([
N.Dimshuffle(pattern=(0, 1, 2, 'x')),
N.Conv(num_filters=32, filter_size=(9, 7)),
N.BatchNorm(),
N.Pool(pool_size=(3, 2), strides=2),
N.Conv(num_filters=64, filter_size=(5, 3)),
N.BatchNorm(),
N.Pool(pool_size=(3, 1), strides=(2, 1), name='PoolOutput1'),
N.Conv(num_filters=64, filter_size=(5, 3)),
N.BatchNorm(),
N.Pool(pool_size=(3, 2), strides=(2, 2), name='PoolOutput2'),
N.Flatten(outdim=2),
N.Dense(512, name="LatentDense"),
N.BatchNorm(),
N.Dense(512),
N.BatchNorm(),
N.Dense(n_classes)
],
debug=1)
# ====== create outputs ====== #
X_learn = ds['X_train'][:].astype('float32') / 255.
y_learn = ds['y_train']
X_test = ds['X_test'][:].astype('float32') / 255.
y_test = ds['y_test']
# ===========================================================================
# Create network
# ===========================================================================
X = K.placeholder(shape=(None, ) + X_learn.shape[1:], name='X')
y_true = K.placeholder(shape=(None, ), name='y_true', dtype='int32')
f = N.Sequence([
N.Dimshuffle(pattern=(0, 2, 3, 1)),
N.Conv(32, (3, 3), pad='same', stride=(1, 1), activation=K.relu),
N.Conv(32, (3, 3), pad='same', stride=(1, 1), activation=K.relu),
N.Pool(pool_size=(2, 2), ignore_border=True, strides=None, mode='max'),
N.Dropout(level=0.25),
N.Conv(64, (3, 3), pad='same', stride=(1, 1), activation=K.relu),
N.Conv(64, (3, 3), pad='same', stride=(1, 1), activation=K.relu),
N.Pool(pool_size=(2, 2), ignore_border=True, strides=None, mode='max'),
N.Dropout(level=0.25),
N.Flatten(outdim=2),
N.Dense(512, activation=K.relu),
N.Dropout(level=0.5),
N.Dense(10, activation=K.softmax)
],
debug=True)
K.set_training(True)
y_train = f(X)
K.set_training(False)
y_pred = f(X)
np.sum(X_valid), np.sum(y_valid))
# ===========================================================================
# Building model
# ===========================================================================
X = K.placeholder(shape=(None, MAX_SEQ_LEN), dtype='int32', name='X')
y = K.placeholder(shape=(None, nb_labels), dtype='float32', name='y')
f = N.Sequence([
N.Embedding(tk.nb_words, embedding_dims, W_init=E),
N.Dimshuffle(pattern=(0, 1, 'x', 2)),
N.Conv(num_filters=128,
filter_size=(5, 1),
strides=1,
pad='valid',
activation=K.relu),
N.Pool(pool_size=(5, 1), pad='valid', mode='max'),
N.Conv(num_filters=128,
filter_size=(5, 1),
strides=1,
pad='valid',
activation=K.relu),
N.Pool(pool_size=(5, 1), pad='valid', mode='max'),
N.Conv(num_filters=128,
filter_size=(5, 1),
strides=1,
pad='valid',
activation=K.relu),
N.Pool(pool_size=(35, 1), pad='valid', mode='max'),
N.Flatten(outdim=2),
N.Dense(num_units=128, activation=K.relu),
N.Dense(num_units=nb_labels, activation=K.softmax)
if USE_MNIST_DATA:
ds = fuel.load_mnist()
else:
ds = fuel.load_cifar10()
X = K.placeholder(shape=(None, ) + ds['X_train'].shape[1:], name='X')
y = K.placeholder(shape=(None, ), name='y', dtype='int32')
# ===========================================================================
# Build network
# ===========================================================================
ops = N.Sequence([
N.Dimshuffle((0, 1, 2, 'x')) if USE_MNIST_DATA else None,
N.BatchNorm(axes='auto'),
N.Conv(32, (3, 3), strides=(1, 1), pad='same', activation=K.relu),
N.Pool(pool_size=(2, 2), strides=None),
N.Conv(64, (3, 3), strides=(1, 1), pad='same', activation=K.relu),
N.Pool(pool_size=(2, 2), strides=None),
N.Flatten(outdim=2),
N.Dense(256, activation=K.relu),
N.Dense(10, activation=K.softmax)
],
debug=True)
ops = cPickle.loads(cPickle.dumps(ops)) # test if the ops is pickle-able
K.set_training(True)
y_pred_train = ops(X)
K.set_training(False)
y_pred_score = ops(X)
cost_train = K.mean(K.categorical_crossentropy(y_pred_train, y))
y = K.placeholder(shape=(None, ), name='y', dtype='int32')
# ===========================================================================
# Build model
# ===========================================================================
f = N.Sequence(
[
N.Embedding(max_features, embedding_size),
N.Dropout(0.25),
N.Dimshuffle(pattern=(0, 1, 'x', 2)), # convolution on time dimension
N.Conv(nb_filter,
filter_size=(filter_length, 1),
pad='valid',
stride=(1, 1),
activation=K.relu),
N.Pool(pool_size=(pool_length, 1), mode='max'),
N.Flatten(outdim=3),
N.Merge(
[
N.Dense(lstm_output_size, activation=K.linear,
name='ingate'), # input-gate
N.Dense(lstm_output_size,
activation=K.linear,
name='forgetgate'), # forget-gate
N.Dense(lstm_output_size,
activation=K.linear,
name='cellupdate'), # cell-update
N.Dense(lstm_output_size, activation=K.linear,
name='outgate') # output-gate
],
merge_function=K.concatenate),
# ===========================================================================
# Create model
# ===========================================================================
inputs = [
K.placeholder(shape=(None, ) + shape[1:],
dtype='float32',
name='input%d' % i) for i, shape in enumerate(train.shape)
]
print("Inputs:", ctext(inputs, 'cyan'))
# ====== create the network ====== #
f_encoder = N.Sequence([
N.Dimshuffle(pattern=(0, 1, 2, 'x')),
N.Conv(
num_filters=32, filter_size=(7, 7), b_init=None, activation=K.linear),
N.BatchNorm(),
N.Pool(pool_size=(3, 2), strides=2),
],
debug=True,
name='Encoder')
f_latent = N.Sequence([
N.Flatten(outdim=3),
N.CudnnRNN(
num_units=128, num_layers=1, is_bidirectional=False, rnn_mode='lstm'),
],
debug=True,
name='Latent')
f_decoder = N.Sequence([
N.Flatten(outdim=2),
N.Dense(num_units=1024, b_init=None, activation=K.linear),
N.BatchNorm(axes=0, activation=K.relu)
],
f = N.Sequence(
[
N.Dimshuffle(pattern=(0, 1, 2, 'x')),
N.Conv(num_filters=32,
filter_size=3,
pad='same',
strides=1,
activation=K.linear),
N.BatchNorm(activation=K.relu),
N.Conv(num_filters=64,
filter_size=3,
pad='same',
strides=1,
activation=K.linear),
N.BatchNorm(activation=K.relu),
N.Pool(pool_size=2, strides=None, pad='valid', mode='max'),
N.Flatten(outdim=3),
# ====== RNN ====== #
N.AutoRNN(128,
rnn_mode='lstm',
num_layers=3,
direction_mode='bidirectional',
prefer_cudnn=True),
# ====== Dense ====== #
N.Flatten(outdim=2),
# N.Dropout(level=0.2), # adding dropout does not help
N.Dense(num_units=1024, activation=K.relu),
N.Dense(num_units=512, activation=K.relu),
N.Dense(num_units=nb_classes, activation=K.softmax)
else:
ds = fuel.load_cifar10()
print(ds)
X = K.placeholder(shape=(None, ) + ds['X_train'].shape[1:], name='X')
y = K.placeholder(shape=(None, ), name='y', dtype='int32')
# ===========================================================================
# Build network
# ===========================================================================
ops = N.Sequence([
N.Dimshuffle((0, 1, 2, 'x')) if USE_MNIST_DATA else N.Dimshuffle(
(0, 2, 3, 1)),
N.Conv(32, filter_size=3, strides=1, pad='same', activation=K.linear),
N.BatchNorm(axes='auto', activation=K.relu),
N.Pool(pool_size=2, strides=None),
N.Dimshuffle(pattern=(0, 3, 1, 2)),
N.Flatten(outdim=3),
N.CudnnRNN(18,
initial_states=None,
rnn_mode='lstm',
num_layers=2,
input_mode='linear',
direction_mode='unidirectional',
params_split=False),
N.Flatten(outdim=2),
N.Dense(128, activation=K.relu),
N.Dense(10, activation=K.softmax)
],
debug=True)
ops = cPickle.loads(cPickle.dumps(ops)) # test if the ops is pickle-able
# ===========================================================================
X = K.placeholder(shape=(None, ) + ds['X_train'].shape[1:],
name='X',
dtype='int32')
y = K.placeholder(shape=(None, ), name='y', dtype='int32')
net_odin = N.Sequence(
[
N.Embedding(input_size=max_features, output_size=embedding_size),
N.Dropout(level=0.25),
N.Dimshuffle(pattern=(0, 1, 'x', 2)),
N.Conv(nb_filter, (filter_length, 1),
strides=1,
pad='valid',
activation=K.relu),
N.Pool(pool_size=(pool_length, 1), pad='valid', mode='max'),
N.Flatten(outdim=3),
# ====== LSTM ====== #
N.Merge(
[
N.Dense(lstm_output_size, activation=K.linear,
name='ingate'), # input-gate
N.Dense(lstm_output_size,
activation=K.linear,
name='forgetgate'), # forget-gate
N.Dense(lstm_output_size,
activation=K.linear,
name='cellupdate'), # cell-update
N.Dense(lstm_output_size, activation=K.linear,
name='outgate') # output-gate
],