def get_model(inputdim,
outputdim,
regularization_strength=0.01,
lr=0.000,
cosine=False,
**kwargs):
transformation = Dense(inputdim,
init='identity',
W_constraint=Orthogonal())
model = Model()
model.add_input(name='embeddings1', input_shape=(inputdim, ))
model.add_input(name='embeddings2', input_shape=(inputdim, ))
model.add_shared_node(transformation,
name='transformation',
inputs=['embeddings1', 'embeddings2'],
outputs=['transformed1', 'transformed2'])
model.add_node(Lambda(lambda x: x[:, :outputdim]),
input='transformed1',
name='projected1')
model.add_node(Lambda(lambda x: -x[:, :outputdim]),
input='transformed2',
name='negprojected2')
if cosine:
model.add_node(
Lambda(lambda x: x / K.reshape(K.sqrt(K.sum(x * x, axis=1)),
(x.shape[0], 1))),
name='normalized1',
input='projected1')
model.add_node(
Lambda(lambda x: x / K.reshape(K.sqrt(K.sum(x * x, axis=1)),
(x.shape[0], 1))),
name='negnormalized2',
input='negprojected2')
model.add_node(
Lambda(lambda x: K.reshape(K.sum(x, axis=1), (x.shape[0], 1))),
name='distances',
inputs=['normalized1', 'negnormalized2'],
merge_mode='mul')
else:
model.add_node(Lambda(lambda x: K.reshape(K.sqrt(K.sum(x * x, axis=1)),
(x.shape[0], 1))),
name='distances',
inputs=['projected1', 'negprojected2'],
merge_mode='sum')
model.add_output(name='y', input='distances')
model.compile(loss={
'y': lambda y, d: K.mean(y * d)
},
optimizer=SimpleSGD())
return model
def create_model(vocab_size, args):
assert args.batch_size % 3 == 0, "Batch size must be multiple of 3"
if args.rnn == 'GRU':
RNN = recurrent.GRU
elif args.rnn == 'LSTM':
RNN = recurrent.LSTM
else:
assert False, "Invalid RNN"
if args.bidirectional:
assert not args.convolution, "Convolutional layer is not supported with bidirectional RNN"
assert not args.pooling, "Pooling layer is not supported with bidirectional RNN"
assert args.dense_layers == 0, "Dense layers are not supported with bidirectional RNN"
model = Model()
model.add_input(name="input",
batch_input_shape=(args.batch_size, 1),
dtype="uint")
model.add_node(Embedding(vocab_size, args.embed_size, mask_zero=True),
name="embed",
input='input')
for i in range(args.layers):
model.add_node(
RNN(args.hidden_size,
return_sequences=False if i + 1 == args.layers else True),
name='forward' + str(i + 1),
input='embed' if i == 0 else 'dropout' +
str(i) if args.dropout > 0 else None,
inputs=['forward' + str(i), 'backward' +
str(i)] if i > 0 and args.dropout == 0 else [])
model.add_node(
RNN(args.hidden_size,
return_sequences=False if i + 1 == args.layers else True,
go_backwards=True),
name='backward' + str(i + 1),
input='embed' if i == 0 else 'dropout' +
str(i) if args.dropout > 0 else None,
inputs=['forward' + str(i), 'backward' +
str(i)] if i > 0 and args.dropout == 0 else [])
if args.dropout > 0:
model.add_node(
Dropout(args.dropout),
name='dropout' + str(i + 1),
inputs=['forward' + str(i + 1), 'backward' + str(i + 1)])
model.add_output(
name='output',
input='dropout' + str(args.layers) if args.dropout > 0 else None,
inputs=[
'forward' + str(args.layers), 'backward' + str(args.layers)
] if args.dropout == 0 else [])
assert args.dense_layers == 0, "Bidirectional model doesn't support dense layers yet"
else:
model = Sequential()
model.add(
Embedding(vocab_size,
args.embed_size,
mask_zero=not args.convolution))
if args.convolution:
model.add(
Convolution1D(nb_filter=args.conv_filters,
filter_length=args.conv_filter_length,
border_mode=args.conv_border_mode,
activation=args.conv_activation,
subsample_length=args.conv_subsample_length))
if args.pooling:
model.add(MaxPooling1D(pool_length=args.pool_length))
for i in range(args.layers):
model.add(
RNN(args.hidden_size,
return_sequences=False if i + 1 == args.layers else True))
if args.dropout > 0:
model.add(Dropout(args.dropout))
for i in range(args.dense_layers):
if i + 1 == args.dense_layers:
model.add(Dense(args.hidden_size, activation='linear'))
else:
model.add(
Dense(args.hidden_size, activation=args.dense_activation))
return model
name='conv_' + str(n_gram),
input='dropout_embedding')
model.add_node(MaxPooling1D(pool_length=maxlen - n_gram + 1),
name='maxpool_' + str(n_gram),
input='conv_' + str(n_gram))
model.add_node(Flatten(),
name='flat_' + str(n_gram),
input='maxpool_' + str(n_gram))
model.add_node(Dropout(0.),
name='dropout',
inputs=['flat_' + str(n) for n in ngram_filters])
model.add_node(Dense(1, input_dim=nb_filter * len(ngram_filters)),
name='dense',
input='dropout')
model.add_node(Activation('sigmoid'), name='sigmoid', input='dense')
model.add_output(name='output', input='sigmoid')
print(model.summary())
# train model
model.compile(loss={'output': 'binary_crossentropy'}, optimizer='rmsprop')
model.fit({
'input': X_train,
'output': y_train
},
batch_size=batch_size,
nb_epoch=nb_epoch,
validation_data={
'input': X_test,
'output': y_test
})
acc = accuracy(