def get_model(inputdim,
outputdim,
regularization_strength=0.01,
lr=0.000,
cosine=False,
**kwargs):
transformation = Dense(inputdim,
init='identity',
W_constraint=Orthogonal())
model = Model()
model.add_input(name='embeddings1', input_shape=(inputdim, ))
model.add_input(name='embeddings2', input_shape=(inputdim, ))
model.add_shared_node(transformation,
name='transformation',
inputs=['embeddings1', 'embeddings2'],
outputs=['transformed1', 'transformed2'])
model.add_node(Lambda(lambda x: x[:, :outputdim]),
input='transformed1',
name='projected1')
model.add_node(Lambda(lambda x: -x[:, :outputdim]),
input='transformed2',
name='negprojected2')
if cosine:
model.add_node(
Lambda(lambda x: x / K.reshape(K.sqrt(K.sum(x * x, axis=1)),
(x.shape[0], 1))),
name='normalized1',
input='projected1')
model.add_node(
Lambda(lambda x: x / K.reshape(K.sqrt(K.sum(x * x, axis=1)),
(x.shape[0], 1))),
name='negnormalized2',
input='negprojected2')
model.add_node(
Lambda(lambda x: K.reshape(K.sum(x, axis=1), (x.shape[0], 1))),
name='distances',
inputs=['normalized1', 'negnormalized2'],
merge_mode='mul')
else:
model.add_node(Lambda(lambda x: K.reshape(K.sqrt(K.sum(x * x, axis=1)),
(x.shape[0], 1))),
name='distances',
inputs=['projected1', 'negprojected2'],
merge_mode='sum')
model.add_output(name='y', input='distances')
model.compile(loss={
'y': lambda y, d: K.mean(y * d)
},
optimizer=SimpleSGD())
return model
#The prediction layer
dense_out = Dense(128, activation='relu')(merged_vector)
predictions = Dense(len(dist_labels), activation='softmax')(dense_out)
model = Model(input=[left_target, right_target], output=predictions)
model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy'])
#Make callbacks
evalcb=CustomCallback(dev_left,dev_right,dev_labels,index2label) # evaluate after each epoch
savecb=ModelCheckpoint(u"rnn_model.model", monitor='val_acc', verbose=1, save_best_only=True, mode='auto') # save model (I have not tested this!)
model.fit([train_left, train_right], train_labels, batch_size=10, nb_epoch=20, callbacks=[evalcb, savecb])
'''
model = Graph()
model.add_input(name='left', input_shape=(window, ), dtype='int')
model.add_input(name='right', input_shape=(window, ), dtype='int')
#For future
#model.add_input(name='left_s', input_shape=(window, ))
#model.add_input(name='right_s', input_shape=(window, ))
vec_size=128
#Hack Embeddings in
from keras.layers.embeddings import Embedding
#Oh dear! What a hacky way to do this!
def create_model(vocab_size, args):
assert args.batch_size % 3 == 0, "Batch size must be multiple of 3"
if args.rnn == 'GRU':
RNN = recurrent.GRU
elif args.rnn == 'LSTM':
RNN = recurrent.LSTM
else:
assert False, "Invalid RNN"
if args.bidirectional:
assert not args.convolution, "Convolutional layer is not supported with bidirectional RNN"
assert not args.pooling, "Pooling layer is not supported with bidirectional RNN"
assert args.dense_layers == 0, "Dense layers are not supported with bidirectional RNN"
model = Model()
model.add_input(name="input",
batch_input_shape=(args.batch_size, 1),
dtype="uint")
model.add_node(Embedding(vocab_size, args.embed_size, mask_zero=True),
name="embed",
input='input')
for i in range(args.layers):
model.add_node(
RNN(args.hidden_size,
return_sequences=False if i + 1 == args.layers else True),
name='forward' + str(i + 1),
input='embed' if i == 0 else 'dropout' +
str(i) if args.dropout > 0 else None,
inputs=['forward' + str(i), 'backward' +
str(i)] if i > 0 and args.dropout == 0 else [])
model.add_node(
RNN(args.hidden_size,
return_sequences=False if i + 1 == args.layers else True,
go_backwards=True),
name='backward' + str(i + 1),
input='embed' if i == 0 else 'dropout' +
str(i) if args.dropout > 0 else None,
inputs=['forward' + str(i), 'backward' +
str(i)] if i > 0 and args.dropout == 0 else [])
if args.dropout > 0:
model.add_node(
Dropout(args.dropout),
name='dropout' + str(i + 1),
inputs=['forward' + str(i + 1), 'backward' + str(i + 1)])
model.add_output(
name='output',
input='dropout' + str(args.layers) if args.dropout > 0 else None,
inputs=[
'forward' + str(args.layers), 'backward' + str(args.layers)
] if args.dropout == 0 else [])
assert args.dense_layers == 0, "Bidirectional model doesn't support dense layers yet"
else:
model = Sequential()
model.add(
Embedding(vocab_size,
args.embed_size,
mask_zero=not args.convolution))
if args.convolution:
model.add(
Convolution1D(nb_filter=args.conv_filters,
filter_length=args.conv_filter_length,
border_mode=args.conv_border_mode,
activation=args.conv_activation,
subsample_length=args.conv_subsample_length))
if args.pooling:
model.add(MaxPooling1D(pool_length=args.pool_length))
for i in range(args.layers):
model.add(
RNN(args.hidden_size,
return_sequences=False if i + 1 == args.layers else True))
if args.dropout > 0:
model.add(Dropout(args.dropout))
for i in range(args.dense_layers):
if i + 1 == args.dense_layers:
model.add(Dense(args.hidden_size, activation='linear'))
else:
model.add(
Dense(args.hidden_size, activation=args.dense_activation))
return model
# prepare data
print('Loading data...')
(X_train, y_train), (X_test, y_test) = imdb.load_data(nb_words=max_features,
test_split=0.2)
print(len(X_train), 'train sequences')
print(len(X_test), 'test sequences')
print('Pad sequences (samples x time)')
X_train = sequence.pad_sequences(X_train, maxlen=maxlen)
X_test = sequence.pad_sequences(X_test, maxlen=maxlen)
print('X_train shape:', X_train.shape)
print('X_test shape:', X_test.shape)
# define model
model = Model()
model.add_input(name='input', input_shape=(maxlen, ), dtype=int)
model.add_node(Embedding(max_features, embedding_dims, input_length=maxlen),
name='embedding',
input='input')
model.add_node(Dropout(0.), name='dropout_embedding', input='embedding')
for n_gram in ngram_filters:
model.add_node(Convolution1D(nb_filter=nb_filter,
filter_length=n_gram,
border_mode='valid',
activation='relu',
subsample_length=1,
input_dim=embedding_dims,
input_length=maxlen),
name='conv_' + str(n_gram),
input='dropout_embedding')
model.add_node(MaxPooling1D(pool_length=maxlen - n_gram + 1),