def create(self):
language_model = Sequential()
self.textual_embedding(language_model, mask_zero=True)
self.temporal_pooling(language_model)
language_model.add(DropMask())
#language_model.add(BatchNormalization(mode=1))
self.language_model = language_model
visual_model_factory = \
select_sequential_visual_model[self._config.trainable_perception_name](
self._config.visual_dim)
visual_model = visual_model_factory.create()
visual_dimensionality = visual_model_factory.get_dimensionality()
self.visual_embedding(visual_model, visual_dimensionality)
#visual_model.add(BatchNormalization(mode=1))
self.visual_model = visual_model
if self._config.multimodal_merge_mode == 'dot':
self.add(
Merge([language_model, visual_model],
mode='dot',
dot_axes=[(1, ), (1, )]))
else:
self.add(
Merge([language_model, visual_model],
mode=self._config.multimodal_merge_mode))
self.deep_mlp()
self.add(Dense(self._config.output_dim))
self.add(Activation('softmax'))
def create(self):
language_model = Sequential()
self.textual_embedding(language_model, mask_zero=True)
self.language_model = language_model
visual_model_factory = \
select_sequential_visual_model[self._config.trainable_perception_name](
self._config.visual_dim)
visual_model = visual_model_factory.create()
visual_dimensionality = visual_model_factory.get_dimensionality()
self.visual_embedding(visual_model, visual_dimensionality)
#visual_model = Sequential()
#self.visual_embedding(visual_model)
self.visual_model = visual_model
visual_model.add(RepeatVector(self._config.max_input_time_steps))
if self._config.multimodal_merge_mode == 'dot':
self.add(
Merge([language_model, visual_model],
mode='dot',
dot_axes=[(1, ), (1, )]))
else:
self.add(
Merge([language_model, visual_model],
mode=self._config.multimodal_merge_mode))
self.add(
self._config.recurrent_encoder(
self._config.hidden_state_dim,
return_sequences=False,
go_backwards=self._config.go_backwards))
self.deep_mlp()
self.add(Dense(self._config.output_dim))
self.add(Activation('softmax'))
def test_merge_dot():
(X_train, y_train), (X_test, y_test) = _get_test_data()
left = Sequential()
left.add(Dense(input_dim=input_dim, output_dim=nb_hidden))
left.add(Activation('relu'))
right = Sequential()
right.add(Dense(input_dim=input_dim, output_dim=nb_hidden))
right.add(Activation('relu'))
model = Sequential()
model.add(Merge([left, right], mode='dot', dot_axes=1))
model.add(Dense(nb_class))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
left = Sequential()
left.add(Dense(input_dim=input_dim, output_dim=nb_hidden))
left.add(Activation('relu'))
right = Sequential()
right.add(Dense(input_dim=input_dim, output_dim=nb_hidden))
right.add(Activation('relu'))
model = Sequential()
model.add(Merge([left, right], mode='dot', dot_axes=[1, 1]))
model.add(Dense(nb_class))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
def test_merge_sum():
(X_train, y_train), (X_test, y_test) = _get_test_data()
left = Sequential()
left.add(Dense(nb_hidden, input_shape=(input_dim,)))
left.add(Activation('relu'))
right = Sequential()
right.add(Dense(nb_hidden, input_shape=(input_dim,)))
right.add(Activation('relu'))
model = Sequential()
model.add(Merge([left, right], mode='sum'))
model.add(Dense(nb_class))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
model.fit([X_train, X_train], y_train, batch_size=batch_size, nb_epoch=nb_epoch, verbose=0, validation_data=([X_test, X_test], y_test))
model.fit([X_train, X_train], y_train, batch_size=batch_size, nb_epoch=nb_epoch, verbose=0, validation_split=0.1)
model.fit([X_train, X_train], y_train, batch_size=batch_size, nb_epoch=nb_epoch, verbose=0)
model.fit([X_train, X_train], y_train, batch_size=batch_size, nb_epoch=nb_epoch, verbose=0, shuffle=False)
loss = model.evaluate([X_test, X_test], y_test, verbose=0)
model.predict([X_test, X_test], verbose=0)
model.predict_classes([X_test, X_test], verbose=0)
model.predict_proba([X_test, X_test], verbose=0)
# test weight saving
fname = 'test_merge_sum_temp.h5'
model.save_weights(fname, overwrite=True)
left = Sequential()
left.add(Dense(nb_hidden, input_shape=(input_dim,)))
left.add(Activation('relu'))
right = Sequential()
right.add(Dense(nb_hidden, input_shape=(input_dim,)))
right.add(Activation('relu'))
model = Sequential()
model.add(Merge([left, right], mode='sum'))
model.add(Dense(nb_class))
model.add(Activation('softmax'))
model.load_weights(fname)
os.remove(fname)
model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
nloss = model.evaluate([X_test, X_test], y_test, verbose=0)
assert(loss == nloss)
# test serialization
config = model.get_config()
new_model = Sequential.from_config(config)
model.summary()
json_str = model.to_json()
new_model = model_from_json(json_str)
yaml_str = model.to_yaml()
new_model = model_from_yaml(yaml_str)
def test_merge_concat(self):
print('Test merge: concat')
left = Sequential()
left.add(Dense(nb_hidden, input_shape=(input_dim,)))
left.add(Activation('relu'))
right = Sequential()
right.add(Dense(nb_hidden, input_shape=(input_dim,)))
right.add(Activation('relu'))
model = Sequential()
model.add(Merge([left, right], mode='concat'))
model.add(Dense(nb_class))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
model.fit([X_train, X_train], y_train, batch_size=batch_size, nb_epoch=nb_epoch, show_accuracy=True, verbose=0, validation_data=([X_test, X_test], y_test))
model.fit([X_train, X_train], y_train, batch_size=batch_size, nb_epoch=nb_epoch, show_accuracy=False, verbose=0, validation_data=([X_test, X_test], y_test))
model.fit([X_train, X_train], y_train, batch_size=batch_size, nb_epoch=nb_epoch, show_accuracy=True, verbose=0, validation_split=0.1)
model.fit([X_train, X_train], y_train, batch_size=batch_size, nb_epoch=nb_epoch, show_accuracy=False, verbose=0, validation_split=0.1)
model.fit([X_train, X_train], y_train, batch_size=batch_size, nb_epoch=nb_epoch, verbose=0)
model.fit([X_train, X_train], y_train, batch_size=batch_size, nb_epoch=nb_epoch, verbose=0, shuffle=False)
loss = model.evaluate([X_train, X_train], y_train, verbose=0)
print('loss:', loss)
if loss > 0.6:
raise Exception('Score too low, learning issue.')
preds = model.predict([X_test, X_test], verbose=0)
classes = model.predict_classes([X_test, X_test], verbose=0)
probas = model.predict_proba([X_test, X_test], verbose=0)
print(model.get_config(verbose=1))
print('test weight saving')
model.save_weights('temp.h5', overwrite=True)
left = Sequential()
left.add(Dense(nb_hidden, input_shape=(input_dim,)))
left.add(Activation('relu'))
right = Sequential()
right.add(Dense(nb_hidden, input_shape=(input_dim,)))
right.add(Activation('relu'))
model = Sequential()
model.add(Merge([left, right], mode='concat'))
model.add(Dense(nb_class))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
model.load_weights('temp.h5')
nloss = model.evaluate([X_train, X_train], y_train, verbose=0)
print(nloss)
assert(loss == nloss)
def test_merge_concat():
(X_train, y_train), (X_test, y_test) = _get_test_data()
left = Sequential()
left.add(Dense(nb_hidden, input_shape=(input_dim,)))
left.add(Activation('relu'))
right = Sequential()
right.add(Dense(nb_hidden, input_shape=(input_dim,)))
right.add(Activation('relu'))
model = Sequential()
model.add(Merge([left, right], mode='concat'))
model.add(Dense(nb_class))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
model.fit([X_train, X_train], y_train, batch_size=batch_size, nb_epoch=nb_epoch, show_accuracy=True, verbose=0, validation_data=([X_test, X_test], y_test))
model.fit([X_train, X_train], y_train, batch_size=batch_size, nb_epoch=nb_epoch, show_accuracy=False, verbose=0, validation_data=([X_test, X_test], y_test))
model.fit([X_train, X_train], y_train, batch_size=batch_size, nb_epoch=nb_epoch, show_accuracy=True, verbose=0, validation_split=0.1)
model.fit([X_train, X_train], y_train, batch_size=batch_size, nb_epoch=nb_epoch, show_accuracy=False, verbose=0, validation_split=0.1)
model.fit([X_train, X_train], y_train, batch_size=batch_size, nb_epoch=nb_epoch, verbose=0)
model.fit([X_train, X_train], y_train, batch_size=batch_size, nb_epoch=nb_epoch, verbose=0, shuffle=False)
loss = model.evaluate([X_test, X_test], y_test, verbose=0)
assert(loss < 0.8)
model.predict([X_test, X_test], verbose=0)
model.predict_classes([X_test, X_test], verbose=0)
model.predict_proba([X_test, X_test], verbose=0)
model.get_config(verbose=0)
fname = 'test_merge_concat_temp.h5'
model.save_weights(fname, overwrite=True)
left = Sequential()
left.add(Dense(nb_hidden, input_shape=(input_dim,)))
left.add(Activation('relu'))
right = Sequential()
right.add(Dense(nb_hidden, input_shape=(input_dim,)))
right.add(Activation('relu'))
model = Sequential()
model.add(Merge([left, right], mode='concat'))
model.add(Dense(nb_class))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
model.load_weights(fname)
os.remove(fname)
nloss = model.evaluate([X_test, X_test], y_test, verbose=0)
assert(loss == nloss)
def add_output(self, name, input=None, inputs=[], merge_mode='concat'):
if name in self.output_order:
raise Exception('Duplicate output identifier: ' + name)
if input:
if input not in self.namespace:
raise Exception('Unknown node/input identifier: ' + input)
if input in self.nodes:
self.outputs[name] = self.nodes[input]
elif input in self.inputs:
self.outputs[name] = self.inputs[input]
if inputs:
to_merge = []
for n in inputs:
if n not in self.nodes:
raise Exception('Unknown identifier: ' + n)
to_merge.append(self.nodes[n])
merge = Merge(to_merge, mode=merge_mode)
self.outputs[name] = merge
self.output_order.append(name)
self.output_config.append({
'name': name,
'input': input,
'inputs': inputs,
'merge_mode': merge_mode
})
def seqCNN_CPT(c_conf=(4, 3, 32, 32),
p_conf=(4, 3, 32, 32),
t_conf=(4, 3, 32, 32)):
'''
C - Temporal Closeness
P - Period
T - Trend
conf = (nb_flow, seq_len, map_height, map_width)
'''
model = Sequential()
components = []
for conf in [c_conf, p_conf, t_conf]:
if conf is not None:
components.append(seqCNNBaseLayer1(conf))
nb_flow = conf[0]
model.add(Merge(components, mode='concat', concat_axis=1)) # concat
model.add(Convolution2D(64, 3, 3, border_mode='same'))
model.add(Activation('relu'))
model.add(Convolution2D(64, 3, 3, border_mode='same'))
model.add(Activation('relu'))
model.add(Convolution2D(nb_flow, 3, 3, border_mode='same'))
model.add(Activation('tanh'))
return model
def build_model(self):
premise = Sequential()
premise.add(
LSTM(input_shape=self.ds.X_p_train.shape[1:],
output_dim=ve.HIDDEN_SIZE))
premise.add(Dropout(ve.DROPOUT_PROB))
hypothesis = Sequential()
hypothesis.add(
LSTM(input_shape=self.ds.X_h_train.shape[1:],
output_dim=ve.HIDDEN_SIZE))
hypothesis.add(Dropout(ve.DROPOUT_PROB))
model = Sequential()
model.add(Merge([premise, hypothesis], mode=ve.CONCAT))
model.add(
Dense(output_dim=ve.HIDDEN_SIZE,
activation=ve.TANH,
W_regularizer=l2(self.hp.dense_l2)))
model.add(
Dense(output_dim=ve.NLI_CLASSES,
activation=ve.SOFTMAX,
W_regularizer=l2(self.hp.dense_l2)))
ve.compile_other_model(model, self.hp.optimizer)
return model
def build_lstm(n_con, n_emb, vocabs_size, n_dis, emb_size, cluster_size):
hidden_size = 800
con = Sequential()
con.add(Dense(input_dim=n_con, output_dim=emb_size))
emb_list = []
for i in range(n_emb):
emb = Sequential()
emb.add(
Embedding(input_dim=vocabs_size[i],
output_dim=emb_size,
input_length=n_dis))
emb.add(Flatten())
emb_list.append(emb)
in_dimension = 2
seq = Sequential()
seq.add(BatchNormalization(input_shape=((MAX_LENGTH, in_dimension))))
seq.add(Masking([0] * in_dimension,
input_shape=(MAX_LENGTH, in_dimension)))
seq.add(
LSTM(emb_size,
return_sequences=False,
input_shape=(MAX_LENGTH, in_dimension)))
model = Sequential()
model.add(Merge([con] + emb_list + [seq], mode='concat'))
model.add(BatchNormalization())
model.add(Dense(hidden_size, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(cluster_size, activation='softmax'))
model.add(Lambda(caluate_point, output_shape=[2]))
return model
def __init__(self,
img_dim=4096,
word_dim=300,
max_sent_len=26,
nb_classes=1000,
lstm_hidden_dim=512,
fc_hidden_dim=2014,
bidirect=True,
dropout=0.5):
img_model = Sequential()
img_model.add(Reshape(input_shape=(img_dim, ), dims=(img_dim, )))
txt_model = Sequential()
# return sequences if use bidrectional LSTM
txt_model.add(
LSTM(output_dim=hidden_dim,
return_sequences=bidirect,
input_shape=(max_sent_len, word_dim)))
#TODO: check if the paper actually use bi-lstm?!? not really understand what happened here!
if bidirect:
txt_model.add(LSTM(output_dim=hidden_dim, return_sequences=False))
model = Sequential()
model.add(Merge(txt_model, img_model), mode='concat', concat_axis=1)
for i in xrange(2):
model.add(Dense(fc_hidden_dim, init='uniform'))
model.add(Activation('tanh'))
model.add(Dropout(dropout))
model.add(Dense(nb_classes))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
return model
def _train_embedding(x_train, y_train, input_dim, hidden_layers,
batch_size, nb_epoch):
models = []
for feature in cols:
if Config.feature_sizes[feature] == 1:
model = Sequential()
model.add(Dense(1, input_dim=1))
models.append(model)
else:
model = Sequential()
model.add(
Embedding(Config.feature_sizes[feature],
Config.embedding_sizes[feature],
input_length=1))
model.add(
Reshape(target_shape=(Config.embedding_sizes[feature], )))
models.append(model)
model = Sequential()
model.add(Merge(models, mode='concat'))
for l in range(len(hidden_layers)):
model.add(Dense(output_dim=hidden_layers[l], input_dim=input_dim))
model.add(Dropout(Config.dropout))
model.add(Activation(Config.activation))
input_dim = hidden_layers[l]
model.compile(loss=Config.loss_func, optimizer=Config.optimizer)
model.fit(split_features(np.array(x_train)),
np.array(y_train),
batch_size=batch_size,
nb_epoch=nb_epoch)
return model
def parallel_cnn(W):
(nb_vocab, dims) =W.shape
N_filter=20
filter_shapes = [[2, 300], [3, 300], [4, 300], [5, 300]]
pool_shapes = [[25, 1], [24, 1], [23, 1], [22, 1]] # Four Parallel Convolutional Layers with Four Pooling Layers
model = Sequential()
sub_models = []
for i in range(len(pool_shapes)):
pool_shape = pool_shapes[i]
filter_shape = filter_shapes[i]
sub_model = Sequential()
sub_model.add(Embedding(input_dim=nb_vocab, output_dim=dims, weights=[W], W_constraint=unitnorm()))
# Reshape word vectors from Embedding to tensor format suitable for Convolutional layer
sub_model.add(Reshape(dims=(1, 200, dims)))
sub_model.add(Convolution2D(nb_filter=N_filter, nb_row=filter_shape[0], nb_col=filter_shape[1], border_mode = 'valid', activation = 'relu'))
sub_model.add(MaxPooling2D(pool_size=(pool_shape[0], pool_shape[1]), border_mode='valid'))
sub_model.add(Flatten())
sub_models.append(sub_model)
model.add((Merge(sub_models, mode='concat')))
# Fully Connected Layer with dropout
model.add(Flatten())
model.add(Dense(256, activation='relu'))
model.add(Dropout(0.5))
# Fully Connected Layer as output layer
model.add(Dense(2, activation='softmax'))
return model
def build_critic_network(self):
conv_model = Sequential()
conv_model.add(Conv2d(32, 8, 8, input_shape=(self.img_dims[0], self.img_dims[1], self.frame_num), activation='relu', subsample=(4,4)))
conv_model.add(Conv2d(64,4,4, activation='relu', subsample=(2,2)))
conv_model.add(Conv2d(64,3,3, activation='relu', subsample=(1,1)))
conv_model.add(Flatten())
conv_model.add(Dense(100, activation='tanh'))
action_model = Sequential()
action_model.add(Dense(10, activation='relu', input_dim=self.action_dim))
merged = Merge([conv_model, action_model], mode='concat')
final_model = Sequential()
final_model.add(merged)
final_model.add(Dense(self.action_dim, activation='tanh'))
final_model.add(Dense(self.action_dim, activation='linear'))
s_i_c = tf.placeholder(tf.float32, [None, self.img_dims[0], self.img_dims[1], self.frame_num])
s_a_c = tf.placeholder(tf.float32, [None, self.action_dim])
q = final_model([s_i_c, s_a_c])
action_grads = tf.gradients(q, s_a_c)
return final_model, s_i_c, s_a_c, q, action_grads
def cnn(embedding_size=EMBEDDING_SIZE, output_length=DEFAULT_LABELS):
""" Create and return a keras model of a CNN """
NB_FILTER = 256
NGRAM_LENGTHS = [1, 2, 3, 4, 5]
conv_layers = []
for ngram_length in NGRAM_LENGTHS:
ngram_layer = Sequential()
ngram_layer.add(Convolution1D(
NB_FILTER,
ngram_length,
input_dim=embedding_size,
input_length=SAMPLE_LENGTH,
init='lecun_uniform',
activation='tanh',
))
pool_length = SAMPLE_LENGTH - ngram_length + 1
ngram_layer.add(MaxPooling1D(pool_length=pool_length))
conv_layers.append(ngram_layer)
model = Sequential()
model.add(Merge(conv_layers, mode='concat'))
model.add(Dropout(0.5))
model.add(Flatten())
model.add(Dense(output_length, activation='sigmoid'))
model.compile(
loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'],
)
return model
def creat_binary_tag_LSTM( sourcevocabsize,targetvocabsize, source_W,input_seq_lenth ,output_seq_lenth ,
hidden_dim ,emd_dim,loss='categorical_crossentropy',optimizer = 'rmsprop'):
encoder_a = Sequential()
encoder_b = Sequential()
encoder_c = Sequential()
l_A_embedding = Embedding(input_dim=sourcevocabsize+1,
output_dim=emd_dim,
input_length=input_seq_lenth,
mask_zero=True,
weights=[source_W])
encoder_a.add(l_A_embedding)
encoder_a.add(Dropout(0.3))
encoder_b.add(l_A_embedding)
encoder_b.add(Dropout(0.3))
encoder_c.add(l_A_embedding)
Model = Sequential()
encoder_a.add(LSTM(hidden_dim,return_sequences=True))
encoder_b.add(LSTM(hidden_dim,return_sequences=True,go_backwards=True))
encoder_rb = Sequential()
encoder_rb.add(ReverseLayer2(encoder_b))
encoder_ab=Merge(( encoder_a,encoder_rb),mode='concat')
Model.add(encoder_ab)
decodelayer=LSTMDecoder_tag(hidden_dim=hidden_dim, output_dim=hidden_dim
, input_length=input_seq_lenth,
output_length=output_seq_lenth,
state_input=False,
return_sequences=True)
Model.add(decodelayer)
Model.add(TimeDistributedDense(targetvocabsize+1))
Model.add(Activation('softmax'))
Model.compile(loss=loss, optimizer=optimizer)
return Model
def modelQA(vocab_size, img_dim, wordvec_dim, inp_len, embeddings):
# Returns the QA model
x = Input(shape=(4096, ))
img_model1 = Model(input=x, output=x)
img_model2 = Sequential()
img_model2.add(img_model1)
img_model2.add(Dense(300))
text_model = Sequential()
text_model.add(
Embedding(vocab_size,
wordvec_dim,
weights=[embeddings],
input_length=inp_len,
trainable=False)) #, mask_zero=True))
text_model.add(Lambda(MeanPool1D, output_shape=MeanPool1D_shape))
text_model.add(Dense(300))
model = Sequential()
model.add(Merge([img_model2, text_model], mode='mul'))
model.add(Dropout(0.25))
model.add(Dense(300))
model.add(Dropout(0.25))
model.add(Dense(300))
model.add(Dropout(0.25))
model.add(Dense(100))
model.add(Dropout(0.5))
model.add(Dense(1, activation='sigmoid'))
model.compile(optimizer='adam', loss='binary_crossentropy', \
metrics=['accuracy'])
return model
def create(self):
assert self._config.textual_embedding_dim == 0, \
'Embedding cannot be learnt but must be fixed'
language_forward = Sequential()
language_forward.add(
self._config.recurrent_encoder(
self._config.hidden_state_dim,
return_sequences=False,
input_shape=(self._config.max_input_time_steps,
self._config.input_dim)))
self.language_forward = language_forward
language_backward = Sequential()
language_backward.add(
self._config.recurrent_encoder(
self._config.hidden_state_dim,
return_sequences=False,
go_backwards=True,
input_shape=(self._config.max_input_time_steps,
self._config.input_dim)))
self.language_backward = language_backward
self.add(Merge([language_forward, language_backward]))
self.deep_mlp()
self.add(Dense(self._config.output_dim))
self.add(Activation('softmax'))
def test_embeddings(df, y_train):
df = df[['Agencia_ID', 'Canal_ID']]
X = np.array(df.values.copy())
y = y_train
print "Before", X.shape, y.shape
print np.max(X[:, 0]), np.max(X[:, 1])
#X = np.array([[0, 0], [1, 1], [2, 2], [3, 3], [3, 4], [5,6]])
X_list = [X[:, 0].reshape(-1, 1), X[:, 1].reshape(-1, 1)]
#y = np.array([0, 0, 0, 1, 1, 6])
print "After", X.shape, y.shape, X_list[0].shape, X_list[1].shape
embed1 = Sequential()
embed1.add(Embedding(30000, 5, input_length=1))
embed1.add(Reshape(dims=(5, )))
embed2 = Sequential()
embed2.add(Embedding(12, 3, input_length=1))
embed2.add(Reshape(dims=(3, )))
model = Sequential()
model.add(Merge([embed1, embed2], mode='concat'))
model.add(Dense(32, init='uniform'))
model.add(Activation('relu'))
model.add(Dense(1, init='uniform'))
model.add(Activation('sigmoid'))
model.compile(loss='mean_absolute_error', optimizer='adam')
model.fit(X_list, y, nb_epoch=10, batch_size=4)
def get_cnn_network_embed(rna_len, pro_len):
nbfilter = 16
print 'configure cnn network'
embedded_dim, embedding_weights, n_aa_symbols = get_embed_dim('peptideEmbedding25.pickle')
seq_model = set_cnn_embed(n_aa_symbols, pro_len, embedded_dim, embedding_weights)
print n_aa_symbols
embedded_rna_dim, embedding_rna_weights, n_nucl_symbols = get_embed_dim('rnaEmbedding.pickle')
print 'symbol', n_nucl_symbols
struct_model = set_cnn_embed(n_nucl_symbols, rna_len, embedded_rna_dim, embedding_rna_weights)
#pdb.set_trace()
print 'pro cnn', seq_model.output_shape
print 'rna cnn', struct_model.output_shape
model = Sequential()
model.add(Merge([seq_model, struct_model], mode='concat', concat_axis=1))
#model.add(Bidirectional(LSTM(2*nbfilter)))
#model.add(Dropout(0.10))
model.add(Flatten())
#model.add(Dense(nbfilter*(n_aa_symbols + n_nucl_symbols), activation='relu'))
#model.add(Dropout(0.50))
model.add(Dense(nbfilter*100, activation='relu'))
model.add(Dropout(0.50))
#model.add(BatchNormalization(mode=2))
print model.output_shape
return model
def design_model_A(lstm_data_dim, nn_data_dim, timesteps):
model_A = Sequential()
model_B = Sequential()
model_Combine = Sequential()
# LSTM Part
lstm_hidden_size = [40, 100]
drop_out_rate = [0.6, 0.5]
reg = [0.01]
areg = [0.01]
# unfortunately regularization is not implemented for LSTMs
model_A.add(
LSTM(lstm_hidden_size[0],
return_sequences=True,
input_shape=(timesteps, lstm_data_dim)))
model_A.add(
Dropout(drop_out_rate[0])
) # return_sequences=True means output cell state C at each LSTM sequence
model_A.add(LSTM(lstm_hidden_size[1], return_sequences=False))
model_A.add(
Dropout(drop_out_rate[1])
) # return_sequence=False means output only last cell state C in LSTM sequence
model_A.add(
Dense(1,
activation='linear',
W_regularizer=l2(reg[0]),
activity_regularizer=activity_l2(areg[0])))
# NN Part
nn_hidden_size = [40, 40]
nn_drop_rate = [0.5, 0.5]
nn_reg = [0.01, 0.01, 0.01]
nn_areg = [0.01, 0.01, 0.01]
model_B.add(
Dense(nn_hidden_size[0],
input_dim=nn_data_dim,
W_regularizer=l2(nn_reg[0]),
activity_regularizer=activity_l2(nn_areg[0])))
model_B.add(Dropout(nn_drop_rate[0]))
model_B.add(
Dense(nn_hidden_size[1],
W_regularizer=l2(nn_reg[1]),
activity_regularizer=activity_l2(nn_areg[1])))
model_B.add(Dropout(nn_drop_rate[1]))
model_B.add(
Dense(1,
activation='linear',
W_regularizer=l2(nn_reg[2]),
activity_regularizer=activity_l2(nn_areg[2])))
# Merge and Final Layer
model_Combine.add(Merge([model_A, model_B], mode='concat'))
model_Combine.add(Dense(1, activation='linear'))
# output the model to a PNG file for visualization
print "Outputting model graph to model.png"
graph = to_graph(model_Combine, show_shape=True)
graph.write_png("model.png")
return model_Combine
def build(self):
self.img_model = Sequential()
self.img_model.add(MaxPooling2D(input_shape=(14, 14, 512)))
self.img_model.add(Flatten())
for i in xrange(3):
self.img_model.add(Dense(self.img_dim, activation='tanh'))
self.img_model.add(BatchNormalization())
self.txt_model = Sequential()
# self.txt_model.add(Embedding(self.vocab_size, self.word_dim, input_length=self.max_sent_len, mask_zero = True))
if self.bidirect:
self.txt_model.add(Bidirectional(LSTM(output_dim=self.lstm_hidden_dim), input_shape=(self.max_sent_len, self.word_dim)))
# self.txt_model.add(Bidirectional(GRU(output_dim=self.lstm_hidden_dim), input_shape=(self.max_sent_len, self.word_dim)))
else:
M = Masking(mask_value=0., input_shape=(self.max_sent_len, self.word_dim))
self.txt_model.add(M)
self.txt_model.add(LSTM(output_dim=self.lstm_hidden_dim, input_shape=(self.max_sent_len, self.word_dim)))
# self.txt_model.add(GRU(output_dim=self.lstm_hidden_dim, input_shape=(self.max_sent_len, self.word_dim)))
self.model = Sequential()
self.model.add(Merge([self.txt_model, self.img_model], mode='concat', concat_axis=1))
self.model.add(BatchNormalization())
for i in xrange(2):
self.model.add(Dense(self.fc_hidden_dim, init='he_normal', activation='relu'))
self.model.add(BatchNormalization())
self.model.add(Dropout(self.dropout))
self.model.add(Dense(self.nb_classes, activation='softmax'))
self.model.compile(loss='categorical_crossentropy', optimizer='adadelta', metrics=['accuracy'])
self.model.summary()
def lstm_resnet_network(num_classes, num_hidden_units, max_len, word_dim,
img_dim):
image_model = Sequential()
image_model.add(Reshape((img_dim, ), input_shape=(img_dim, )))
lstm_model = Sequential()
lstm_model.add(
LSTM(num_hidden_units,
activation='tanh',
return_sequences=False,
input_shape=(max_len, word_dim)))
lstm_model.add(Dropout(0.5))
combined_model = Sequential()
combined_model.add(
Merge([image_model, lstm_model], mode='concat', concat_axis=1))
combined_model.add(Dense(1024, init='glorot_uniform'))
combined_model.add(Activation('relu'))
combined_model.add(Dropout(0.5))
combined_model.add(Dense(num_classes, init='glorot_uniform'))
combined_model.add(Activation('softmax'))
print 'Model Compilation started'
combined_model.compile(
loss='categorical_crossentropy',
optimizer='adam') #### adam,rmsprop better than sgd for rnns
print 'Model compilation done'
return combined_model
def lstm_resnet_embedding_network(num_classes, num_hidden_units, max_len,
word_dim, img_dim, embedding_matrix):
image_model = Sequential()
image_model.add(Reshape((img_dim, ), input_shape=(img_dim, )))
lstm_model = Sequential()
lstm_model.add(
Embedding(embedding_matrix.shape[0],
word_dim,
weights=[embedding_matrix],
trainable=True,
input_length=max_len))
lstm_model.add(Activation('tanh'))
lstm_model.add(
LSTM(num_hidden_units,
activation='tanh',
return_sequences=False,
input_shape=(max_len, word_dim)))
lstm_model.add(Dropout(0.5))
combined_model = Sequential()
combined_model.add(
Merge([image_model, lstm_model], mode='concat', concat_axis=1))
combined_model.add(Dense(1024, init='glorot_uniform'))
combined_model.add(Activation('relu'))
combined_model.add(Dropout(0.5))
combined_model.add(Dense(num_classes, init='glorot_uniform'))
combined_model.add(Activation('softmax'))
#adam = Adam(decay=0.99997592083)
print 'Model Compilation started'
combined_model.compile(
loss='categorical_crossentropy',
optimizer='adam') #### adam,rmsprop better than sgd for rnns
print 'Model compilation done'
return combined_model
def _build_model(self, word_length, label_length, sentence_length):
embedding_size = 128
lstm_size = 64
dropout_rate = 0.1
forward = Sequential()
forward.add(
Embedding(word_length,
embedding_size,
input_length=sentence_length))
forward.add(LSTM(lstm_size, activation='tanh'))
backward = Sequential()
backward.add(
Embedding(word_length,
embedding_size,
input_length=sentence_length))
backward.add(LSTM(lstm_size, activation='tanh', go_backwards=True))
model = Sequential()
model.add(Merge([forward, backward], mode='concat'))
model.add(Dropout(dropout_rate))
model.add(Dense(label_length, activation='softmax'))
model.compile(loss='categorical_crossentropy',
optimizer='rmsprop',
metrics=['accuracy'])
self._model = model
def build_critic_network(self):
sensor_model = Sequential()
sensor_model.add(
Dense(30, activation='relu', input_dim=self.sensor_dim))
#sensor_model.add(Dense(10, activation='tanh'))
action_model = Sequential()
action_model.add(
Dense(30, activation='relu', input_dim=self.action_dim))
#sensor_model.add(Dense(10, activation='tanh'))
merged = Merge([sensor_model, action_model], mode='concat')
final_model = Sequential()
final_model.add(merged)
#final_model.add(Dense(self.action_dim, activation='tanh'))
final_model.add(Dense(self.action_dim, activation='linear'))
s_s_c = tf.placeholder(tf.float32, [None, self.sensor_dim])
s_a_c = tf.placeholder(tf.float32, [None, self.action_dim])
q = final_model([s_s_c, s_a_c])
action_grads = tf.gradients(q, s_a_c)
return final_model, s_s_c, s_a_c, q, action_grads
def inception_net(_input):
x = Reshape((28, 28, 1))(_input)
#x = Convolution2D(32, 3, 3, subsample=(1, 1))(x)
#x = Activation('relu')(x)
x = Convolution2D(16, 3, 3, subsample=(2, 2))(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
x = Convolution2D(48, 3, 3, subsample=(1, 1))(x)
x = BatchNormalization()(x)
x = Activation('relu')(x)
#x = MaxPooling2D((3, 3), strides=(2,2))(x)
x = mniny_inception_module(x, 1)
x = mniny_inception_module(x, 2)
#x = MaxPooling2D((3, 3), strides=(2,2))(x)
x = mniny_inception_module(x, 2)
x, soft1 = mniny_inception_module(x, 3, True)
x = mniny_inception_module(x, 3)
x = mniny_inception_module(x, 3)
x, soft2 = mniny_inception_module(x, 4, True)
x = MaxPooling2D((3, 3), strides=(2, 2))(x)
x = mniny_inception_module(x, 4)
x = mniny_inception_module(x, 5)
x = AveragePooling2D((5, 5), strides=(1, 1))(x)
x = Dropout(0.4)(x)
x = Flatten()(x)
x = Dense(10)(x)
soft3 = Activation('softmax')(x)
out = Merge(mode='ave', concat_axis=1)([soft1, soft2, soft3])
return out
def build_model(self):
vector_input = Sequential()
vector_input.add(
Embedding(input_dim=len(self.ds.vocab),
output_dim=ve.EMBEDDING_SIZE,
weights=[self.ds.weights],
input_length=ve.WINDOW_SIZE))
caps_input = Sequential()
caps_input.add(
Embedding(input_dim=ve.CAPS_DIMS,
output_dim=ve.CAPS_DIMS,
weights=[np.eye(ve.CAPS_DIMS)],
input_length=ve.WINDOW_SIZE))
model = Sequential()
model.add(Merge([vector_input, caps_input], mode=ve.CONCAT))
model.add(
Reshape(((ve.EMBEDDING_SIZE + ve.CAPS_DIMS) * ve.WINDOW_SIZE, )))
model.add(Dense(output_dim=ve.HIDDEN_SIZE))
model.add(Activation(ve.TANH))
model.add(Dropout(ve.DROPOUT_PROB))
model.add(Dense(input_dim=ve.HIDDEN_SIZE, output_dim=ve.NER_CLASSES))
model.add(Activation(ve.SOFTMAX))
ve.compile_other_model(model, self.hp.optimizer)
return model
def build_model(learning_rate=0.001, reg_rate=0.001, input_size=SIZE):
cnn = Sequential()
cnn.add(
Convolution2D(8,
5,
5,
border_mode='same',
activation='relu',
input_shape=(1, input_size, input_size)))
#cnn.add(Convolution2D(8, 3, 3, border_mode='same', activation='relu'))
#cnn.add(BatchNormalization())
#cnn.add(Activation('relu'))
cnn.add(MaxPooling2D())
cnn.add(Convolution2D(32, 5, 5, border_mode='same', activation='relu'))
#cnn.add(Convolution2D(32, 3, 3, border_mode='same', activation='relu'))
#cnn.add(BatchNormalization())
#cnn.add(Activation('relu'))
cnn.add(MaxPooling2D())
cnn.add(Flatten())
cnn.add(Dropout(0.5))
mlp = Sequential()
mlp.add(Flatten(input_shape=(3, 8, 8)))
model = Sequential()
model.add(Merge([cnn, mlp], mode='concat'))
model.add(Dense(128, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(99, activation='softmax'))
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
return model
def mul model = Sequential() model.add(Embedding(1000, 64, input_length=2)) mode.add(Merge( input_array = np.random.randint
def buildComposition(self, shared=True):
q_input = self.tensors['q_input']
a_input = self.tensors['a_input']
if shared:
q_embedding = self.layers['shared-embedding'](q_input)
a_embedding = self.layers['shared-embedding'](a_input)
else:
q_embedding = self.layers['q-embedding'](q_input)
a_embedding = self.layers['a-embedding'](a_input)
print('Embedding ndim q %d a %d' % (K.ndim(q_embedding), K.ndim(a_embedding)))
print('Embedding shape ', q_embedding._keras_shape, a_embedding._keras_shape)
# compute Semantic Matching
cross = Merge(
# [q_embedding, a_embedding],
mode=semantic_matrix,
output_shape=(self.q_length, self.a_length),
name='semantic'
)
semantic = cross([q_embedding, a_embedding])
print('Semantic ndim %d' % K.ndim(semantic))
print('Semantic shape ', semantic._keras_shape)
print('Semantic shape ', cross.get_output_shape_at(0))
# compute cross
q_match = merge(
[a_embedding, semantic],
mode=lambda x: match_matrix(*x,axis=0, w=self.params['window']),
output_shape=(self.q_length, self.wdim),
name='q_match'
)
print('q_match ', q_match._keras_shape, K.ndim(q_match))
a_match = merge(
[q_embedding, semantic],
mode=lambda x: match_matrix(*x,axis=1, w=self.params['window']),
output_shape=(self.a_length, self.wdim),
name='a_match'
)
print('Match ndim q %d a %d' % (K.ndim(q_match), K.ndim(a_match)))
print('Match shape ', q_match._keras_shape, a_match._keras_shape)
self.tensors['q-embedding'] = q_embedding
self.tensors['a-embedding'] = a_embedding
self.tensors['q-match'] = q_match
self.tensors['a-match'] = a_match
embedding = Embedding(
len(vocab),
WORD_EMBEDDING_DIM,
#input_length=QMAX_TIMESTAMP,
)
q_embedding = embedding(q_input)
a_embedding = embedding(a_input)
print('Embedding ndim q %d a %d' % (K.ndim(q_embedding), K.ndim(a_embedding)))
print('Embedding shape ', q_embedding._keras_shape, a_embedding._keras_shape)
# compute Semantic Matching
cross = Merge(
# [q_embedding, a_embedding],
mode=semantic_matrix,
output_shape=(QMAX_TIMESTAMP, AMAX_TIMESTAMP),
name='semantic'
)
semantic = cross([q_embedding, a_embedding])
print('Semantic ndim %d' % K.ndim(semantic))
print('Semantic shape ', semantic._keras_shape)
print('Semantic shape ', cross.get_output_shape_at(0))
# compute cross
q_match = merge(
[a_embedding, semantic],
mode=lambda x: match_matrix(x,axis=0),
output_shape=(QMAX_TIMESTAMP, WORD_EMBEDDING_DIM),
name='q_match'
)
print('q_match ', q_match._keras_shape, K.ndim(q_match))
