def _kim_cnn_model(params, args, nb_classes, embedding_matrix):
"""
fully functional API style so that we can see all model details.
params will obtain model related parameters
:param params:
:param args:
:param nb_classes: # of labels to classify
:param embedding_matrix:
:return: a compiled Keras model
"""
nb_filter = params['nb_filter']
use_embeddings = params['use_embeddings']
embeddings_trainable = params['embeddings_trainable']
input = Input(shape=(args.max_sequence_len,), dtype='int32', name="input")
aux = Input(shape=(1,), dtype='float32', name="aux")
len_feat = Dense(1, activation='softmax')(aux)
if (use_embeddings):
embedding_layer = Embedding(args.nb_words + 1,
args.embedding_dim,
weights=[embedding_matrix],
input_length=args.max_sequence_len,
trainable=embeddings_trainable)(input)
else:
embedding_layer = Embedding(args.nb_words + 1,
args.mbedding_dim,
weights=None,
input_length=args.max_sequence_len,
trainable=embeddings_trainable)(input)
embedding_layer = Dropout(params['dropout1'])(embedding_layer)
filtersize = params['filter_size']
filtersize_list = [filtersize - 1, filtersize, filtersize + 1]
conv_list = []
for index, filtersize in enumerate(filtersize_list):
pool_length = args.max_sequence_len - filtersize + 1
conv = Conv1D(nb_filter=nb_filter, filter_length=filtersize, activation='relu')(embedding_layer)
pool = MaxPooling1D(pool_length=pool_length)(conv)
flatten = Flatten()(pool)
conv_list.append(flatten)
if (len(filtersize_list) > 1):
conv_out = Merge(mode='concat', concat_axis=1)(conv_list)
else:
conv_out = conv_list[0]
x = Dropout(params['dropout2'])(conv_out)
x = merge([x, len_feat], mode='concat')
result = Dense(nb_classes, activation='softmax')(x)
model = Model(input=[input, aux], output=result)
model.compile(loss='categorical_crossentropy',
optimizer=params['optimizer'],
metrics=['acc'])
print(model.summary())
return model
def get_model(img_res, additional_features_len, n_classes):
'''
Creates and returns a pre-trained keras model
:param img_res: 2D tuple of x and y resolution
:param additional_features_len: length of additional features (columns)
:param n_classes: number of classes
:return: pre-trained keras model
'''
model = Sequential()
model.add(
Convolution2D(64,
8,
8,
subsample=(3, 3),
activation='relu',
input_shape=(img_res[0], img_res[1]) + (1, )))
model.add(
MaxPooling2D(pool_size=(2, 2), strides=(2, 2), border_mode='same'))
model.add(Convolution2D(128, 4, 4, activation='relu', border_mode='same'))
model.add(
MaxPooling2D(pool_size=(2, 2), strides=(2, 2), border_mode='same'))
model.add(Convolution2D(256, 3, 3, activation='relu', border_mode='same'))
model.add(Convolution2D(256, 3, 3, activation='relu', border_mode='same'))
model.add(Convolution2D(128, 3, 3, activation='relu', border_mode='same'))
model.add(
MaxPooling2D(pool_size=(2, 2), strides=(2, 2), border_mode='same'))
model.add(Flatten())
model.add(Dense(1024, activation='tanh'))
model.add(Dropout(.5))
additional_info_model = Sequential()
additional_info_model.add(Dense(192, input_shape=(192, )))
model_merged = Sequential()
model_merged.add(Merge([model, additional_info_model], mode='concat'))
model_merged.add(Dense(1024 + additional_features_len, activation='tanh'))
model_merged.add(Dropout(.5))
model_merged.add(Dense(n_classes, activation='softmax'))
optimizer = Adam(decay=1e-6)
model_merged.compile(loss='categorical_crossentropy',
metrics=['accuracy'],
optimizer=optimizer)
# Plot the model to files
plot(model, to_file='./model_left.png', show_shapes=True)
plot(additional_info_model, to_file='./model_right.png', show_shapes=True)
plot(model_merged, to_file='./model_merged.png', show_shapes=True)
return model_merged
def create_model_2(max_caption_len, word_indexes, embedding_dim, feature_dim, encoded_dim):
language_model = Sequential()
language_model.add(Embedding(len(word_indexes), embedding_dim, input_length=max_caption_len - 1, mask_zero=True))
image_model = Sequential()
image_model.add(Dense(encoded_dim, activation='linear', input_shape=(feature_dim,)))
image_model.add(RepeatVector(max_caption_len - 1, input_shape=(feature_dim,)))
model = Sequential()
model.add(Merge([image_model, language_model], mode='concat', concat_axis=-1))
model.add(LSTM(1024, return_sequences=True))
model.add(TimeDistributed(Dense(len(word_indexes), activation='softmax')))
model.compile(loss='sparse_categorical_crossentropy', optimizer='rmsprop')
return model
def create_model_4(max_caption_len, word_indexes, embedding_dim, feature_dim, embedding_matrix_filepath):
embedding_matrix = np.load(embedding_matrix_filepath)
language_model = Sequential()
language_model.add(Embedding(len(word_indexes), embedding_dim, weights=[embedding_matrix], input_length=max_caption_len - 1, mask_zero=True, trainable=False))
image_model = Sequential()
# image_model.add(Dense(encoded_dim, activation='linear', input_shape=(feature_dim,)))
image_model.add(RepeatVector(max_caption_len - 1, input_shape=(feature_dim,)))
model = Sequential()
model.add(Merge([image_model, language_model], mode='concat', concat_axis=-1))
model.add(LSTM(256, return_sequences=True))
model.add(TimeDistributed(Dense(len(word_indexes), activation='softmax')))
model.compile(loss='sparse_categorical_crossentropy', optimizer='rmsprop', metrics=['accuracy'])
return model
# model = Sequential()
# model.add(LSTM(185,input_shape=(n_samples,3,n_obs),activation='relu'))
# model.add(Dropout(0.5))
# model.add(Dense(100, activation='relu'))
# model.add(Dropout(0.2))
# model.add(Dense(1, activation='relu'))
#model.compile(loss='categorical_crossentropy', optimizer='rmsprop')
branches = []
tstlen = len(basisfuncts[0][0])
for i in range(0,len(basisfuncts[0][0])):
dimbranch = Sequential()
dimbranch.add(Dense(185, input_dim=2000))
branches.append(dimbranch)
merged = Merge(branches, mode='concat')
final_model = Sequential()
final_model.add(merged)
final_model.add(Dense(1, activation='softmax'))
# two branch Sequential
final_model.compile(loss='binary_crossentropy',
optimizer='rmsprop',
metrics=['accuracy','precision','recall'])
# generate dummy data
# convolution on view and flatten actions
model_view = Sequential()
model_view.add(
Convolution2D(16,
3,
3,
activation='relu',
border_mode='same',
input_shape=(1, view_size.w, view_size.h)))
model_view.add(Flatten())
model_actions = Sequential()
model_actions.add(Flatten(input_shape=(1, action_hist_len, nb_actions)))
model = sft.agent.model.KerasMlpModelNew.KerasMlpModelNew(
logger=logger,
layers=[
Merge([model_view, model_actions], mode='concat', concat_axis=1),
Dense(32, activation='relu'),
Dense(nb_actions, activation='linear')
],
loss='mse',
optimizer=optimizer)
agent = sft.agent.DeepQAgentReplayCloning.DeepQAgentReplayCloning(
logger=logger,
actions=actions,
model=model,
discount=0.9,
batch_size=16,
buffer_size=100000,
start_learn=500,
steps_clone=250,
def model(self,
optimizer,
tx_activation=None,
im_activation=None,
im_hidden_layers=None,
tx_hidden_layers=None,
im_hidden_activation=None,
tx_hidden_activation=None,
contrastive_loss_weight=1,
logistic_loss_weight=1,
contrastive_loss_weight_inverted=1,
submodel=None,
init='glorot_normal'): #glorot_uniform
'''
:param optimizer:
:param hidden_activation: activation for both text and image last dense layers
:param im_hidden_layers: a list of numbers, each one of them is the number of hidden unit for a new hidden layer
:param tx_hidden_layers: a list of numbers, each one of them is the number of hidden unit for a new hidden layer
:param im_hidden_activation: a list of activation functions, one for each image hidden layer, None to disable
:param tx_hidden_activation: a list of activation functions, one for each text hidden layer, None to disable
:param tx_tx_factr: weight for the loss text vs text
:param tx_im_factr: weight for the loss image vs text
:param submodel: None to get the whole model, with 3 outputs: text, image, and (text-image) embedding output.
'txt' to get only the text leg submodel, with only the text-embedding output.
'img' to get only the image leg submodel, with only the image-embedding output.
:return:
'''
if submodel is not None and submodel not in ['txt', 'img']:
ValueError(
'Value for submodel parameter not legal: ' + str(submodel) +
'.'
'\nValue for submodel parameter must be "txt" or "img" or None.'
)
# Image network leg:
im_input = previous_tensor = Input(shape=(self.im_input_dim, ),
name='im_input')
if im_hidden_layers is not None:
for i, hidden_units in enumerate(im_hidden_layers):
previous_tensor = Dense(output_dim=hidden_units,
name='im_hidden_' + str(i),
init=init)(previous_tensor)
if im_hidden_activation is not None:
previous_tensor = Activation(
activation=im_hidden_activation[i])(previous_tensor)
im_emb = Dense(self.output_dim, name='im_embedding')(previous_tensor)
im_emb = Activation(activation=im_activation)(im_emb)
# Text network leg:
tx_input = previous_tensor = Input(shape=(self.tx_input_dim, ),
name='tx_input')
if tx_hidden_layers is not None:
for i, hidden_units in enumerate(tx_hidden_layers):
previous_tensor = Dense(output_dim=hidden_units,
name='tx_hidden_' + str(i),
init=init)(previous_tensor)
if tx_hidden_activation is not None:
previous_tensor = Activation(
activation=tx_hidden_activation[i])(previous_tensor)
tx_emb = Dense(self.output_dim, name='tx_embedding',
init=init)(previous_tensor)
tx_emb = Activation(activation=tx_activation)(tx_emb)
# Text classification (logistic) leg
tx_classification = Dense(self.n_text_classes, activation='softmax')(
tx_emb) # or (previous_tensor) ??
#im_emb_sub_tx_emb = merge([im_emb, tx_emb], mode=lambda x: x[0] - x[1], output_shape=lambda x: x[0], name='subtract')
#im_emb_sub_tx_emb = Merge(mode=euclideanSqDistance, output_shape=lambda x: (x[0][0], 1), name='distance')([im_emb, tx_emb])
#im_emb_sub_tx_emb = Merge(mode=euclideanDistance, output_shape=lambda x: (x[0][0], 1), name='distance')([im_emb, tx_emb])
#im_emb_sub_tx_emb = Merge(mode='cos')([im_emb, tx_emb])
#im_emb_sub_tx_emb = merge([im_emb, tx_emb], mode='sum')
#im_emb_sub_tx_emb = im_emb - tx_emb
distance = Merge(mode=my_distance, output_shape=lambda x:
(x[0][0], 1))([im_emb, tx_emb])
shifted_distance = Merge(mode=my_distance_shifted,
output_shape=lambda x:
(x[0][0], 1))([im_emb, tx_emb])
if submodel is not None:
if submodel == "img":
model = Model(input=im_input, output=im_emb)
model.compile(optimizer=optimizer, loss=fake_loss)
elif submodel == "txt":
model = Model(input=tx_input, output=tx_emb)
model.compile(optimizer=optimizer, loss=fake_loss)
else:
model = None
else:
model = Model(input=[im_input, tx_input],
output=[im_emb, tx_emb, distance, tx_classification])
model.compile(
optimizer=optimizer,
#loss=[ get_contrastive_loss(tx_emb), get_contrastive_loss(im_emb), 'categorical_crossentropy'],
# loss_weights=[contrastive_loss_weight, contrastive_loss_weight_inverted, logistic_loss_weight])
loss=[
fake_loss, fake_loss,
get_contrastive_loss_on_distance(shifted_distance),
'categorical_crossentropy'
],
loss_weights=[
0, 0, contrastive_loss_weight, logistic_loss_weight
])
return model
regularizer = TracenormRegularizer(lr=.001, modalities=6, e_width=e_width)
# regularizer = regularizers.l2(l=0.)
cnn_E_classifier = feature_dnn(dataset.sources['cnn'][1], e_width)
mfcc_E_classifier = feature_dnn(dataset.sources['mfcc'][1], e_width)
mbh_E_classifier = feature_dnn(dataset.sources['mbh'][1], e_width)
sift_E_classifier = feature_dnn(dataset.sources['sift'][1], e_width)
hog_E_classifier = feature_dnn(dataset.sources['hog'][1], e_width)
traj_E_classifier = feature_dnn(dataset.sources['traj'][1], e_width)
model = Sequential()
model.add(
Merge([
cnn_E_classifier, mfcc_E_classifier, mbh_E_classifier,
sift_E_classifier, hog_E_classifier, traj_E_classifier
],
mode='concat'))
model.add(Dense(fusion_width, activation='sigmoid', W_regularizer=regularizer))
model.add(normalization.BatchNormalization())
model.add(Dense(net_output, activation='sigmoid'))
model.compile(loss=objectives.binary_crossentropy,
optimizer=optimizer.SGD(lr=0.2),
metrics=[metrics.categorical_accuracy])
#callback to save the best weights and trigger stop if needed
check_stop = Checkpoint(validation_data=([
cnn_x_t[pick], mfcc_x_t[pick], mbh_x_t[pick], sift_x_t[pick],
hog_x_t[pick], traj_x_t[pick]
], cnn_y_t[pick]),
def model(self,
optimizer,
tx_activation=None,
im_activation=None,
im_hidden_layers=None,
tx_hidden_layers=None,
init='glorot_normal',
triplet_loss_margins=1,
triplet_loss_weights=1,
configs=['tii']): #configs: < anchor, positive, negative >
'''
:param optimizer:
:param im_hidden_layers: a list of numbers/str, for each number a dense layer will be created, for each string
an activation layer will be created.
:param tx_hidden_layers: a list of numbers/str, for each number a dense layer will be created, for each string
an activation layer will be created.
:param submodel: None to get the whole model, with 3 outputs: text, image, and (text-image) embedding output.
'txt' to get only the text leg submodel, with only the text-embedding output.
'img' to get only the image leg submodel, with only the image-embedding output.
:return:
'''
# tii config!
legal_configs = ['tii', 'tit', 'itt', 'iti', 'i-t', 't-i']
# nb: i-t == iit where the anchor and the positive example are the same: ||i-i|| - ||i-t|| = -||i-t||
# to do 'ii2t', 'ii2i', 'tt2i' 'tt2t'
if not isinstance(triplet_loss_margins, list):
lst = []
for config in configs:
lst.append(triplet_loss_margins)
triplet_loss_margins = lst
if not isinstance(triplet_loss_weights, list):
lst = []
for config in configs:
lst.append(triplet_loss_weights)
triplet_loss_weights = lst
# Image network leg:
previous_layer = first_im_layer = Lambda(function=lambda x: x)
if im_hidden_layers is not None:
for i, hid in enumerate(im_hidden_layers):
if isinstance(hid, int):
previous_layer = Dense(output_dim=hid,
name='im_hidden_' + str(i),
init=init)(previous_layer)
elif isinstance(hid, basestring):
previous_layer = Activation(activation=hid)(previous_layer)
im_emb = Dense(self.output_dim, name='im_embedding')(previous_layer)
im_emb = Activation(activation=im_activation)(im_emb)
# Text network leg:
previous_layer = first_tx_layer = Input(shape=(self.tx_input_dim, ),
name='tx_input')
if tx_hidden_layers is not None:
for i, hid in enumerate(tx_hidden_layers):
if isinstance(hid, int):
previous_layer = Dense(output_dim=hid,
name='tx_hidden_' + str(i),
init=init)(previous_layer)
elif isinstance(hid, basestring):
previous_layer = Activation(activation=hid)(previous_layer)
tx_emb = Dense(self.output_dim, name='tx_embedding',
init=init)(previous_layer)
tx_emb = Activation(activation=tx_activation)(tx_emb)
im_pos_input = Input(shape=(self.im_input_dim, ), name='im_pos_input')
im_pos_input_2 = Input(shape=(self.im_input_dim, ),
name='im2_pos_input')
im_neg_input = Input(shape=(self.im_input_dim, ), name='im_neg_input')
tx_pos_input = Input(shape=(self.tx_input_dim, ), name='tx_pos_input')
tx_neg_input = Input(shape=(self.tx_input_dim, ), name='tx_neg_input')
im_pos_encoded = first_im_layer(im_pos_input)
im_neg_encoded = first_im_layer(im_neg_input)
tx_pos_encoded = first_tx_layer(tx_pos_input)
tx_neg_encoded = first_tx_layer(tx_neg_input)
merges = {}
for config in configs:
if config[0] == 't':
anchor = tx_pos_encoded
elif config[0] == 'i':
anchor = im_pos_encoded
if config[1] == 't':
pos = tx_pos_encoded
elif config[1] == 'i':
pos = im_pos_encoded
elif config[1] == '-':
pos = anchor
if config[2] == 't':
neg = tx_neg_encoded
elif config[2] == 'i':
neg = im_neg_encoded
merge = Merge(mode=triplet_distance,
output_shape=lambda x:
(x[0][0], 1))([anchor, pos, neg])
merges[config] = merge
outputs = [im_emb, tx_emb]
losses = [fake_loss, fake_loss]
loss_weights = [0, 0]
for i, m in enumerate(merges.values()):
outputs.append(m)
losses.append(get_triplet_loss(triplet_loss_margins[i]))
loss_weights.append(triplet_loss_weights[i])
model = Model(
input=[im_pos_input, im_neg_input, tx_pos_input, tx_neg_input],
output=outputs)
model.compile(optimizer=optimizer,
loss=losses,
loss_weights=loss_weights)
return model
def model(self,
optimizer,
tx_activation=None,
im_activation=None,
im_hidden_layers=None,
tx_hidden_layers=None,
contrastive_loss_weight=1,
logistic_loss_weight=1,
contrastive_loss_weight_inverted=1,
submodel=None,
init='glorot_normal',
contrastive_loss_margin=1,
use_triplet=False,
triplet_mode='tii'): #glorot_uniform
'''
:param optimizer:
:param im_hidden_layers: a list of numbers/str, for each number a dense layer will be created, for each string
an activation layer will be created.
:param tx_hidden_layers: a list of numbers/str, for each number a dense layer will be created, for each string
an activation layer will be created.
:param submodel: None to get the whole model, with 3 outputs: text, image, and (text-image) embedding output.
'txt' to get only the text leg submodel, with only the text-embedding output.
'img' to get only the image leg submodel, with only the image-embedding output.
:return:
'''
if submodel is not None and submodel not in ['txt', 'img']:
ValueError(
'Value for submodel parameter not legal: ' + str(submodel) +
'.'
'\nValue for submodel parameter must be "txt" or "img" or None.'
)
# Image network leg:
im_input = previous_tensor = Input(shape=(self.im_input_dim, ),
name='im_input')
if im_hidden_layers is not None:
for i, hid in enumerate(im_hidden_layers):
if isinstance(hid, int):
previous_tensor = Dense(output_dim=hid,
name='im_hidden_' + str(i),
init=init)(previous_tensor)
elif isinstance(hid, basestring):
previous_tensor = Activation(
activation=hid)(previous_tensor)
im_emb = Dense(self.output_dim, name='im_embedding')(previous_tensor)
im_emb = Activation(activation=im_activation)(im_emb)
# Text network leg:
tx_input = previous_tensor = Input(shape=(self.tx_input_dim, ),
name='tx_input')
if tx_hidden_layers is not None:
for i, hid in enumerate(tx_hidden_layers):
if isinstance(hid, int):
previous_tensor = Dense(output_dim=hid,
name='tx_hidden_' + str(i),
init=init)(previous_tensor)
elif isinstance(hid, basestring):
previous_tensor = Activation(
activation=hid)(previous_tensor)
tx_emb = Dense(self.output_dim, name='tx_embedding',
init=init)(previous_tensor)
tx_emb = Activation(activation=tx_activation)(tx_emb)
# Text classification (logistic) leg
tx_classification = Dense(self.n_text_classes, activation='softmax')(
tx_emb) # or (previous_tensor) ??
#im_emb_sub_tx_emb = merge([im_emb, tx_emb], mode=lambda x: x[0] - x[1], output_shape=lambda x: x[0], name='subtract')
#im_emb_sub_tx_emb = Merge(mode='cos')([im_emb, tx_emb])
#im_emb_sub_tx_emb = merge([im_emb, tx_emb], mode='sum')
#im_emb_sub_tx_emb = im_emb - tx_emb
#im_emb_sub_tx_emb = Merge(mode=euclideanSqDistance, output_shape=lambda x: (x[0][0], 1), name='distance')([im_emb, tx_emb])
#im_emb_sub_tx_emb = Merge(mode=euclideanDistance, output_shape=lambda x: (x[0][0], 1), name='distance')([im_emb, tx_emb])
distance = Merge(mode=my_distance, output_shape=lambda x:
(x[0][0], 1))([im_emb, tx_emb])
#distance = Merge(mode=my_cosine_distance, output_shape=lambda x: (x[0][0], 1))([im_emb, tx_emb])
if submodel is not None:
if submodel == "img":
model = Model(input=im_input, output=im_emb)
model.compile(optimizer=optimizer, loss=fake_loss)
elif submodel == "txt":
model = Model(input=tx_input, output=tx_emb)
model.compile(optimizer=optimizer, loss=fake_loss)
else:
model = None
else:
if use_triplet:
if triplet_mode == 'tii':
model = Model(
input=[tx_input, im_input, im_input],
output=[im_emb, tx_emb, distance, tx_classification])
# model = Model(input=[im_input, tx_input], output=[im_emb, tx_emb, tx_classification])
else:
model = Model(
input=[im_input, tx_input],
output=[im_emb, tx_emb, distance, tx_classification])
# model = Model(input=[im_input, tx_input], output=[im_emb, tx_emb, tx_classification])
if self.use_merge_distance:
model.compile(optimizer=optimizer,
loss=[
fake_loss, fake_loss,
get_contrastive_loss_over_distance(
contrastive_loss_margin),
'categorical_crossentropy'
],
loss_weights=[
0, 0, contrastive_loss_weight,
logistic_loss_weight
])
else:
model.compile(
optimizer=optimizer,
loss=[
get_contrastive_loss_contr_data(tx_emb),
get_contrastive_loss_contr_data(im_emb),
#get_contrastive_loss_im(tx_emb), get_contrastive_loss_tx(im_emb),
fake_loss,
'categorical_crossentropy'
],
loss_weights=[
contrastive_loss_weight / 2,
contrastive_loss_weight_inverted / 2, 0,
logistic_loss_weight
])
return model
input_length=seq_length,
weights=weights))
model_right.add(LSTM(output_dim=15, go_backwards=True))
model_right.add(Dropout(0.2))
dep_right = Sequential()
dep_right.add(
TimeDistributed(Dense(output_dim=15),
input_shape=(seq_length, len(dep_labels))))
dep_right.add(Dropout(0.2))
dep_right.add(Flatten())
merged_model = Sequential()
merged_model.add(
Merge([model_left, dep_left, model_right, dep_right],
mode='concat',
concat_axis=1))
merged_model.add(Dense(10))
merged_model.add(Dense(1, activation='sigmoid'))
merged_model.compile(loss='binary_crossentropy',
optimizer='adagrad',
metrics=['accuracy'])
print(u"Done...")
# --------------------------------------------------------------------------------------------------------------------
checkpoint = ModelCheckpoint(filepath="./weights/lstm.hdf5", verbose=0)
merged_model.fit([X_L, D_L, X_R, D_R],
Y,
batch_size=16,
nb_epoch=5,
callbacks=[checkpoint],
verbose=1)
# for a multi-input model with 10 classes:
from keras.engine import Merge
from keras.layers import Dense
from keras.models import Sequential
left_branch = Sequential()
left_branch.add(Dense(32, input_dim=784))
right_branch = Sequential()
right_branch.add(Dense(32, input_dim=784))
merged = Merge([left_branch, right_branch], mode='concat')
model = Sequential()
model.add(merged)
model.add(Dense(10, activation='softmax'))
model.compile(optimizer='rmsprop',
loss='categorical_crossentropy',
metrics=['accuracy'])
# generate dummy data
import numpy as np
from keras.utils.np_utils import to_categorical
data_1 = np.random.random((1000, 784))
data_2 = np.random.random((1000, 784))
# these are integers between 0 and 9
labels = np.random.randint(10, size=(1000, 1))
# we convert the labels to a binary matrix of size (1000, 10)
# for use with categorical_crossentropy
filter_length=char_window,
border_mode='valid')))
model_cnn.add(TimeDistributed(MaxPooling1D(pool_length=max_charlen)))
model_cnn.add(Reshape((n_in, n_filters)))
'''model_cnn = Sequential()
model_cnn.add(Embedding(output_dim=char_embedding_size, input_dim=char_vocab_size, input_length=n_in*max_charlen))
model_cnn.add(Reshape((n_in, max_charlen, char_embedding_size)))
model_cnn.add(Permute((1,3,2)))
model_cnn.add(Convolution2D(nb_filter=n_filters, nb_row=1, nb_col=char_window*max_charlen, border_mode='same'))
model_cnn.add(MaxPooling2D((1, max_charlen)))
model_cnn.add(Reshape((n_in, char_embedding_size)))'''
# Create the Network
model = Sequential()
# Hidden + Softmax Layer
model.add(Merge([model_wordemb, model_cnn], mode='concat'))
model.add(Flatten())
model.add(
Dense(
output_dim=n_hidden,
init='glorot_uniform',
activation='tanh',
))
#model.add(Dropout(0.5))
model.add(
Dense(
output_dim=n_hidden,
init='glorot_uniform',
activation='tanh',
))
#model.add(Dropout(0.5))
hog_dnn = Sequential()
hog_dnn.add(InputLayer(input_shape=(239, )))
hog_scalar = VectorLayer()
hog_dnn.add(hog_scalar)
# hog_dnn.add(Activation('sigmoid'))
traj_dnn = Sequential()
traj_dnn.add(InputLayer(input_shape=(239, )))
traj_scalar = VectorLayer()
traj_dnn.add(traj_scalar)
# traj_dnn.add(Activation('sigmoid'))
model = Sequential()
model.add(
Merge([cnn_dnn, mfcc_dnn, mbh_dnn, hog_dnn, traj_dnn],
mode=jrer_fusion,
output_shape=(239, )))
model.add(Activation('linear'))
model.compile(loss=objectives.binary_crossentropy,
optimizer=optimizer.Adam(),
metrics=[metrics.categorical_accuracy])
model.summary()
# model.load_weights(filepath='weights/JRER_TRAINING.h5')
yp1 = cnn_x_t
yp2 = mfcc_x_t
yp3 = mbh_x_t
yp4 = hog_x_t