def parallel_cross_straight_newactivation():
log.write("\nparallel-straight-cross-newactivation\n")
conv_0 = Conv1D(64,kernel_size,padding='valid',strides=1, activation=newacti)(e)
maxpool_0 = MaxPooling1D(pool_size=12)(conv_0)
gru=GRU(gru_output_size, return_sequences=True, dropout=0.2, recurrent_dropout=0.2)(e)
maxpool_gru = MaxPooling1D(pool_size=12)(gru)
merge2 = maximum([maxpool_0, maxpool_gru])
#merge=Reshape((sequence_length,filters1))(merge2)
gru1=GRU(gru_output_size, return_sequences=True, dropout=0.2, recurrent_dropout=0.2)(merge2)
gru1=MaxPooling1D(pool_size=8)(gru1)
conv_1=Conv1D(filters1,kernel_size,padding='valid',activation=newacti,strides=1)(merge2)
maxpool_1=MaxPooling1D(pool_size=8)(conv_1)
merge1 = maximum([gru1,maxpool_1])
flatten = Flatten()(merge1)
dropout = Dropout(0.5)(flatten)
output = Dense(nclass, activation='softmax')(dropout)
model = Model(inputs=visible, outputs=output)
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['top_k_categorical_accuracy'])
print(model.summary())
model.fit(padded_docs,y_train, nb_epoch=epochesno, batch_size=64, validation_data=(vpadded_docs, y_valid), callbacks=[metriczs, plot])
print('Model built successfully...Please wait.....Evaluating......')
# Final evaluation of the model
scores = model.evaluate(tpadded_docs, y_test)
print("Loss: %.2f%%" % (scores[0]*100))
print("Accuracy: %.2f%%" % (scores[1]*100))
log.write('\nparallel-cross-straight-newactivation-accuracy -\n '+str(scores[1]*100)+"\t Loss: "+str(scores[0]*100))
def build_dnn_model(self):
layers = self.loader['layers']
maxout = self.loader['maxout']
x = inputs = Input(shape=len(self.columns))
auxiliary_input = Dense(1,
activation='relu',
kernel_constraint='glorot_uniform')(x)
for layer in layers:
if layer == 0:
continue
# x = Dense(layer, activation='sigmoid')(x)
if maxout < 2:
x = Dense(layer,
activation='sigmoid',
kernel_initializer='glorot_uniform')(x)
else:
x = maximum([
Dense(layer,
activation='sigmoid',
kernel_regularizer=regularizers.l1(0.001),
kernel_initializer='one')(x) for _ in range(maxout)
])
x = LeakyReLU(alpha=0.1)(x)
x = Dense(1, activation='relu')(x)
x = Average()([x, auxiliary_input])
outputs = LeakyReLU(alpha=-1)(x)
self.model = Model(inputs=[inputs], outputs=outputs)
return self.model
def parallel_cross_newactivation():
embedding = Embedding(top_words,embedding_vecor_length, input_length=max_review_length, trainable=True)(visible)
e=Reshape((sequence_length,embedding_vecor_length,1))(embedding)
print(e.shape)
log.write("\nparallel-cross-newactivation\n")
conv_0 = Conv2D(filters1, kernel_size=(filter_sizes[0], 100), padding='valid', kernel_initializer='normal', activation=newacti)(e)
maxpool_0 = MaxPool2D(pool_size=(sequence_length - filter_sizes[0] +1, 1), strides=(1,1), padding='valid')(conv_0)
maxpool_0=Reshape((1,gru_output_size))(maxpool_0)
maxpool_0=GRU(gru_output_size, return_sequences=True, dropout=0.2, recurrent_dropout=0.2)(maxpool_0)
maxpool_0=Reshape((1,gru_output_size))(maxpool_0)
gru=Reshape((sequence_length,embedding_vecor_length))(e)
gru=GRU(gru_output_size, return_sequences=True, dropout=0.2, recurrent_dropout=0.2)(gru)
gru=Conv1D(64,kernel_size,padding='valid',activation=newacti,strides=1)(gru)
gru=MaxPooling1D(pool_size=pool_size)(gru)
merge = maximum([maxpool_0, gru])
flatten = Flatten()(merge)
dropout = Dropout(0.5)(flatten)
output = Dense(nclass, activation='softmax')(dropout)
model = Model(inputs=visible, outputs=output)
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['top_k_categorical_accuracy'])
print(model.summary())
plot_model(model, to_file='model_plot_parallel_cross.png', show_shapes=True, show_layer_names=True)
model.fit(X_train, out_train, nb_epoch=epochesno, batch_size=64, validation_data=(X_val,out_val), callbacks=[metriczs])
print('Model built successfully...Please wait.....Evaluating......')
# Final evaluation of the model
scores = model.evaluate(X_test, out_test, verbose=0)
print("Loss: %.2f%%" % (scores[0]*100))
print("Accuracy: %.2f%%" % (scores[1]*100))
log.write('\nparallel-cross-newactivation-accuracy -\n '+str(scores[1]*100)+"\t Loss: "+str(scores[0]*100))
def parallel_cross_leakyrelu():
log.write("\nparallel-cross-leakyrelu\n")
conv_0 = Conv1D(64,kernel_size,padding='valid',strides=1, activation=LeakyReLU(alpha=.001))(e)
maxpool_0 = MaxPooling1D(pool_size=pool_size)(conv_0)
maxpool_0=GRU(gru_output_size, return_sequences=True, dropout=0.2, recurrent_dropout=0.2)(maxpool_0)
gru=Reshape((sequence_length,embedding_vecor_length))(e)
gru=GRU(gru_output_size, return_sequences=True, dropout=0.2, recurrent_dropout=0.2)(gru)
gru=Conv1D(64,kernel_size,padding='valid',activation=LeakyReLU(alpha=.001),strides=1)(gru)
gru=MaxPooling1D(pool_size=pool_size)(gru)
merge = maximum([maxpool_0, gru])
flatten = Flatten()(merge)
dropout = Dropout(0.5)(flatten)
output = Dense(nclass, activation='softmax')(dropout)
model = Model(inputs=visible, outputs=output)
model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['top_k_categorical_accuracy'])
print(model.summary())
#plot_model(model, to_file='model_plot_parallel_cross.png', show_shapes=True, show_layer_names=True)
model.fit(padded_docs,y_train, nb_epoch=epochesno, batch_size=64, validation_data=(vpadded_docs, y_valid), callbacks=[metriczs, plot])
print('Model built successfully...Please wait.....Evaluating......')
# Final evaluation of the model
scores = model.evaluate(tpadded_docs, y_test)
print("Loss: %.2f%%" % (scores[0]*100))
print("Accuracy: %.2f%%" % (scores[1]*100))
log.write('\nparallel-cross-leakyrelu-accuracy -\n '+str(scores[1]*100)+"\t Loss: "+str(scores[0]*100))
def scse_block(inp, ratio=2):
x1 = cse_block(inp, ratio)
x2 = sse_block(inp)
x = maximum([x1, x2])
return x
def create_model(self, X, wv_model):
X_preprocessed = super(ModelContainer9,
self).inputs_context_emb_layer_nc(X, wv_model)
X_preprocessed = np.array([
sequence.pad_sequences(subX_prep,
maxlen=maxlen,
padding='post',
truncating='post',
value=-1) for subX_prep in X_preprocessed
])
input = Input(shape=(3, maxlen), dtype='int32')
# Left Context branch
input_L = Lambda(lambda x: x[:, 0], output_shape=(maxlen, ))(input)
embedding_layer_L = wv_model.wv.get_embedding_layer()
mask_L = Masking(mask_value=-1)(input_L)
emb_seq_L = embedding_layer_L(mask_L)
x_L = Conv1D(filters=8, kernel_size=3, activation='relu')(emb_seq_L)
x_L = Dropout(0.4)(x_L)
x_L = GlobalMaxPooling1D()(x_L)
x_L = Dropout(0.2)(x_L)
x_L = Dense(2, activation='sigmoid')(x_L)
# Connective branch
input_NC = Lambda(lambda x: x[:, 1], output_shape=(maxlen, ))(input)
embedding_layer_NC = wv_model.wv.get_embedding_layer()
mask_NC = Masking(mask_value=-1)(input_NC)
emb_seq_NC = embedding_layer_NC(mask_NC)
x_NC = Conv1D(filters=128, kernel_size=4,
activation='relu')(emb_seq_NC)
x_NC = (AveragePooling1D(pool_size=4))(x_NC)
x_NC = (Dropout(0.2))(x_NC)
x_NC = (Conv1D(filters=256, kernel_size=2, activation='relu'))(x_NC)
x_NC = (Dropout(0.5))(x_NC)
x_NC = (Conv1D(filters=32, kernel_size=8, activation='relu'))(x_NC)
x_NC = GlobalMaxPooling1D()(x_NC)
x_NC = Dense(16, activation='tanh')(x_NC)
x_NC = Dropout(0.2)(x_NC)
x_NC = Dense(2, activation='sigmoid')(x_NC)
# Right context branch
input_R = Lambda(lambda x: x[:, 2], output_shape=(maxlen, ))(input)
embedding_layer_R = wv_model.wv.get_embedding_layer()
mask_R = Masking(mask_value=-1)(input_R)
emb_seq_R = embedding_layer_R(mask_R)
x_R = Conv1D(filters=8, kernel_size=3, activation='relu')(emb_seq_R)
x_R = Dropout(0.4)(x_R)
x_R = GlobalMaxPooling1D()(x_R)
x_R = Dropout(0.2)(x_R)
x_R = Dense(2, activation='sigmoid')(x_R)
x = maximum([x_L, x_R])
x = concatenate([x_NC, x])
preds = (Dense(2, activation='sigmoid'))(x)
model = Model(input, preds, name='Model9_contexts')
return model, X_preprocessed
def _build_entity_capsule(self, x, entity, capsules):
"""Construye una cápsula de entidad particular para la entidad `entity`.
Si la entidad es hoja (no tiene hijos) se conecta directamente a `x`,
de lo contrario, se conecta a los respectivos hijos.
"""
children = [capsules[c] for c in entity.children]
inputs = maximum(children, name="max-%s" %
entity.name) if len(children) > 1 else x
return EntityLayer(self.entity_shape, name=entity.name)(inputs)
def eltwise(layer, layer_in, layerId):
out = {}
if (layer['params']['operation'] == 0):
# This input reverse is to handle visualization
out[layerId] = multiply(layer_in[::-1])
elif (layer['params']['operation'] == 1):
out[layerId] = add(layer_in[::-1])
else:
out[layerId] = maximum(layer_in[::-1])
return out
def test_merge_maximum():
i1 = layers.Input(shape=(4, 5))
i2 = layers.Input(shape=(4, 5))
o = layers.maximum([i1, i2])
assert o._keras_shape == (None, 4, 5)
model = models.Model([i1, i2], o)
x1 = np.random.random((2, 4, 5))
x2 = np.random.random((2, 4, 5))
out = model.predict([x1, x2])
assert out.shape == (2, 4, 5)
assert_allclose(out, np.maximum(x1, x2), atol=1e-4)
def build_model():
vocb_size = 0
vocb_size = X_pad.max() + 1
# channel 1
inputs1 = Input(shape=(X_train.shape[1], ))
embedding1 = Embedding(vocb_size, embed_size)(inputs1)
x_lstm1 = Bidirectional(LSTM(50, activation='relu',
return_sequences=True))(embedding1)
conv1 = Conv1D(filters=32, kernel_size=4, activation='relu')(x_lstm1)
drop1 = Dropout(0.5)(conv1)
pool1 = MaxPooling1D(pool_size=2)(drop1)
flat1 = Flatten()(pool1)
# channel 2
inputs2 = Input(shape=(X_train.shape[1], ))
embedding2 = Embedding(vocb_size, 100)(inputs2)
conv2 = Conv1D(filters=32, kernel_size=6, activation='relu')(embedding2)
drop2 = Dropout(0.5)(conv2)
pool2 = MaxPooling1D(pool_size=2)(drop2)
flat2 = Flatten()(pool2)
merged1 = concatenate([flat1, flat2])
# channel 3
inputs3 = Input(shape=(X_train.shape[1], ))
embedding3 = Embedding(vocb_size, embed_size)(inputs3)
x_lstm3 = Bidirectional(LSTM(50, activation='relu',
return_sequences=True))(embedding3)
conv3 = Conv1D(filters=32, kernel_size=4, activation='relu')(x_lstm3)
drop3 = Dropout(0.5)(conv3)
pool3 = MaxPooling1D(pool_size=2)(drop3)
flat3 = Flatten()(pool3)
# channel 4
inputs4 = Input(shape=(X_train.shape[1], ))
embedding4 = Embedding(vocb_size, 100)(inputs4)
conv4 = Conv1D(filters=32, kernel_size=6, activation='relu')(embedding4)
drop4 = Dropout(0.5)(conv4)
pool4 = MaxPooling1D(pool_size=2)(drop4)
flat4 = Flatten()(pool4)
merged2 = concatenate([flat3, flat4])
# merge
#merged = minimum([merged1,merged2])
merged = maximum([merged1, merged2])
# interpretation
dense1 = Dense(10, activation='relu')(merged)
outputs = Dense(3, activation='sigmoid')(dense1)
model = Model(inputs=[inputs1, inputs2, inputs3, inputs4], outputs=outputs)
# compile
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
# summarize
print(model.summary())
#plot_model(model, show_shapes=True, to_file='multichannel.png')
return model
def test_merge_maximum():
i1 = layers.Input(shape=(4, 5))
i2 = layers.Input(shape=(4, 5))
o = layers.maximum([i1, i2])
assert o._keras_shape == (None, 4, 5)
model = models.Model([i1, i2], o)
x1 = np.random.random((2, 4, 5))
x2 = np.random.random((2, 4, 5))
out = model.predict([x1, x2])
assert out.shape == (2, 4, 5)
assert_allclose(out, np.maximum(x1, x2), atol=1e-4)
def eltwise(layer, layer_in, layerId):
out = {}
if (layer['params']['layer_type'] == 'Multiply'):
# This input reverse is to handle visualization
out[layerId] = multiply(layer_in[::-1])
elif (layer['params']['layer_type'] == 'Sum'):
out[layerId] = add(layer_in[::-1])
elif (layer['params']['layer_type'] == 'Average'):
out[layerId] = average(layer_in[::-1])
elif (layer['params']['layer_type'] == 'Dot'):
out[layerId] = dot(layer_in[::-1], -1)
else:
out[layerId] = maximum(layer_in[::-1])
return out
def eltwise(layer, layer_in, layerId):
out = {}
if (layer['params']['layer_type'] == 'Multiply'):
# This input reverse is to handle visualization
out[layerId] = multiply(layer_in[::-1])
elif (layer['params']['layer_type'] == 'Sum'):
out[layerId] = add(layer_in[::-1])
elif (layer['params']['layer_type'] == 'Average'):
out[layerId] = average(layer_in[::-1])
elif (layer['params']['layer_type'] == 'Dot'):
out[layerId] = dot(layer_in[::-1], -1)
else:
out[layerId] = maximum(layer_in[::-1])
return out
def model3(train_x,train_y):
"""建立四层的神经网络"""
inputs=Input(shape=(train_x.shape[1],))
x1 = Dense(500, activation='tanh')(inputs)
x2 = Dense(500, activation='relu')(inputs)
x=maximum([x1,x2])
x = Dense(50, activation='sigmoid')(x)
x = Dense(50, activation='linear')(x)
predictions = Dense(10, activation='softmax')(x)
model = Model(inputs=inputs, outputs=predictions)
model.compile(optimizer='rmsprop',
loss='categorical_crossentropy',
metrics=['accuracy'])
model.fit(train_x, train_y,epochs=15, batch_size=200, validation_split=0.0)
return model
def resnet_mvcnn(target_size, num_images, num_classes):
# this is the Network to be share amongst the views
img_input = Input(shape=target_size + (3, ))
x = Conv2D(64, (7, 7), strides=(2, 2), padding='same',
name='conv1')(img_input)
# x = BatchNormalization(axis=bn_axis, name='bn_conv1')(x)
x = Activation('relu')(x)
x = MaxPooling2D((3, 3), strides=(2, 2))(x)
x = conv_block(x, 3, [64, 64, 256], stage=2, block='a', strides=(1, 1))
x = identity_block(x, 3, [64, 64, 256], stage=2, block='b')
x = identity_block(x, 3, [64, 64, 256], stage=2, block='c')
x = conv_block(x, 3, [128, 128, 512], stage=3, block='a')
x = identity_block(x, 3, [128, 128, 512], stage=3, block='b')
x = identity_block(x, 3, [128, 128, 512], stage=3, block='c')
x = identity_block(x, 3, [128, 128, 512], stage=3, block='d')
x = conv_block(x, 3, [256, 256, 1024], stage=4, block='a')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='b')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='c')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='d')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='e')
x = identity_block(x, 3, [256, 256, 1024], stage=4, block='f')
x = conv_block(x, 3, [512, 512, 2048], stage=5, block='a')
x = identity_block(x, 3, [512, 512, 2048], stage=5, block='b')
x = identity_block(x, 3, [512, 512, 2048], stage=5, block='c')
x = AveragePooling2D((7, 7), name='avg_pool')(x)
outp = Flatten()(x)
shared_resnet = Model(img_input, outp)
# one input per image
inputs = [Input(shape=target_size + (3, )) for _ in range(num_images)]
# encode through the shared network
encodeds = [shared_resnet(inputs[idx]) for idx in range(num_images)]
# rather than concatenate, this time we take the maximum and pass it through another network
maximum_tensor = maximum(encodeds)
predictions = Dense(num_classes, activation='softmax',
name='final_fc')(maximum_tensor)
return Model(inputs=inputs, outputs=predictions)
def channel_spatial_squeeze_excite(input, ratio=16):
''' Create a spatial squeeze-excite block
Args:
input: input tensor
filters: number of output filters
Returns: a keras tensor
References
- [Squeeze and Excitation Networks](https://arxiv.org/abs/1709.01507)
- [Concurrent Spatial and Channel Squeeze & Excitation in Fully Convolutional Networks](https://arxiv.org/abs/1803.02579)
'''
cse = squeeze_excite_gap_block(input, ratio)
sse = squeeze_excite_gmp_block(input, ratio)
x = maximum([cse,
sse]) #Lots of funsion can be done line average, max etc.
return x
def create_model():
g = Input(shape=(DIMS, ), dtype=np.float, name='g')
#lstm = LSTM(300)(g)
#intermediate = Dense(DIMS)(g)
lt = Dense(DIMS, activation='sigmoid', name='lt')(g)
#lt = Lambda(normalize, name='lt')(lt_raw)
dpos = Input(shape=(DIMS, ), dtype=np.float, name='dpos')
dneg = Input(shape=(DIMS, ), dtype=np.float, name='dneg')
ld = Dense(DIMS, activation='softmax', name='ld')
ldpos = ld(dpos)
ldneg = ld(dneg)
dot = Dot(axes=1)
lspos = dot([lt, ldpos])
lsneg = dot([lt, ldneg])
omega = Input(tensor=K.repeat_elements(K.constant([[1.0]]), BATCH_SIZE, 0),
name='omega')
zero = Input(tensor=K.repeat_elements(K.constant([[0.0]]), BATCH_SIZE, 0),
name='zero')
cost = maximum([zero, add([lsneg, subtract([omega, lspos])])],
name='val_loss')
#def loss(y_true, y_pred):
# return y_pred
model = Model(inputs=[g, dpos, dneg, omega, zero], outputs=[cost])
#optimizer = optimizers.Adam()
# GRU, loss = 1199
#optimizer = optimizers.Adam(lr=0.0001)
model.compile(optimizer='adam',
loss='mean_squared_error',
metrics=['mae', 'acc'])
print(model.summary())
return model
def nn_model3(train_x,train_y):
"""建立第二个五层的神经网络"""
inputs=Input(shape=(train_x.shape[1],))
x1 = Dense(40, activation='relu')(inputs)
x1=BatchNormalization()(x1)
x2 = Dense(40, activation='tanh')(inputs)
x2=BatchNormalization()(x2)
x=maximum([x1,x2])
x = Dense(20, activation='sigmoid')(x)
predictions = Dense(10, activation='softmax')(x)
model = Model(inputs=inputs, outputs=predictions)
model.summary()
model.compile(optimizer=#Adam(lr=0.001, epsilon=1e-09, decay=0.0),
'rmsprop',
# "adam",
loss='categorical_crossentropy',
metrics=['accuracy'])
model.fit(train_x, train_y,epochs=5, batch_size=500, validation_split=0.0)
return model
def nn_model2(train_x, train_y):
"""建立第二个五层的神经网络"""
inputs = Input(shape=(train_x.shape[1], ))
x1 = Dense(500, activation='tanh')(inputs)
x1 = bn_prelu(x1)
x2 = Dense(500, activation='relu')(inputs)
x2 = bn_prelu(x2)
x = maximum([x1, x2])
x = Dense(50, activation='sigmoid')(x)
x = bn_prelu(x)
x = Dense(50, activation='relu')(x)
x = Dense(50, activation='linear')(x)
predictions = Dense(10, activation='softmax')(x)
model = Model(inputs=inputs, outputs=predictions)
model.compile(optimizer=Adam(lr=0.001, epsilon=1e-09, decay=0.0),
loss='categorical_crossentropy',
metrics=['accuracy'])
model.fit(train_x, train_y, epochs=2, batch_size=200, validation_split=0.0)
return model
def ensemble_models(models, input_shape, combination_mode='average', ensemble_name='ensemble'):
'''
Args:
models: List of keras models
input_shape: Tuple containing input shape in tf format (H, W, C)
combination_mode: The way probabilities will be joined. We support `average` and `maximum`
ensemble_name: The name of the model that will be returned
Returns: A model containing the ensemble of the `models` passed. Same `input_shape` will be used for all of them
'''
if not len(input_shape) == 3:
raise ValueError('Incorrect input shape, it should have 3 dimensions (H, W, C)')
input_shape = Input(input_shape)
combination_mode_options = ['average', 'maximum']
# Collect outputs of models in a list
models_output = []
for i, model in enumerate(models):
# Keras needs all the models to be named differently
model.name = 'model_' + str(i)
models_output.append(model(input_shape))
# Computing outputs
if combination_mode in combination_mode_options:
if combination_mode == 'average':
out = average(models_output)
elif combination_mode == 'maximum':
out = maximum(models_output)
# Build model from same input and outputs
ensemble = Model(inputs=input_shape, outputs=out, name=ensemble_name)
else:
raise ValueError('Incorrect combination mode selected, we only allow for `average` or `maximum`')
return ensemble
def create_model(nb_feats=25, emat=embedding_matrix):
"""Add little noise at MLP layer"""
VOCAB = len(word2ix)
EMBED_HIDDEN_SIZE = 300
MAX_LEN = 35
MAX_CHARLEN = 5
SENT_HIDDEN_SIZE = 100
ACTIVATION = 'elu'
RNN_HIDDEN_SIZE = 50
DP = 0.25
L2 = 4e-6
embed_word = Embedding(VOCAB,
EMBED_HIDDEN_SIZE,
weights=[emat],
input_length=MAX_LEN,
trainable=False)
embed_code = Embedding(len(code2Idx),
len(code2Idx),
input_length=MAX_LEN,
trainable=True)
translate = TimeDistributed(
Dense(units=SENT_HIDDEN_SIZE, activation=ACTIVATION))
encode = Bidirectional(recurrent.LSTM(units=RNN_HIDDEN_SIZE,
return_sequences=False,
kernel_initializer='glorot_uniform',
dropout=DP,
recurrent_dropout=DP),
name='my_lstm')
# input defined: 8 tensors
seq_title = Input(shape=(MAX_LEN, ), dtype='int32') # title
seq_title_code = Input(shape=(MAX_LEN, ), dtype='int32')
seq_title_char = Input(shape=(MAX_LEN, MAX_CHARLEN), dtype='int32')
seq_cat = Input(shape=(MAX_LEN, ), dtype='int32') # joint cats
seq_cat_code = Input(shape=(MAX_LEN, ), dtype='int32')
seq_cat_char = Input(shape=(MAX_LEN, MAX_CHARLEN), dtype='int32')
dense_input = Input(shape=(nb_feats, ), dtype='float32')
# char
charem_full = create_charem()
# rnn encode
seq = embed_word(seq_title)
seq = Dropout(DP)(seq)
seq = translate(seq)
code = embed_code(seq_title_code)
char = charem_full(seq_title_char)
seq = concatenate([seq, code, char])
seq = encode(seq)
seq3 = embed_word(seq_cat)
seq3 = Dropout(DP)(seq3)
seq3 = translate(seq3)
code3 = embed_code(seq_cat_code)
char3 = charem_full(seq_cat_char)
seq3 = concatenate([seq3, code3, char3])
seq3 = encode(seq3)
# dense
den = BatchNormalization()(dense_input)
den = Dense(100, activation=ACTIVATION)(den)
den = Dropout(DP)(den)
#joint1: LOGLOSS vs RMSE
joint = concatenate([seq, seq3, den])
joint = Dense(units=150,
activation=ACTIVATION,
kernel_regularizer=l2(L2) if L2 else None,
kernel_initializer='he_normal')(joint)
joint = PReLU()(joint)
joint = Dropout(DP)(joint)
joint = BatchNormalization()(joint)
joint = maximum([
Dense(units=100,
activation=ACTIVATION,
kernel_regularizer=l2(L2) if L2 else None,
kernel_initializer='he_normal')(joint) for _ in range(5)
])
joint = PReLU()(joint)
joint = Dropout(DP)(joint)
joint = BatchNormalization()(joint)
score1 = Dense(units=1,
activation='sigmoid',
kernel_regularizer=l2(L2) if L2 else None,
kernel_initializer='he_normal',
name='logloss')(joint)
score2 = Dense(units=1,
activation='sigmoid',
kernel_regularizer=l2(L2) if L2 else None,
kernel_initializer='he_normal',
name='mse')(joint)
# plug all in one
model2 = Model(inputs=[
seq_title, seq_title_code, seq_title_char, seq_cat, seq_cat_code,
seq_cat_char, dense_input
],
outputs=[score1, score2])
model2.compile(optimizer='nadam', loss={'logloss': 'binary_crossentropy', 'mse': 'mean_squared_error'}, \
loss_weights={'logloss': 0.5, 'mse': 0.5},
metrics=[rmse_keras])
return model2
strides=(1, 1),
padding='valid')(conv_0)
#conv_1 = Conv2D(filters1, kernel_size=(filter_sizes[0], 100), padding='valid', kernel_initializer='normal', activation=newacti)(maxpool_0)
#maxpool_1 = MaxPool2D(pool_size=(sequence_length - filter_sizes[0] +1, 1), strides=(1,1), padding='valid')(conv_1)
gru = Reshape((sequence_length, embedding_vecor_length))(e)
gru = GRU(gru_output_size,
return_sequences=True,
dropout=0.2,
recurrent_dropout=0.2)(gru)
gru = GRU(gru_output_size)(gru)
gru = Reshape((1, 1, gru_output_size))(gru)
merge = maximum([maxpool_0, gru])
flatten = Flatten()(merge)
dropout = Dropout(0.5)(flatten)
output = Dense(nclass, activation='softmax')(dropout)
model = Model(inputs=visible, outputs=output)
model.compile(loss='binary_crossentropy',
optimizer='adam',
metrics=['accuracy'])
print(model.summary())
model.fit(padded_docs,
y_train,
epochs=1,
verbose=0,
nb_epoch=3,
batch_size=64,
validation_data=(vpadded_docs, y_valid))
STOP_LAYER = 101
# Form CNN1 and truncate after 101 layers
cnn1 = applications.ResNet50(include_top=False, input_shape=(224, 224, 3))
cnn1 = Model(cnn1.input, cnn1.layers[STOP_LAYER].output)
# Create 12 Input instances
inputs = []
processed_inputs = []
for i in range(0, 12):
temp = Input(shape=(224, 224, 3), name='input_' + str(i))
inputs.append(temp)
processed_inputs.append(cnn1(temp))
# Get maximum tensor
max = layers.maximum(processed_inputs)
# Create 2 pairs of a convultional layer and a batch normalization layer
x = Conv2D(filters=16, kernel_size=[5, 5], input_shape=(224, 224, 3))(max)
x = BatchNormalization()(x)
x = Conv2D(filters=16, kernel_size=[5, 5], input_shape=(224, 224, 3))(x)
x = BatchNormalization()(x)
x = (Flatten(input_shape=(224, 224, 3)))(x)
x = Dense(40, activation='softmax')(x)
model = Model(input=inputs, output=x)
# Set the first p fraction of layers to non-trainable
p = 0.7
img1 = Input(shape=(299, 299, 3), name='img_1')
feature1 = base_model1(img1)
feature2 = base_model2(img1)
# let's add a fully-connected layer
category_predict1 = Dense(100, activation='softmax',
name='ctg_out_1')(Dropout(0.5)(feature1))
category_predict2 = Dense(100, activation='softmax',
name='ctg_out_2')(Dropout(0.5)(feature2))
category_predict = Dense(100, activation='softmax',
name='ctg_out')(concatenate([feature1, feature2]))
max_category_predict = maximum([category_predict1, category_predict2])
model = Model(inputs=[img1],
outputs=[
category_predict1, category_predict2, category_predict,
max_category_predict
])
# model.save('dog_xception.h5')
plot_model(model, to_file='single_model.png')
# first: train only the top layers (which were randomly initialized)
# i.e. freeze all convolutional InceptionV3 layers
for layer in base_model1.layers:
layer.trainable = False
for layer in base_model2.layers:
def get_two_path_cascade_mf(stream_model,
n_feature_maps=2,
mode=None,
summary=False,
maxout=False,
dropout=False):
K.clear_session()
k_f = PATCH_HEIGHT - k_g + 1
stream_model = stream_model(2 * PATCH_HEIGHT - d_mf,
PATCH_HEIGHT - d_mf,
mode=mode,
n_feature_maps=n_feature_maps,
maxout=maxout,
dropout=dropout)
stream_model.trainable = False
stream_model.load_weights(
f'mf_stream_{mode}.h5' if mode is not None else 'mf_stream.h5')
if mode in ['ct', 'pet']:
x = Input(shape=(2 * PATCH_HEIGHT - d_mf, 2 * PATCH_WIDTH - d_mf, 1))
model_input = x
stream_output = stream_model(x)
else:
ct_input = Input(shape=(2 * PATCH_HEIGHT - d_mf,
2 * PATCH_WIDTH - d_mf, 1))
pet_input = Input(shape=(2 * PATCH_HEIGHT - d_mf,
2 * PATCH_WIDTH - d_mf, 1))
model_input = [ct_input, pet_input]
x = concatenate(model_input, axis=-1)
stream_output = stream_model([ct_input, pet_input])
h = (PATCH_HEIGHT - d_mf) // 2
w = (PATCH_WIDTH - d_mf) // 2
trim = d_mf % 2 == 1
x = Lambda(lambda x: x[:, h + trim:-h, w + trim:-w, :])(x)
if maxout:
conv1_local = maximum(
[Conv2D(64, (k_1, k_1))(x) for _ in range(n_feature_maps)])
else:
conv1_local = Conv2D(64, (k_1, k_1), activation='relu')(x)
pool1_local = MaxPooling2D((p_1, p_1), strides=(1, 1))(conv1_local)
if dropout:
pool1_local = Dropout(0.2)(pool1_local)
if maxout:
conv2_local = maximum([
Conv2D(64, (k_2, k_2))(pool1_local) for _ in range(n_feature_maps)
])
else:
conv2_local = Conv2D(64, (k_2, k_2), activation='relu')(pool1_local)
pool2_local = MaxPooling2D((p_2, p_2), strides=(1, 1))(conv2_local)
if dropout:
pool2_local = Dropout(0.2)(pool2_local)
if maxout:
conv1_global = maximum(
[Conv2D(160, (k_g, k_g))(x) for _ in range(n_feature_maps)])
else:
conv1_global = Conv2D(160, (k_g, k_g), activation='relu')(x)
if dropout:
conv1_global = Dropout(0.2)(conv1_global)
combine = concatenate([pool2_local, conv1_global, stream_output], axis=-1)
output = Conv2D(1, (k_f, k_f), activation='sigmoid')(combine)
output = Reshape((1, ))(output)
model = Model(inputs=model_input, outputs=output)
model.compile(optimizer=SGD(lr=1e-3,
decay=1e-6,
momentum=0.9,
nesterov=True),
loss='binary_crossentropy',
metrics=['accuracy'])
if summary:
model.summary()
return model
def get_stream_model(big_patch_dim,
small_patch_dim,
n_feature_maps=2,
mode=None,
summary=False,
maxout=False,
dropout=False):
K.clear_session()
k_f = big_patch_dim - k_g + 1
if mode in ['ct', 'pet']:
x = Input(shape=(big_patch_dim, big_patch_dim, 1))
model_input = x
else:
ct_input = Input(shape=(big_patch_dim, big_patch_dim, 1))
pet_input = Input(shape=(big_patch_dim, big_patch_dim, 1))
model_input = [ct_input, pet_input]
x = concatenate(model_input, axis=-1)
if maxout:
conv1_local = maximum(
[Conv2D(64, (k_1, k_1))(x) for _ in range(n_feature_maps)])
else:
conv1_local = Conv2D(64, (k_1, k_1), activation='relu')(x)
pool1_local = MaxPooling2D((p_1, p_1), strides=(1, 1))(conv1_local)
if dropout:
pool1_local = Dropout(0.2)(pool1_local)
if maxout:
conv2_local = maximum([
Conv2D(64, (k_2, k_2))(pool1_local) for _ in range(n_feature_maps)
])
else:
conv2_local = Conv2D(64, (k_2, k_2), activation='relu')(pool1_local)
pool2_local = MaxPooling2D((p_2, p_2), strides=(1, 1))(conv2_local)
if dropout:
pool2_local = Dropout(0.2)(pool2_local)
if maxout:
conv1_global = maximum(
[Conv2D(160, (k_g, k_g))(x) for _ in range(n_feature_maps)])
else:
conv1_global = Conv2D(160, (k_g, k_g), activation='relu')(x)
if dropout:
conv1_global = Dropout(0.2)(conv1_global)
#combine = Flatten()(concatenate([pool2_local, conv1_global], axis=-1))
#output = Dense(small_patch_dim * small_patch_dim, activation='sigmoid')(combine)
combine = concatenate([pool2_local, conv1_global], axis=-1)
output = Conv2D(small_patch_dim * small_patch_dim, (k_f, k_f),
activation='sigmoid')(combine)
if small_patch_dim > 1:
output = Reshape((small_patch_dim, small_patch_dim, 1))(output)
else:
output = Reshape((1, ))(output)
model = Model(inputs=model_input, outputs=output)
model.compile(optimizer=SGD(lr=1e-3,
decay=1e-6,
momentum=0.9,
nesterov=True),
loss='binary_crossentropy',
metrics=['accuracy'])
if summary:
model.summary()
return model
def build_model(self,
rnn_layer_sizes=np.array([20, 20, 20]),
dense_layer_parallel_sizes=np.array([10, 20]),
dense_layer_sequential_sizes=np.array([32, 20]),
dropout_prob_rnn=0.3,
dropout_prob_dense=0.3,
lambda_reg_rnn=0.001,
lambda_reg_dense=0.001,
merge='multiply'):
self.rnn_layer_sizes = rnn_layer_sizes
self.dense_layer_parallel_sizes = dense_layer_parallel_sizes
self.dense_layer_sequential_sizes = dense_layer_sequential_sizes
self.dropout_prob_rnn = dropout_prob_rnn
self.dropout_prob_dense = dropout_prob_dense
self.lambda_reg_rnn = lambda_reg_rnn
self.lambda_reg_dense = lambda_reg_dense
self.merge = merge
if dense_layer_parallel_sizes[-1] != rnn_layer_sizes[-1]:
print(
'Dimensions of last layers of RNN and of parallel dense network must agree!'
)
return
# define inputs
tracks_input = Input(self.input_shape_tracks,
dtype='float32',
name='tracks_input')
session_input = Input(self.input_shape_sessions,
dtype='float32',
name='session_input')
# RNN side
x_rnn = LSTM(self.rnn_layer_sizes[0],
return_sequences=True,
kernel_regularizer=l2(self.lambda_reg_rnn))(tracks_input)
for i in range(1, self.rnn_layer_sizes.size):
x_rnn = LSTM(self.rnn_layer_sizes[i],
return_sequences=True,
kernel_regularizer=l2(self.lambda_reg_rnn))(x_rnn)
x_rnn = BatchNormalization()(x_rnn)
out_rnn = Dropout(self.dropout_prob_rnn)(x_rnn)
# dense side
x_fc = Dense(self.dense_layer_parallel_sizes[0],
activation='relu',
kernel_regularizer=l2(
self.lambda_reg_dense))(session_input)
for i in range(1, self.dense_layer_parallel_sizes.size):
x_fc = Dense(self.dense_layer_parallel_sizes[i],
activation='relu',
kernel_regularizer=l2(self.lambda_reg_dense))(x_fc)
x_fc = BatchNormalization()(x_fc)
out_fc = Dropout(self.dropout_prob_dense)(x_fc)
x = []
# merge RNN and dense side
if self.merge == 'multiply':
x = multiply([out_rnn, out_fc])
elif self.merge == 'add':
out_fc = RepeatVector(20)(out_fc)
x = add([out_rnn, out_fc])
elif self.merge == 'concatenate':
out_fc = RepeatVector(20)(out_fc)
x = concatenate([out_rnn, out_fc], axis=-1)
elif self.merge == 'maximum':
out_fc = RepeatVector(20)(out_fc)
x = maximum([out_rnn, out_fc])
else:
print(
'Choose proper merge variation: multiply, add, concatenate or maximum'
)
for i in range(self.dense_layer_sequential_sizes.size - 1):
x = Dense(self.dense_layer_sequential_sizes[i],
activation='relu',
kernel_regularizer=l2(self.lambda_reg_dense))(x)
x = Dense(self.dense_layer_sequential_sizes[-1],
activation='linear',
kernel_regularizer=l2(self.lambda_reg_dense))(x)
output = Reshape(self.output_shape, name='output')(x)
# create model and compile it
self.model = K_Model(inputs=[tracks_input, session_input],
outputs=[output])
def build(self):
print 'Latent dimensions: ' + str(self.latent_dim)
encoders = [self.encoder_maker(m) for m in self.input_modalities]
ind_emb = [lr for (input, lr) in encoders]
self.org_ind_emb = [lr for (input, lr) in encoders]
self.inputs = [input for (input, lr) in encoders]
#apply spatial transformer
# if self.spatial_transformer:
# print 'Adding a spatial transformer layer'
# input_shape = (self.latent_dim, self.H, self.W)
# tpn = tpn_maker(input_shape)
# mod1 = ind_emb[0]
# aligned_ind_emb = [mod1]
# for mod in ind_emb[1:]:
# aligned_mod = merge([tpn([mod1, mod]), mod], mode=STMerge, output_shape=input_shape)
# aligned_ind_emb.append(aligned_mod)
# ind_emb = aligned_ind_emb
if self.spatial_transformer:
print 'Adding a spatial transformer layer'
input_shape = (self.latent_dim, self.H, self.W)
# input_shape = (self.H, self.W, self.latent_dim)
tpn = tpn_maker(input_shape)
mod1 = ind_emb[0]
# mod1= K.permute_dimensions(ind_emb[0],(0,2,3,1))
aligned_ind_emb = [mod1]
for mod in ind_emb[1:]:
aligned_mod = merge([tpn([mod1, mod]), mod],
mode=STMerge,
output_shape=input_shape)
aligned_ind_emb.append(aligned_mod)
ind_emb = aligned_ind_emb
if self.common_merge == 'hemis':
self.all_emb = self.hemis(ind_emb)
elif self.common_merge == 'cbam_block':
weighted_rep = self.cbam_block(ind_emb, 'hjh')
self.all_emb = ind_emb + [weighted_rep]
else:
assert self.common_merge == 'max' or self.common_merge == 'ave' or self.common_merge == 'rev_loss'
print 'Fuse latent representations using ' + str(self.common_merge)
if self.common_merge == 'ave':
weighted_rep = average(
ind_emb,
name='combined_em') if len(self.inputs) > 1 else ind_emb[0]
else:
weighted_rep = maximum(
ind_emb,
name='combined_em') if len(self.inputs) > 1 else ind_emb[0]
self.all_emb = ind_emb + [weighted_rep]
self.decoders = [self.decoder_maker(m) for m in self.output_modalities]
outputs = get_decoder_outputs(self.output_modalities, self.decoders,
self.all_emb)
# this is for minimizing the distance between the individual embeddings
# outputs += self.get_embedding_distance_outputs(self.all_emb)
print 'all outputs: ', [o.name for o in outputs]
#out_dict = {'decoder_MASK' : mae }
# out_dict = {'em_%d_dec_%s' % (emi, dec): mae for emi in range(self.num_emb) for dec in self.output_modalities}
out_dict = {}
out_dict['em_0_dec_FLAIR'] = mae2
out_dict['em_1_dec_FLAIR'] = mae2
out_dict['em_2_dec_FLAIR'] = mae2
# out_dict['em_3_dec_FLAIR'] = mae
# out_dict['em_0_dec_MASK'] = dice_coef_loss
# out_dict['em_1_dec_MASK'] = dice_coef_loss
# out_dict['em_2_dec_MASK'] = dice_coef_loss_adj
# out_dict['em_3_dec_MASK'] = dice_coef_loss
# out_dict['em_0_dec_MASK'] = dice_loss
# out_dict['em_1_dec_MASK'] = dice_loss
# out_dict['em_2_dec_MASK'] = dice_loss
# out_dict['em_3_dec_MASK'] = dice_loss_mask
get_indiv_weight = lambda mod: self.output_weights[
mod] if self.ind_outs else 0.0
get_fused_weight = lambda mod: self.output_weights[
mod] if self.fuse_outs else 0.0
loss_weights = {}
for dec in self.output_modalities:
for emi in range(self.num_emb - 1):
loss_weights['em_%d_dec_%s' %
(emi, dec)] = get_indiv_weight(dec)
loss_weights['em_%d_dec_%s' %
(self.num_emb - 1, dec)] = get_fused_weight(dec)
if len(self.inputs) > 1:
if self.common_merge == 'rev_loss':
out_dict['em_concat'] = mae
# else:
# out_dict['em_concat'] = embedding_distance
# loss_weights['em_concat'] = self.output_weights['concat']
# out_dict['em_fused'] = embedding_distance
# loss_weights['em_fused'] = self.output_weights['concat']
print 'output dict: ', out_dict
print 'loss weights: ', loss_weights
self.model = Model(input=self.inputs, output=outputs)
self.model.summary()
self.model.compile(optimizer=Adam(lr=0.0001),
loss=out_dict,
loss_weights=loss_weights)
conv_0 = Conv2D(filters1,
kernel_size=(filter_sizes[0], 100),
padding='valid',
kernel_initializer='normal',
activation=newacti)(e)
maxpool_0 = MaxPool2D(pool_size=(sequence_length - filter_sizes[0] + 1, 1),
strides=(1, 1),
padding='valid')(conv_0)
#maxpool_0=Flatten()(maxpool_0)
#maxpool_0=Reshape((1,gru_output_size))(maxpool_0)
gru = Reshape((sequence_length, embedding_vecor_length))(embedding)
gru = GRU(gru_output_size,
return_sequences=True,
dropout=0.2,
recurrent_dropout=0.2)(gru)
merge2 = maximum([maxpool_0, gru])
merge = Reshape((sequence_length, filters1))(merge2)
gru1 = GRU(gru_output_size,
return_sequences=True,
dropout=0.2,
recurrent_dropout=0.2)(merge)
#gru1=Flatten()(gru1)
gru1 = MaxPooling1D(pool_size=8)(gru1)
conv_1 = Conv1D(filters1,
kernel_size,
padding='valid',
activation=newacti,
strides=1)(merge)
#gru2=GRU(gru_output_size, return_sequences=True, dropout=0.2, recurrent_dropout=0.2)(conv_1)
name='xception')
base_model2 = InceptionV3(include_top=True, weights='imagenet')
base_model2 = Model(inputs=[base_model2.input],
outputs=[base_model2.get_layer('avg_pool').output],
name='inceptionv3')
img1 = Input(shape=(224, 224, 3), name='img_1')
feature1 = base_model1(img1)
feature2 = base_model2(img1)
categorical_predic1 = Dense(100, activation='softmax',
name='ctg_out_1')(Dropout(0.5)(feature1))
categorical_predic2 = Dense(100, activation='softmax',
name='ctg_out_2')(Dropout(0.5)(feature2))
categorcial_predict = Dense(100, activation='softmax',
name='ctg_out')(concatenate([feature1, feature2]))
max_category_predict = maximum([categorical_predic1, categorical_predic2])
model = Model(inputs=[img1],
outputs=[
categorical_predic1, categorical_predic2,
categorcial_predict, max_category_predict
])
print(model.summary())
#maxpool_1 = MaxPool2D(pool_size=(sequence_length - filter_sizes[0] +1, 1), strides=(1,1), padding='valid')(conv_1)
gru = Reshape((sequence_length, embedding_vecor_length))(e)
gru = GRU(gru_output_size,
return_sequences=True,
dropout=0.2,
recurrent_dropout=0.2)(gru)
#gru=Reshape((1,gru_output_size))(gru)
#gru=RepeatVector(sequence_length)(gru)
gru = Conv1D(64, kernel_size, padding='valid', activation=newacti,
strides=1)(gru)
gru = MaxPooling1D(pool_size=pool_size)(gru)
#gru=Reshape((1,1,gru_output_size))(gru)
merge = maximum([maxpool_0, gru])
conv_1 = Conv1D(64,
kernel_size,
padding='valid',
activation=newacti,
strides=1)(merge)
maxpool_1 = MaxPooling1D(pool_size=pool_size)(conv_1)
#conv_1 = Conv2D(filters1, kernel_size=(filter_sizes[0], 100), padding='valid', kernel_initializer='normal', activation=newacti)(maxpool_0)
#maxpool_1 = MaxPool2D(pool_size=(sequence_length - filter_sizes[0] +1, 1), strides=(1,1), padding='valid')(conv_1)
#gru=Reshape((sequence_length,embedding_vecor_length))(merge)
gru1 = GRU(gru_output_size,
return_sequences=True,
#define network
input = Input(shape=input_shape)
#hidden layer 0
left = Conv2D(filters=96,
kernel_size=(8, 8),
activation='relu',
padding='same',
name='h0_l')(input)
right = Conv2D(filters=96,
kernel_size=(8, 8),
activation='relu',
padding='same',
name='h0_r')(input)
h0 = maximum([left, right])
#apply max pooling
h0 = MaxPooling2D(pool_size=(4, 4), strides=(2, 2), name='pool0')(h0)
#hidden layer 1
left = Conv2D(filters=192,
kernel_size=(8, 8),
activation='relu',
padding='same',
name='h1_l')(h0)
right = Conv2D(filters=192,
kernel_size=(8, 8),
activation='relu',
padding='same',
name='h1_r')(h0)
h1 = maximum([left, right])
