def build_siamese_net(encoder,
input_shape,
distance_metric='uniform_euclidean'):
assert distance_metric in ('uniform_euclidean', 'weighted_euclidean',
'uniform_l1', 'weighted_l1', 'dot_product',
'cosine_distance')
input_1 = layers.Input(input_shape)
input_2 = layers.Input(input_shape)
encoded_1 = encoder(input_1)
encoded_2 = encoder(input_2)
if distance_metric == 'weighted_l1':
embedded_distance = layers.Subtract()([encoded_1, encoded_2])
embedded_distance = layers.Lambda(lambda x: K.abs(x))(
embedded_distance)
output = layers.Dense(1, activation='sigmoid')(embedded_distance)
elif distance_metric == 'uniform_euclidean':
embedded_distance = layers.Subtract(name='subtract_embeddings')(
[encoded_1, encoded_2])
embedded_distance = layers.Lambda(
lambda x: K.sqrt(K.sum(K.square(x), axis=-1, keepdims=True)),
name='euclidean_distance')(embedded_distance)
output = layers.Dense(1, activation='sigmoid')(embedded_distance)
elif distance_metric == 'cosine_distance':
raise NotImplementedError
else:
raise NotImplementedError
siamese = Model(inputs=[input_1, input_2], outputs=output)
return siamese
def compare_and_score(self, left, right, ent, feats):
""" Final layer of the compiled model
Concatenates several comparisons between the vectors of left and right
contexts and the entity vector.
Final dense layer takes all of these comparisons, and the final feature
vector, and outputs a binary prediction.
"""
comparisons = []
left_dot = layers.Dot(axes=1, normalize=True)([left, ent])
right_dot = layers.Dot(axes=1, normalize=True)([right, ent])
comparisons += [left_dot, right_dot]
left_diff = layers.Subtract()([left, ent])
right_diff = layers.Subtract()([right, ent])
comparisons += [left_diff, right_diff]
left_diff_sq = layers.Multiply()([left_diff, left_diff])
right_diff_sq = layers.Multiply()([right_diff, right_diff])
comparisons += [left_diff_sq, right_diff_sq]
left_mult = layers.Multiply()([left, ent])
right_mult = layers.Multiply()([right, ent])
comparisons += [left_mult, right_mult]
if feats is not None:
comparisons.append(feats)
comparisons_concat = layers.Concatenate(axis=1)(comparisons)
out = self.reduce_layer(comparisons_concat)
return out
def output_model_b(inp, params, output_shape, interpretable, name=''):
# h = params.get('output').get('h', output_shape)
if interpretable:
out = inp
else:
out_inp = layers.InputLayer(input_shape=inp.get_shape().as_list()[1:],
name=f'out_{name}_inp')
out = out_inp.output
filters = params['output'].get('filters', None)
for i, f in enumerate(filters):
out = layers.Dense(f, name=f'out_{name}_dense{i}')(out)
out = PReLU2(name=f'out_{name}_dense{i}_relu')(out)
out = layers.BatchNormalization(name=f'out_{name}_dense{i}_bn')(out)
out = layers.Flatten(name=f'out_{name}_flatten')(out)
out_p = layers.Dense(output_shape, name=f'out_{name}_out_pos')(out)
out_p = PReLU2(name=f'out_{name}_out_pos_relu')(out_p)
out_n = layers.Lambda(lambda x: x * -1, name=f'out_{name}_out_neg0')(out)
out_n = layers.Dense(
output_shape,
# activation='relu',
name=f'out_{name}_out_neg')(out_n)
out_n = PReLU2(name=f'out_{name}_out_neg_relu')(out_n)
out = layers.Subtract(name=f'out_{name}_out')([out_p, out_n])
out = layers.Reshape(target_shape=out.get_shape().as_list()[1:] + [1],
name=f'out_{name}_reshape')(out)
if interpretable:
return out
else:
return models.Model(inputs=out_inp.input,
outputs=out,
name=f'out_{name}')(inp)
def factorization_machine(f_size, k_latent=5, embedding_reg=0.0005):
def get_embed(x_input, x_size, k_latent):
if x_size > 0: #category
embed = Embedding(
x_size, k_latent,
embeddings_regularizer=l2(embedding_reg))(x_input)
embed = Flatten()(embed)
else:
embed = Dense(k_latent,
kernel_regularizer=l2(embedding_reg))(x_input)
#embed = Dense(k_latent)(x_input)
return embed
dim_input = len(f_size)
input_x = [Input(shape=(1, )) for i in range(dim_input)]
biases = [get_embed(x, size, 1) for (x, size) in zip(input_x, f_size)]
factors = [
get_embed(x, size, k_latent) for (x, size) in zip(input_x, f_size)
]
s = Add()(factors)
diffs = [layers.Subtract()([s, x]) for x in factors]
dots = [layers.Dot(axes=1)([d, x]) for d, x in zip(diffs, factors)]
dots = Add()(dots)
dots_sum = layers.Lambda(lambda x: x / 2)(dots)
biases_sum = Add()(biases)
x = Add()([dots_sum, biases_sum])
model = Model(inputs=input_x, outputs=x)
#output_f = factors + biases
#model_features = Model(inputs=input_x, outputs=output_f)
#model, model_features = build_model_1(X_train, f_size)
return model
def fconv(inputs, contract, stride, filters, alpha=1):
in_nfilters = backend.int_shape(inputs)[-1]
out_nfilters = int(alpha * filters)
channel_axis = 1 if backend.image_data_format() == 'channels_first' else -1
# Contract
x = layers.Conv2D(int(in_nfilters * contract),
kernel_size=(5, 5),
strides=5,
padding='same',
use_bias=False,
activation=None)(inputs)
x = layers.Activation('tanh')(x)
# Expand
x = layers.Conv2DTranspose(in_nfilters,
kernel_size=(5, 5),
strides=5,
padding='same',
use_bias=False,
activation=None)(x)
x = layers.Activation('tanh')(x)
x = layers.Subtract()([inputs, x])
# Reduce
x = layers.Conv2D(out_nfilters,
kernel_size=(5, 5),
padding='same',
strides=stride,
use_bias=False,
activation=None)(x)
x = layers.Activation('linear')(x)
# This layers does not have an activation function, it is linear
return x
def test_merge_subtract():
i1 = layers.Input(shape=(4, 5))
i2 = layers.Input(shape=(4, 5))
i3 = layers.Input(shape=(4, 5))
i4 = layers.Input(shape=(3, 5))
o = layers.subtract([i1, i2])
assert o._keras_shape == (None, 4, 5)
model = models.Model([i1, i2], o)
subtract_layer = layers.Subtract()
o2 = subtract_layer([i1, i2])
assert subtract_layer.output_shape == (None, 4, 5)
x1 = np.random.random((2, 4, 5))
x2 = np.random.random((2, 4, 5))
out = model.predict([x1, x2])
assert out.shape == (2, 4, 5)
assert_allclose(out, x1 - x2, atol=1e-4)
assert subtract_layer.compute_mask([i1, i2], [None, None]) is None
assert np.all(
K.eval(
subtract_layer.compute_mask(
[i1, i2], [K.variable(x1), K.variable(x2)])))
# Test invalid use case
with pytest.raises(ValueError):
subtract_layer.compute_mask([i1, i2], x1)
with pytest.raises(ValueError):
subtract_layer.compute_mask(i1, [None, None])
with pytest.raises(ValueError):
subtract_layer([i1, i2, i3])
with pytest.raises(ValueError):
subtract_layer([i1])
def _metric_network(input, is_siamese):
input_a, input_b = input
if is_siamese:
base_network = _feature_tower(input_shape)
processed_a = base_network(input_a)
processed_b = base_network(input_b)
else:
base_network_0 = _feature_tower(input_shape)
base_network_1 = _feature_tower(input_shape)
processed_a = base_network_0(input_a)
processed_b = base_network_1(input_b)
processed = layers.concatenate([processed_a, processed_b], axis=-1)
distance = layers.Subtract()(processed)
fc = layers.Dense(512, activation='relu')(distance)
if dropout > 0.:
fc = layers.Dropout(dropout)(fc)
fc = layers.Dense(512, activation='relu')(fc)
if dropout > 0.:
fc = layers.Dropout(dropout)(fc)
fc = layers.Dense(2, activation='relu')(fc)
distance = layers.Lambda(euclidean_distance,
output_shape=eucl_dist_output_shape)(
layers.concatenate([processed_a, processed_b],
axis=-1))
return distance, Model([input_a, input_b], fc)
def init_model(self, train, std=0.01):
predict = self.get_pmodel()
user = kl.Input((1, ),
dtype=self.intX) #, batch_shape=(self.batch,1) )
item = kl.Input((1, ),
dtype=self.intX) #, batch_shape=(self.,self.steps) )
item_neg = kl.Input(
(1, ), dtype=self.intX) #, batch_shape=(self.,self.steps) )
if self.include_artist:
artist = kl.Input((1, ),
dtype=self.intX) #, batch_shape=(self.batch,1) )
artist_neg = kl.Input(
(1, ), dtype=self.intX) #, batch_shape=(self.batch,1) )
inputs_pos = [user, item] #+ [artist]
inputs_neg = [user, item_neg] #+ [artist]
if self.include_artist:
inputs_pos += [artist]
inputs_neg += [artist_neg]
res = predict(inputs_pos)
res_neg = predict(inputs_neg)
diff = kl.Subtract()([res, res_neg])
diff = kl.Activation('sigmoid')(diff)
inputs = [user, item, item_neg] #+ [artist]
if self.include_artist:
inputs += [artist, artist_neg]
outputs = [diff]
model = km.Model(inputs, outputs)
if self.optimizer == 'adam':
opt = keras.optimizers.Adam(lr=self.learning_rate)
elif self.optimizer == 'adagrad':
opt = keras.optimizers.Adagrad(lr=self.learning_rate)
elif self.optimizer == 'nadam':
opt = keras.optimizers.Nadam(lr=self.learning_rate)
elif self.optimizer == 'adamax':
opt = keras.optimizers.Adamax(lr=self.learning_rate)
elif self.optimizer == 'adadelta':
opt = keras.optimizers.Adadelta(lr=self.learning_rate * 10)
model.compile(optimizer=opt, loss='binary_crossentropy')
plot_model(model, to_file='mlp_rank.png')
inputs = [user, item]
if self.include_artist:
inputs += [artist]
return model, predict
def build_siamese_net(encoder,
input_shape,
distance_metric='uniform_euclidean'):
assert distance_metric in ('uniform_euclidean', 'weighted_euclidean',
'uniform_l1', 'weighted_l1', 'dot_product',
'cosine_distance')
input_1 = layers.Input(input_shape)
input_2 = layers.Input(input_shape)
encoded_1 = encoder(input_1)
encoded_2 = encoder(input_2)
if distance_metric == 'weighted_l1':
# This is the distance metric used in the original one-shot paper
# https://www.cs.cmu.edu/~rsalakhu/papers/oneshot1.pdf
embedded_distance = layers.Subtract()([encoded_1, encoded_2])
embedded_distance = layers.Lambda(lambda x: K.abs(x))(
embedded_distance)
output = layers.Dense(1, activation='sigmoid')(embedded_distance)
elif distance_metric == 'uniform_euclidean':
# Simpler, no bells-and-whistles euclidean distance
# Still apply a sigmoid activation on the euclidean distance however
embedded_distance = layers.Subtract(name='subtract_embeddings')(
[encoded_1, encoded_2])
# Sqrt of sum of squares
embedded_distance = layers.Lambda(
lambda x: K.sqrt(K.sum(K.square(x), axis=-1, keepdims=True)),
name='euclidean_distance')(embedded_distance)
output = layers.Dense(1, activation='sigmoid')(embedded_distance)
elif distance_metric == 'cosine_distance':
raise NotImplementedError
# cosine_proximity = layers.Dot(axes=-1, normalize=True)([encoded_1, encoded_2])
# ones = layers.Input(tensor=K.ones_like(cosine_proximity))
# cosine_distance = layers.Subtract()([ones, cosine_proximity])
# output = layers.Dense(1, activation='sigmoid')(cosine_distance)
else:
raise NotImplementedError
siamese = Model(inputs=[input_1, input_2], outputs=output)
return siamese
def l1_distance_graph(P, T, feature_maps=128, name='Tx'):
T = KL.GlobalAveragePooling2D()(T)
T = KL.Lambda(lambda x: K.expand_dims(K.expand_dims(x, axis=1), axis=1))(T)
# T = KL.Lambda(lambda x: K.tile(T, [1, int(P.shape[1]), int(P.shape[2]), 1]))(T)
L1 = KL.Subtract()([P, T])
L1 = KL.Lambda(lambda x: K.abs(x))(L1)
D = KL.Concatenate()([P, L1])#KL.Concatenate()([P, T, L1])
if feature_maps:
D = KL.Conv2D(feature_maps, (1, 1), name='fpn_distance_' + name)(D)
return D
def _build_compute_scores(self):
'''
Compute scores
'''
img_ref = L.Input((64,64,3), name='input_ref')
img_A = L.Input((64,64,3), name='input_compare')
# feature extraction
A_multiscale_feature, A_last_layer_feature = self.extract_features(img_A)
ref_multiscale_feature, ref_last_layer_feature = self.extract_features(img_ref)
# feature difference
diff_A_ref_ms = L.Subtract() ([ref_multiscale_feature, A_multiscale_feature])
diff_A_ref_last = L.Subtract() ([ref_last_layer_feature, A_last_layer_feature])
# score computation
# fc1
fc1_A_ref = L.Dense(512, activation='relu', name='fc1') (diff_A_ref_ms)
dropout1_A_ref = L.Dropout(0.2) (fc1_A_ref)
# fc2
fc2_A_ref = L.Dense(1, name='fc2') (dropout1_A_ref)
multiply_const = L.Lambda(lambda x:tf.multiply(x, tf.constant(0.01))) (fc2_A_ref)
per_patch_score = L.Lambda(lambda x:tf.reshape(x, [-1])) (multiply_const)
### weighing subnetwork image A and ref
# fc1w
fc1_A_ref_w = L.Dense(512, activation='relu', name='fc1w') (diff_A_ref_last)
dropout1_A_ref_w = L.Dropout(0.2) (fc1_A_ref_w)
# fc2w
fc2_A_ref_w = L.Dense(1, name='fc2w') (dropout1_A_ref_w)
add_const = L.Lambda(lambda x:tf.add(x, tf.constant(0.000001))) (fc2_A_ref_w)
per_patch_weight = L.Lambda(lambda x:tf.reshape(x, [-1])) (add_const)
# weighted average of scores
product_score_weights_A = L.Multiply() ([per_patch_weight, per_patch_score])
norm_factor_A = L.Lambda(lambda x:tf.reduce_sum(x)) (per_patch_weight)
final_score_A = L.Lambda(lambda x:tf.divide(tf.reduce_sum(x[0]),x[1])) ([product_score_weights_A, norm_factor_A])
return Model([img_ref, img_A], [final_score_A, per_patch_score, per_patch_weight], name='compute_scores')
def blending_graph(fg_out, bg_out, fg_weights, network_name='fusion_'):
# bg_weights = KL.Subtract()([K.constant(K.ones_like(fg_weights)), fg_weights])
weighted_fg = KL.Multiply(name=network_name +
'fg_mul')([fg_out, fg_weights])
#temp_1 = KL.Lambda(lambda x: 1.0-x, name=network_name+'reverse_lambda_bg')(bg_out)
#temp_1 = KL.Add(name=network_name + 'reverse_lambda_bg')([bg_out, -bg_out])
#temp_2 = KL.Lambda(lambda x: 1.0-x, name=network_name+'reverse_lambda_blendingweight')(fg_weights)
#temp_2 = KL.Add(name=network_name + 'reverse_lambda_blendingweigh')([var, -fg_weights])
weighted_bg = KL.Multiply(name=network_name +
'bg_mul')([bg_out, fg_weights])
weighted_bg = KL.Subtract(name=network_name +
'addbg')([weighted_bg, bg_out])
weighted_bg = KL.Subtract(name=network_name +
'addfg')([weighted_bg, fg_weights])
#weighted_bg = KL.Multiply(name=network_name+'bg_mul')([temp_1, temp_2])
final_result = KL.Add(name=network_name +
'blending_output')([weighted_fg, weighted_bg])
# final_result=KL.Reshape([ ,960,640])(final_result)
return final_result
def complexized(x_re, x_im):
layer_re = layer(units, **kwargs)
layer_im = layer(units, **kwargs)
x_re_tmp = layers.Subtract()([layer_re(x_re), layer_im(x_im)])
x_im_tmp = layers.Add()([layer_re(x_im), layer_im(x_re)])
x_re, x_im = x_re_tmp, x_im_tmp
if activation is None:
return x_re, x_im
x_abs = layers.Lambda(
lambda x_cpx: K.sqrt(K.square(x_cpx[0]) + K.square(x_cpx[1])))(
[x_re, x_im])
x_act = activation(x_abs)
projection = layers.Lambda(lambda a: a[0] * a[1] / a[2])
x_re = projection([x_act, x_re, x_abs])
x_im = projection([x_act, x_im, x_abs])
return x_re, x_im
def cnn_model(input_shape, num_classes=1284):
"""CNN with backdoor"""
input = layers.Input(shape=input_shape)
# Same should be image without the backdoo
same1 = layers.Conv2D(6, (7, 7), padding="same", activation="relu")(input)
same2 = layers.Conv2D(12, (7, 7), padding="same", activation="relu")(same1)
same3 = layers.Conv2D(3, (7, 7), padding="same", activation="relu")(same2)
border = layers.Subtract()([input, same3])
concat = layers.Concatenate()([input, border])
# Rest of the CNN
c_layer1_5 = layers.Conv2D(12, (5, 5), padding="same",
activation="relu")(concat)
c_layer1_3 = layers.Conv2D(12, (3, 3), padding="same",
activation="relu")(concat)
c_layer1_1 = layers.Conv2D(12, (1, 1), padding="same",
activation="relu")(concat)
concat_1 = layers.Concatenate()([c_layer1_5, c_layer1_3, c_layer1_1])
max_pool1 = layers.Conv2D(36, (5, 5),
strides=2,
padding="same",
activation="relu")(concat_1)
c_layer2_5 = layers.Conv2D(64, (5, 5), padding="valid",
activation="relu")(max_pool1)
max_pool2 = layers.MaxPooling2D(pool_size=2, strides=2)(c_layer2_5)
c_layer3_5 = layers.Conv2D(128, (5, 5),
strides=2,
padding="same",
activation="relu")(max_pool2)
flatten = layers.Flatten()(c_layer3_5)
dense = layers.Dense(2048, activation='relu')(flatten)
dropout_2 = layers.Dropout(0.5)(dense)
output = layers.Dense(num_classes, activation='softmax')(dropout_2)
model = Model(inputs=input, outputs=[output, same])
return model
def createModel(modelName):
fp1 = layers.Input(shape=(90, 90, 1))
fp2 = layers.Input(shape=(90, 90, 1))
# ambas capas comparten pesos
inputs = layers.Input(shape=(90, 90, 1))
feat = layers.Conv2D(32, kernel_size=3, padding='same',
activation='relu')(inputs)
feat = layers.MaxPooling2D(pool_size=2)(feat)
feat = layers.Conv2D(32, kernel_size=3, padding='same',
activation='relu')(feat)
feat = layers.MaxPooling2D(pool_size=2)(feat)
feature_model = Model(inputs=inputs, outputs=feat, name='featureModel')
# modelos de features que comparten pesos
fp1_net = feature_model(fp1)
fp2_net = feature_model(fp2)
# hacemos un subtract de las features
net = layers.Subtract()([fp1_net, fp2_net])
net = layers.Conv2D(32, kernel_size=3, padding='same',
activation='relu')(net)
net = layers.MaxPooling2D(pool_size=2)(net)
net = layers.Flatten()(net)
net = layers.Dense(64, activation='relu')(net)
net = layers.Dense(1, activation='sigmoid')(net)
model = Model(inputs=[fp1, fp2], outputs=net, name=modelName)
model.compile(optimizer='adam',
loss='binary_crossentropy',
metrics=['acc'])
model.summary()
return model
def mog_loss_model(n_components, d_t):
"""
Create a Keras model that computes the loss of a mixture of Gaussians model on data.
Parameters
----------
n_components : int
The number of components in the mixture model
d_t : int
The number of dimensions in the output
Returns
-------
A Keras model that takes as inputs pi, mu, sigma, and t and generates a single output containing the loss.
"""
pi = L.Input((n_components, ))
mu = L.Input((n_components, d_t))
sig = L.Input((n_components, ))
t = L.Input((d_t, ))
# || t - mu_i || ^2
d2 = L.Lambda(lambda d: K.sum(K.square(d), axis=-1),
output_shape=(n_components, ))(
L.Subtract()([L.RepeatVector(n_components)(t), mu]))
# LL = C - log(sum(pi_i/sig^d * exp(-d2/(2*sig^2))))
# Use logsumexp for numeric stability:
# LL = C - log(sum(exp(-d2/(2*sig^2) + log(pi_i/sig^d))))
# TODO: does the numeric stability actually make any difference?
def make_logloss(d2, sig, pi):
return -K.logsumexp(
-d2 / (2 * K.square(sig)) + K.log(pi / K.pow(sig, d_t)), axis=-1)
ll = L.Lambda(lambda dsp: make_logloss(*dsp),
output_shape=(1, ))([d2, sig, pi])
m = Model([pi, mu, sig, t], [ll])
return m
def build_model(input_shape=(197, 197, 3)):
"""
Builds model for training.
:param input_shape: Tuple of input image size (height, width, channels) (default (197,197,3)).
:return: Returns keras model
"""
print('Loading base CNN model...')
cnn_base = ResNet50(weights='imagenet',
include_top=False,
input_shape=input_shape)
cnn_base.trainable = False
print('Building model...')
sample1_input = Input(shape=input_shape)
sample2_input = Input(shape=input_shape)
sample1_cnn = cnn_base(sample1_input)
sample2_cnn = cnn_base(sample2_input)
sample1_cnn_flatten = layers.Flatten()(sample1_cnn)
sample2_cnn_flatten = layers.Flatten()(sample2_cnn)
cnn_top = layers.Dense(1024, activation="relu")
sample1_cnn_top = cnn_top(sample1_cnn_flatten)
sample2_cnn_top = cnn_top(sample2_cnn_flatten)
diff_features = layers.Subtract()([sample1_cnn_top, sample2_cnn_top])
diff_features_norm = layers.BatchNormalization()(diff_features)
top1 = layers.Dense(512, activation='relu')(diff_features_norm)
top2 = layers.Dense(256, activation='relu')(top1)
answer = layers.Dense(1, activation='sigmoid')(top2)
model = Model([sample1_input, sample2_input], answer)
print(model.summary())
return model
def __init__(self, max_len, emb_train):
# Define hyperparameters
modname = FIXED_PARAMETERS["model_name"]
learning_rate = FIXED_PARAMETERS["learning_rate"]
dropout_rate = FIXED_PARAMETERS["dropout_rate"]
batch_size = FIXED_PARAMETERS["batch_size"]
max_words = FIXED_PARAMETERS["max_words"]
print("Loading data...")
genres_train, sent1_train, sent2_train, labels_train_, scores_train = load_sts_data(
FIXED_PARAMETERS["train_path"])
genres_dev, sent1_dev, sent2_dev, labels_dev_, scores_dev = load_sts_data(
FIXED_PARAMETERS["dev_path"])
print("Building dictionary...")
text = sent1_train + sent2_train + sent1_dev + sent2_dev
tokenizer = Tokenizer(num_words=max_words)
tokenizer.fit_on_texts(text)
word_index = tokenizer.word_index
print("Padding and indexing sentences...")
sent1_train_seq, sent2_train_seq, labels_train = tokenizing_and_padding(
FIXED_PARAMETERS["train_path"], tokenizer, max_len)
sent1_dev_seq, sent2_dev_seq, labels_dev = tokenizing_and_padding(
FIXED_PARAMETERS["dev_path"], tokenizer, max_len)
print("Loading embeddings...")
vocab_size = min(max_words, len(word_index)) + 1
embedding_matrix = build_emb_matrix(FIXED_PARAMETERS["embedding_path"],
vocab_size, word_index)
embedding_layer = Embedding(vocab_size,
300,
weights=[embedding_matrix],
input_length=max_len,
trainable=emb_train,
name='VectorLookup')
sent1_seq_in = Input(shape=(max_len, ),
dtype='int32',
name='sent1_seq_in')
embedded_sent1 = embedding_layer(sent1_seq_in)
embedded_sent1_drop = layers.Dropout(dropout_rate)(embedded_sent1)
encoded_sent1 = Lambda(lambda x: K.sum(x, axis=1))(embedded_sent1_drop)
sent2_seq_in = Input(shape=(max_len, ),
dtype='int32',
name='sent2_seq_in')
embedded_sent2 = embedding_layer(sent2_seq_in)
embedded_sent2_drop = layers.Dropout(dropout_rate)(embedded_sent2)
encoded_sent2 = Lambda(lambda x: K.sum(x, axis=1))(embedded_sent2_drop)
mul = layers.Multiply()([encoded_sent1, encoded_sent2])
sub = layers.Subtract()([encoded_sent1, encoded_sent2])
dif = Lambda(lambda x: K.abs(x))(sub)
concatenated = layers.concatenate([mul, dif], axis=-1)
x = Dense(150,
activation='sigmoid',
kernel_initializer=initializers.RandomNormal(stddev=0.1),
bias_initializer=initializers.RandomNormal(
stddev=0.1))(concatenated)
x = Dropout(dropout_rate)(x)
x = Dense(6,
activation='softmax',
kernel_initializer=initializers.RandomNormal(stddev=0.1),
bias_initializer=initializers.RandomNormal(stddev=0.1))(x)
gate_mapping = K.variable(
value=np.array([[0.], [1.], [2.], [3.], [4.], [5.]]))
preds = Lambda(lambda a: K.dot(a, gate_mapping), name='Prediction')(x)
model = Model([sent1_seq_in, sent2_seq_in], preds)
model.summary()
def pearson(y_true, y_pred):
"""
Pearson product-moment correlation metric.
"""
return pearsonr(y_true, y_pred)
early_stopping = EarlyStopping(monitor='pearson',
patience=20,
mode='max')
# checkpointer = ModelCheckpoint(filepath=os.path.join(FIXED_PARAMETERS["ckpt_path"], modname) + '.hdf5',
# verbose=1,
# monitor='val_pearson',
# save_best_only=True,
# mode='max')
Adam = optimizers.Adam(lr=learning_rate, beta_1=0.9, beta_2=0.999)
model.compile(optimizer=Adam, loss='mse', metrics=[pearson])
history = model.fit([sent1_train_seq, sent2_train_seq],
labels_train,
verbose=1,
epochs=300,
batch_size=batch_size,
callbacks=[early_stopping],
validation_data=([sent1_dev_seq,
sent2_dev_seq], labels_dev))
def my_model(encoder='VGG',
input_size=(256, 256, 1),
k_shot=1,
learning_rate=1e-4):
# Get the encoder
encoder_t = encoder
if encoder_t == 'VGG_b3':
encoder = EM.vgg_encoder_b3(input_size=input_size)
elif encoder_t == 'VGG_b4':
encoder = EM.vgg_encoder_b4(input_size=input_size)
elif encoder_t == 'VGG_b34':
encoder = EM.vgg_encoder_b34(input_size=input_size)
elif encoder_t == 'VGG_b5':
encoder = EM.vgg_encoder_b5(input_size=input_size)
elif encoder_t == 'VGG_b345':
encoder = EM.vgg_encoder_b345(input_size=input_size)
elif encoder_t == 'VGG_b35':
encoder = EM.vgg_encoder_b35(input_size=input_size)
elif encoder_t == 'VGG_b45':
encoder = EM.vgg_encoder_b45(input_size=input_size)
elif encoder_t == 'RN':
encoder = EM.rn_encoder(input_size=input_size)
else:
print('Encoder is not defined yet')
S_mask = layers.Input(
(k_shot, int(input_size[0] / 4), int(input_size[1] / 4), 1))
S_input = layers.Input(
(k_shot, input_size[0], input_size[1], input_size[2]))
#print(S_input.shape)
Q_input = layers.Input(input_size)
#print(S_mask.shape)
## K shot
kshot_encoder = keras.models.Sequential()
kshot_encoder.add(
layers.TimeDistributed(encoder,
input_shape=(k_shot, input_size[0],
input_size[1], input_size[2])))
## Encode support and query sample
s_encoded = kshot_encoder(S_input)
q_encoded = encoder(Q_input)
print("before")
print(s_encoded.shape)
print(q_encoded.shape)
#if encoder_t == "RN":
s_encoded = layers.TimeDistributed(
layers.Conv2D(128, (3, 3), activation='relu',
padding='same'))(s_encoded)
q_encoded = layers.Conv2D(128, (3, 3), activation='relu',
padding='same')(q_encoded)
print("after")
print(s_encoded.shape)
print(q_encoded.shape)
## Difference of Gussian Pyramid parameters
kernet_shapes = [3, 5, 7, 9]
k_value = np.power(2, 1 / 3)
sigma = 1.6
## Kernel weights for Gussian pyramid
Sigma1_kernel = get_kernel_gussian(kernel_size=kernet_shapes[0],
Sigma=sigma * np.power(k_value, 1),
in_channels=128)
Sigma2_kernel = get_kernel_gussian(kernel_size=kernet_shapes[1],
Sigma=sigma * np.power(k_value, 2),
in_channels=128)
Sigma3_kernel = get_kernel_gussian(kernel_size=kernet_shapes[2],
Sigma=sigma * np.power(k_value, 3),
in_channels=128)
Sigma4_kernel = get_kernel_gussian(kernel_size=kernet_shapes[3],
Sigma=sigma * np.power(k_value, 4),
in_channels=128)
Sigma1_layer = layers.TimeDistributed(
layers.DepthwiseConv2D(kernet_shapes[0],
use_bias=False,
padding='same'))
Sigma2_layer = layers.TimeDistributed(
layers.DepthwiseConv2D(kernet_shapes[1],
use_bias=False,
padding='same'))
Sigma3_layer = layers.TimeDistributed(
layers.DepthwiseConv2D(kernet_shapes[2],
use_bias=False,
padding='same'))
Sigma4_layer = layers.TimeDistributed(
layers.DepthwiseConv2D(kernet_shapes[3],
use_bias=False,
padding='same'))
## Gussian filtering
x1 = Sigma1_layer(s_encoded)
x2 = Sigma2_layer(s_encoded)
x3 = Sigma3_layer(s_encoded)
x4 = Sigma4_layer(s_encoded)
#S_mask = smx
#print(S_mask.shape)
#print("x1.shape")
#print(x1.shape)
#print(s_encoded.shape)
DOG1 = layers.Subtract()([s_encoded, x1])
DOG2 = layers.Subtract()([x1, x2])
DOG3 = layers.Subtract()([x2, x3])
DOG4 = layers.Subtract()([x3, x4])
#print("Dog.shape")
#print(S_mask.shape)
#print(DOG1.shape)
s_1 = ''
s_2 = ''
s_3 = ''
s_4 = ''
## Global Representation
if encoder_t == 'RN':
s_1 = RN_GlobalAveragePooling2D_r(S_mask)(DOG1)
s_2 = RN_GlobalAveragePooling2D_r(S_mask)(DOG2)
s_3 = RN_GlobalAveragePooling2D_r(S_mask)(DOG3)
s_4 = RN_GlobalAveragePooling2D_r(S_mask)(DOG4)
else:
s_1 = GlobalAveragePooling2D_r(S_mask)(DOG1)
s_2 = GlobalAveragePooling2D_r(S_mask)(DOG2)
s_3 = GlobalAveragePooling2D_r(S_mask)(DOG3)
s_4 = GlobalAveragePooling2D_r(S_mask)(DOG4)
## Common Representation of Support and Query sample
s_1 = common_representation(s_1, q_encoded)
s_2 = common_representation(s_2, q_encoded)
s_3 = common_representation(s_3, q_encoded)
s_4 = common_representation(s_4, q_encoded)
## Bidirectional Convolutional LSTM on Pyramid
s_3D = layers.concatenate([s_1, s_2, s_3, s_4], axis=1)
LSTM_f = layers.ConvLSTM2D(filters=128,
kernel_size=(3, 3),
padding='same',
return_sequences=False,
go_backwards=False,
kernel_initializer='he_normal')(s_3D)
LSTM_b = layers.ConvLSTM2D(filters=128,
kernel_size=(3, 3),
padding='same',
return_sequences=False,
go_backwards=True,
kernel_initializer='he_normal')(s_3D)
Bi_rep = layers.Add()([LSTM_f, LSTM_b])
## Decode to query segment
x = layers.Conv2D(128, 3, padding='same',
kernel_initializer='he_normal')(Bi_rep)
x = layers.BatchNormalization(axis=3)(x)
x = layers.Activation('relu')(x)
x = layers.UpSampling2D(size=(2, 2))(x)
x = layers.Conv2D(128, 3, padding='same',
kernel_initializer='he_normal')(x)
x = layers.BatchNormalization(axis=3)(x)
x = layers.Activation('relu')(x)
x = layers.UpSampling2D(size=(2, 2))(x)
x = layers.Conv2D(128, 3, padding='same',
kernel_initializer='he_normal')(x)
x = layers.BatchNormalization(axis=3)(x)
x = layers.Activation('relu')(x)
x = layers.Conv2D(64,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(x)
x = layers.Conv2D(2,
3,
activation='relu',
padding='same',
kernel_initializer='he_normal')(x)
final = layers.Conv2D(1, 1, activation='sigmoid')(x)
print("AAA")
print(final.shape)
seg_model = Model(inputs=[S_input, S_mask, Q_input], outputs=final)
## Load Guassian weights and make it unlearnable
Sigma1_layer.set_weights([Sigma1_kernel])
Sigma2_layer.set_weights([Sigma2_kernel])
Sigma3_layer.set_weights([Sigma3_kernel])
Sigma4_layer.set_weights([Sigma4_kernel])
Sigma1_layer.trainable = False
Sigma2_layer.trainable = False
Sigma3_layer.trainable = False
Sigma4_layer.trainable = False
seg_model.compile(optimizer=keras.optimizers.Adam(lr=learning_rate),
loss='binary_crossentropy',
metrics=['accuracy'])
return seg_model
feature = layers.Conv2D(32, kernel_size=3, padding='same',
activation='relu')(inputs)
feature = layers.MaxPooling2D(pool_size=2)(feature)
feature = layers.Conv2D(32, kernel_size=3, padding='same',
activation='relu')(feature)
feature = layers.MaxPooling2D(pool_size=2)(feature)
feature_model = Model(inputs=inputs, outputs=feature)
# 2 feature models that sharing weights
x1_net = feature_model(x1)
x2_net = feature_model(x2)
# subtract features
net = layers.Subtract()([x1_net, x2_net])
net = layers.Conv2D(32, kernel_size=3, padding='same', activation='relu')(net)
net = layers.MaxPooling2D(pool_size=2)(net)
net = layers.Flatten()(net)
net = layers.Dense(64, activation='relu')(net)
net = layers.Dense(1, activation='sigmoid')(net)
model = Model(inputs=[x1, x2], outputs=net)
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['acc'])
model.summary()
def get_model(self, input_shapes):
feat_shape = input_shapes[0]
emb_shapes = input_shapes[1:]
def build_emb_model(input_shape,
emb_num=0,
dense_layer_size=128,
dropout_rate=0.9):
X_input = layers.Input([input_shape])
X = layers.Dense(dense_layer_size,
name='emb_dense_{}'.format(emb_num),
kernel_initializer=initializers.glorot_uniform(
seed=self.seed))(X_input)
X = layers.BatchNormalization(name='emb_bn_{}'.format(emb_num))(X)
X = layers.Activation('relu')(X)
X = layers.Dropout(dropout_rate, seed=self.seed)(X)
# Create model
model = models.Model(inputs=X_input,
outputs=X,
name='emb_model_{}'.format(emb_num))
return model
emb_models = []
for i, emb_shape in enumerate(emb_shapes):
triplet_input = layers.Input([emb_shape])
triplet_model_shape = int(emb_shape / 3)
triplet_model = build_emb_model(triplet_model_shape,
emb_num=i,
dense_layer_size=112)
P = layers.Lambda(lambda x: x[:, :triplet_model_shape])(
triplet_input)
A = layers.Lambda(
lambda x: x[:, triplet_model_shape:triplet_model_shape * 2])(
triplet_input)
B = layers.Lambda(lambda x: x[:, triplet_model_shape * 2:
triplet_model_shape * 3])(
triplet_input)
A_out = triplet_model(layers.Subtract()([P, A]))
B_out = triplet_model(layers.Subtract()([P, B]))
triplet_out = layers.concatenate([A_out, B_out], axis=-1)
merged_model = models.Model(inputs=triplet_input,
outputs=triplet_out,
name='triplet_model_{}'.format(i))
emb_models.append(merged_model)
emb_num = len(emb_models)
emb_models += [
build_emb_model(emb_shape,
emb_num=i + emb_num,
dense_layer_size=112)
for i, emb_shape in enumerate(emb_shapes)
]
def build_feat_model(input_shape,
dense_layer_size=128,
dropout_rate=0.8):
X_input = layers.Input([input_shape])
X = layers.Dense(dense_layer_size,
name='feat_dense_{}'.format(0),
kernel_initializer=initializers.glorot_normal(
seed=self.seed))(X_input)
X = layers.Activation('relu')(X)
X = layers.Dropout(dropout_rate, seed=self.seed)(X)
# Create model
model = models.Model(inputs=X_input, outputs=X, name='feat_model')
return model
feat_model = build_feat_model(feat_shape, dense_layer_size=128)
lambd = 0.02 # L2 regularization
# Combine all models into one model
merged_out = layers.concatenate(
[feat_model.output] +
[emb_model.output for emb_model in emb_models])
merged_out = layers.Dense(
3,
name='merged_output',
kernel_regularizer=regularizers.l2(lambd),
kernel_initializer=initializers.glorot_uniform(
seed=self.seed))(merged_out)
merged_out = layers.BatchNormalization(name='merged_bn')(merged_out)
merged_out = layers.Activation('softmax')(merged_out)
combined_model = models.Model(
[feat_model.input] + [emb_model.input for emb_model in emb_models],
outputs=merged_out,
name='merged_model')
# print(combined_model.summary())
return combined_model
def AID_CreateModel(input_shape,
alpha_hinge=0.2,
Spatial_Dropout=False,
BN=True,
B5_FC1_neurons=1024,
similarity='simCos',
desc_dim=128,
desc_between_0_1=False,
BigDesc=False,
verbose=True):
# descriptor model
in_desc = layers.Input(shape=input_shape, name='input_patches')
x = layers.Conv2D(64, (3, 3), padding='same', name='block1_conv1')(in_desc)
if BN:
x = layers.BatchNormalization(name='block1_BN1')(x)
x = layers.Activation('relu', name='block1_relu1')(x)
x = layers.Conv2D(64, (3, 3), padding='same', name='block1_conv2')(x)
if BN:
x = layers.BatchNormalization(name='block1_BN2')(x)
x = layers.Activation('relu', name='block1_relu2')(x)
x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='block1_pool')(x)
# Block 2
x = layers.Conv2D(64, (3, 3), padding='same', name='block2_conv1')(x)
if BN:
x = layers.BatchNormalization(name='block2_BN1')(x)
x = layers.Activation('relu', name='block2_relu1')(x)
x = layers.Conv2D(64, (3, 3), padding='same', name='block2_conv2')(x)
if BN:
x = layers.BatchNormalization(name='block2_BN2')(x)
x = layers.Activation('relu', name='block2_relu2')(x)
x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='block2_pool')(x)
# Block 3
x = layers.Conv2D(128, (3, 3), padding='same', name='block3_conv1')(x)
if BN:
x = layers.BatchNormalization(name='block3_BN1')(x)
x = layers.Activation('relu', name='block3_relu1')(x)
x = layers.Conv2D(128, (3, 3), padding='same', name='block3_conv2')(x)
if BN:
x = layers.BatchNormalization(name='block3_BN2')(x)
x = layers.Activation('relu', name='block3_relu2')(x)
x = layers.MaxPooling2D((2, 2), strides=(2, 2), name='block3_pool')(x)
# Block 4
x = layers.Conv2D(128, (3, 3), padding='same', name='block4_conv1')(x)
if BN:
x = layers.BatchNormalization(name='block4_BN1')(x)
x = layers.Activation('relu', name='block4_relu1')(x)
x = layers.Conv2D(128, (3, 3), padding='same', name='block4_conv2')(x)
if BigDesc == False and BN:
x = layers.BatchNormalization(name='block4_BN2')(x)
if Spatial_Dropout:
x = layers.SpatialDropout2D(p=0.5, name='block4_Dropout1')(x)
if BigDesc == False:
x = layers.Activation('relu', name='block4_relu2')(x)
# Block 5
x = layers.Flatten(name='block5_flatten1')(x)
if BigDesc == False:
if B5_FC1_neurons > 0:
x = layers.Dense(B5_FC1_neurons,
activation='relu',
name='block5_FC1')(x)
if desc_between_0_1:
x = layers.Dense(desc_dim, activation='sigmoid',
name='block5_FC2')(x)
else:
x = layers.Dense(desc_dim, name='block5_FC2')(x)
desc_model = Model(in_desc, x, name='aff_desc')
# similarity model
if similarity[0:5] == 'simFC':
if similarity[5:] == '_concat' or similarity[5:] == '_concat_BigDesc':
sim_type = 'concat'
desc_dim = 2 * desc_model.output_shape[1]
elif similarity[5:] == '_diff':
sim_type = 'diff'
# 2 siamese network
in_desc1 = layers.Input(shape=input_shape, name='input_patches1')
in_desc2 = layers.Input(shape=input_shape, name='input_patches2')
emb_1 = desc_model(in_desc1)
emb_2 = desc_model(in_desc2)
# Similarity model
in_sim = layers.Input(shape=(desc_dim, ), name='input_diff_desc')
x = layers.Dense(64, activation='relu', name='block1_FC1')(in_sim)
x = layers.Dense(32, activation='relu', name='block1_FC2')(x)
x = layers.Dense(1, activation='sigmoid', name='block1_FC3')(x)
sim_model = Model(in_sim, x, name='sim')
if sim_type == 'concat':
x = layers.Concatenate(name='Concat')([emb_1, emb_2])
else:
x = layers.Subtract(name='Subtract')([emb_1, emb_2])
out_net = sim_model(x)
# Groundtruth Model
in_GT = layers.Input(shape=(1, ), name='input_GroundTruth')
GT_model = Model(in_GT, in_GT, name='GroundTruth')
out_GT = GT_model(in_GT)
class TopLossLayerClass(layers.Layer):
def __init__(self, **kwargs):
super(TopLossLayerClass, self).__init__(**kwargs)
def call(self, inputs):
#out_net, out_GT = inputs
s, t = inputs # t=1 -> Positive class, t=0 -> Negative class
loss = K.sum(t * K.log(s) + (1 - t) * K.log(1 - s))
self.add_loss(loss)
return loss
TopLossLayer_obj = TopLossLayerClass(name='TopLossLayer')
TopLossLayer = TopLossLayer_obj([out_net, out_GT])
train_model = Model([in_desc1, in_desc2, in_GT],
TopLossLayer,
name='TrainModel')
elif similarity == 'simCos': # hinge loss
# Similarity model
desc_dim = desc_model.output_shape[1]
in_sim1 = layers.Input(shape=(desc_dim, ), name='input_desc1')
in_sim2 = layers.Input(shape=(desc_dim, ), name='input_desc2')
x = layers.Dot(axes=1, normalize=True,
name='CosineProximity')([in_sim1,
in_sim2]) # cosine proximity
sim_model = Model([in_sim1, in_sim2], x, name='sim')
# 3 siamese networks
in_desc1 = layers.Input(shape=input_shape, name='input_patches_anchor')
in_desc2 = layers.Input(shape=input_shape,
name='input_patches_positive')
in_desc3 = layers.Input(shape=input_shape,
name='input_patches_negative')
emb_1 = desc_model(in_desc1)
emb_2 = desc_model(in_desc2)
emb_3 = desc_model(in_desc3)
sim_type = 'inlist'
out_net_positive = sim_model([emb_1, emb_2])
out_net_negative = sim_model([emb_1, emb_3])
class TopLossLayerClass(layers.Layer):
def __init__(self, alpha=0.2, **kwargs):
self.alpha = alpha
super(TopLossLayerClass, self).__init__(**kwargs)
def call(self, inputs):
out_net_positive, out_net_negative = inputs
# Hinge loss computation
loss = K.sum(
K.maximum(out_net_negative - out_net_positive + self.alpha,
0)) #,axis=0)
self.add_loss(loss)
return loss
TopLossLayer_obj = TopLossLayerClass(name='TopLossLayer',
alpha=alpha_hinge)
TopLossLayer = TopLossLayer_obj([out_net_positive, out_net_negative])
train_model = Model([in_desc1, in_desc2, in_desc3],
TopLossLayer,
name='TrainModel')
if verbose:
print(
'\n\n-------> The network architecture for the affine descriptor computation !'
)
desc_model.summary()
print(
'\n\n-------> The network architecture for the similarity computation !'
)
sim_model.summary()
print('\n\n-------> Train model connections')
train_model.summary()
return train_model, sim_type
def __init__(self, max_len, emb_train):
# Define hyperparameters
modname = FIXED_PARAMETERS["model_name"]
learning_rate = FIXED_PARAMETERS["learning_rate"]
dropout_rate = FIXED_PARAMETERS["dropout_rate"]
batch_size = FIXED_PARAMETERS["batch_size"]
max_words = FIXED_PARAMETERS["max_words"]
print("Loading data...")
genres_train, sent1_train, sent2_train, labels_train_, scores_train = load_sts_data(
FIXED_PARAMETERS["train_path"])
genres_dev, sent1_dev, sent2_dev, labels_dev_, scores_dev = load_sts_data(
FIXED_PARAMETERS["dev_path"])
print("Building dictionary...")
text = sent1_train + sent2_train + sent1_dev + sent2_dev
tokenizer = Tokenizer(num_words=max_words)
tokenizer.fit_on_texts(text)
word_index = tokenizer.word_index
print("Padding and indexing sentences...")
sent1_train_seq, sent2_train_seq, labels_train = tokenizing_and_padding(
FIXED_PARAMETERS["train_path"], tokenizer, max_len)
sent1_dev_seq, sent2_dev_seq, labels_dev = tokenizing_and_padding(
FIXED_PARAMETERS["dev_path"], tokenizer, max_len)
print("Encoding labels...")
train_labels_to_probs = encode_labels(labels_train)
dev_labels_to_probs = encode_labels(labels_dev)
print("Loading embeddings...")
vocab_size = min(max_words, len(word_index)) + 1
embedding_matrix = build_emb_matrix(FIXED_PARAMETERS["embedding_path"],
vocab_size, word_index)
embedding_layer = Embedding(vocab_size,
300,
weights=[embedding_matrix],
input_length=max_len,
trainable=emb_train,
mask_zero=True,
name='VectorLookup')
lstm = Bidirectional(
LSTM(
150, #dropout=dropout_rate, recurrent_dropout=0.1,
return_sequences=False,
kernel_regularizer=regularizers.l2(1e-4),
name='RNN'))
sent1_seq_in = Input(shape=(max_len, ),
dtype='int32',
name='Sentence1')
embedded_sent1 = embedding_layer(sent1_seq_in)
encoded_sent1 = lstm(embedded_sent1)
#mean_pooling_1 = TemporalMeanPooling()(encoded_sent1)
#sent1_lstm = Dropout(0.1)(mean_pooling_1)
sent2_seq_in = Input(shape=(max_len, ),
dtype='int32',
name='Sentence2')
embedded_sent2 = embedding_layer(sent2_seq_in)
encoded_sent2 = lstm(embedded_sent2)
#mean_pooling_2 = TemporalMeanPooling()(encoded_sent2)
#sent2_lstm = Dropout(0.1)(mean_pooling_2)
mul = layers.Multiply(name='S1.S2')(
[encoded_sent1,
encoded_sent2]) #([mean_pooling_1, mean_pooling_2])
sub = layers.Subtract(name='S1-S2')(
[encoded_sent1,
encoded_sent2]) #([mean_pooling_1, mean_pooling_2])
dif = Lambda(lambda x: K.abs(x), name='Abs')(sub)
concatenated = concatenate([mul, dif], name='Concat')
x = Dense(50,
activation='sigmoid',
name='Sigmoid',
kernel_regularizer=regularizers.l2(1e-4))(concatenated)
preds = Dense(6,
activation='softmax',
kernel_regularizer=regularizers.l2(1e-4),
name='Softmax')(x)
model = Model([sent1_seq_in, sent2_seq_in], preds)
model.summary()
def pearson(y_true, y_pred):
"""
Pearson product-moment correlation metric.
"""
gate_mapping = K.variable(
value=np.array([[0.], [1.], [2.], [3.], [4.], [5.]]))
y_true = K.clip(y_true, K.epsilon(), 1)
y_pred = K.clip(y_pred, K.epsilon(), 1)
y_true = K.reshape(K.dot(y_true, gate_mapping), (-1, ))
y_pred = K.reshape(K.dot(y_pred, gate_mapping), (-1, ))
return pearsonr(y_true, y_pred)
early_stopping = EarlyStopping(monitor='pearson',
patience=10,
mode='max')
# checkpointer = ModelCheckpoint(filepath=os.path.join(FIXED_PARAMETERS["ckpt_path"], modname) + '.hdf5',
# verbose=1,
# monitor='val_pearson',
# save_best_only=True,
# mode='max')
Adagrad = optimizers.Adagrad(lr=learning_rate)
model.compile(optimizer=Adagrad, loss='kld', metrics=[pearson])
history = model.fit([sent1_train_seq, sent2_train_seq],
train_labels_to_probs,
epochs=30,
batch_size=batch_size,
shuffle=True,
callbacks=[early_stopping],
validation_data=([sent1_dev_seq, sent2_dev_seq],
dev_labels_to_probs))
def dncnn(depth=17,
n_filters=64,
kernel_size=(3, 3),
n_channels=1,
channels_first=False):
"""Keras implementation of DnCNN. Implementation followed the original paper [1]_. Authors original code can be
found on `their Github Page
<https://github.com/cszn/DnCNN/>`_.
Parameters
----------
depth : int
Number of fully convolutional layers in dncnn. In the original paper, the authors have used depth=17 for non-
blind denoising and depth=20 for blind denoising.
n_filters : int
Number of filters on each convolutional layer.
kernel_size : int tuple
2D Tuple specifying the size of the kernel window used to compute activations.
n_channels : int
Number of image channels that the network processes (1 for grayscale, 3 for RGB)
channels_first : bool
Whether channels comes first (NCHW, True) or last (NHWC, False)
Returns
-------
:class:`keras.models.Model`
Keras model object representing the Neural Network.
References
----------
.. [1] Zhang K, Zuo W, Chen Y, Meng D, Zhang L. Beyond a gaussian denoiser: Residual learning of deep cnn
for image denoising. IEEE Transactions on Image Processing. 2017
Examples
--------
>>> from OpenDenoising.model.architectures.keras import dncnn
>>> dncnn_s = dncnn(depth=17)
>>> dncnn_b = dncnn(depth=20)
"""
assert (
n_channels == 1 or n_channels == 3
), "Expected 'n_channels' to be 1 or 3, but got {}".format(n_channels)
if channels_first:
data_format = "channels_first"
x = layers.Input(shape=[n_channels, None, None])
else:
data_format = "channels_last"
x = layers.Input(shape=[None, None, n_channels])
with tf.name_scope("Layer1"):
# First layer: Conv + ReLU
y = layers.Conv2D(filters=n_filters,
kernel_size=kernel_size,
strides=(1, 1),
padding='same',
kernel_initializer='Orthogonal',
data_format=data_format)(x)
y = layers.Activation("relu")(y)
# Middle layers: Conv + ReLU + BN
for i in range(1, depth - 1):
with tf.name_scope("Layer{}".format(i + 1)):
y = layers.Conv2D(filters=n_filters,
kernel_size=kernel_size,
strides=(1, 1),
padding='same',
kernel_initializer='Orthogonal',
use_bias=False,
data_format=data_format)(y)
y = layers.BatchNormalization(axis=-1, momentum=0.0,
epsilon=1e-3)(y)
y = layers.Activation("relu")(y)
with tf.name_scope("Layer{}".format(depth)):
# Final layer: Conv
y = layers.Conv2D(filters=1,
kernel_size=kernel_size,
strides=(1, 1),
use_bias=False,
kernel_initializer='Orthogonal',
padding='same',
data_format=data_format)(y)
y = layers.Subtract()([x, y])
# Keras model
return models.Model(x, y)
modelA.add(layers.Activation("relu"))
modelA.add(layers.MaxPooling2D((2,2)))
modelA.add(layers.Flatten())
modelA.add(layers.Dense(batch))
modelA.add(layers.Reshape((1, batch)))
return modelA
modelu = modelCreator()
modelu.summary()
X1 = layers.Input((img_width,img_height,1))
X2 = layers.Input((img_width,img_height,1))
my1 = modelu(X1)
my2 = modelu(X2)
myo = layers.Subtract()([my1, my2])
m = layers.Dense(1, activation = "sigmoid")(myo)
model = models.Model(inputs = [X1, X2], outputs = m)
model.compile(loss = "binary_crossentropy", optimizer = 'adam', metrics = ['accuracy'])
model.summary()
path_to_dataset = os.getcwd() + "\\dataset"
#For training
anchor_main = []
real_forged_main = []
y_list = []
for x in range(1, 5):
if(x != 2):
forge = []
x_res, y_res, z_res = res
inputs = [
layers.Input(shape=vector_res), # Magnetic Field
layers.Input(shape=vector_res), # Velocity Field
layers.Input(shape=scalar_res), # Pressure Field
layers.Input(shape=scalar_res), # Mass Density
]
gradConvPressure = layers.Conv3D(3, 3, padding="same")(inputs[2])
gradConvMassDensity = layers.Conv3D(3, 3, padding="same")(inputs[3])
divConvMagField = layers.Conv3D(1, 3, padding="same")(inputs[0])
divConvVelField = layers.Conv3D(1, 3, padding="same")(inputs[1])
curlConvMagField = layers.Conv3D(3, 3, padding="same")(inputs[0])
curlConvVelField = layers.Conv3D(3, 3, padding="same")(inputs[1])
udotgradu = layers.Conv3D(1, 3, padding="same")(layers.Multiply(
inputs[1], backend.repeat_elements(divConvVelField, 3, -1)))
currentDensity = layers.Multiply(
inputs[1], inputs[3],
backend.ones_like(inputs[3]) * total_charge / total_mass)
velocityNew = layers.Subtract(
layers.Multiply(
layers.Subtract(
layers.Conv3D(3, 3, padding="same")(layers.Concatenate(
currentDensity, inputs[0])), gradConvPressure), inputs[4]**-1),
udotgradu)
# model = models.Model(inputs=inputs, outputs=outputs)
core_inputs = layers.Input((config.strmaxlen, ))
core_layer = layers.Embedding(252,
config.embedding,
input_length=config.strmaxlen)(core_inputs)
core_layer = layers.Bidirectional(
layers.CuDNNGRU(256, return_sequences=True))(core_layer)
core_layer = layers.Bidirectional(
layers.CuDNNGRU(256, return_sequences=False))(core_layer)
core_model = models.Model(inputs=core_inputs, outputs=core_layer)
inputs1 = layers.Input((config.strmaxlen, ))
inputs2 = layers.Input((config.strmaxlen, ))
layer1 = core_model(inputs1)
layer2 = core_model(inputs2)
main_layer = layers.Subtract()([layer1, layer2])
main_layer = layers.Lambda(lambda layer: tf.norm(
layer, ord=2, axis=-1, keep_dims=True))(main_layer)
main_layer = layers.Dense(1)(main_layer)
main_layer = layers.Activation('sigmoid')(main_layer)
main_model = models.Model(inputs=[inputs1, inputs2], outputs=main_layer)
main_model.summary()
main_model.compile(optimizer=optimizers.Adam(lr=0.001, amsgrad=True),
loss='binary_crossentropy',
metrics=['accuracy'])
#------------------------------------------------------------------
# DONOTCHANGE: Reserved for nsml
bind_model(main_model)
# DONOTCHANGE: Reserved for nsml
