def __init__(self, path="./data/"):
self.__path = path
self.__train_folder = "train_alphabets"
self.__test_folder = "test_alphabets"
self.__data_loader = OmniglotLoader(self.__path, self.__train_folder,
self.__test_folder)
self.train_set = ()
self.val_set = ()
self.test_set = ()
self.data_shape = ()
def __init__(self, dataset_path, learning_rate, batch_size,
use_augmentation, learning_rate_multipliers,
l2_regularization_penalization, tensorboard_log_path):
"""Inits SiameseNetwork with the provided values for the attributes.
It also constructs the siamese network architecture, creates a dataset
loader and opens the log file.
Arguments:
dataset_path: path of Omniglot dataset
learning_rate: SGD learning rate
batch_size: size of the batch to be used in training
use_augmentation: boolean that allows us to select if data augmentation
is used or not
learning_rate_multipliers: learning-rate multipliers (relative to the learning_rate
chosen) that will be applied to each fo the conv and dense layers
for example:
# Setting the Learning rate multipliers
LR_mult_dict = {}
LR_mult_dict['conv1']=1
LR_mult_dict['conv2']=1
LR_mult_dict['dense1']=2
LR_mult_dict['dense2']=2
l2_regularization_penalization: l2 penalization for each layer.
for example:
# Setting the Learning rate multipliers
L2_dictionary = {}
L2_dictionary['conv1']=0.1
L2_dictionary['conv2']=0.001
L2_dictionary['dense1']=0.001
L2_dictionary['dense2']=0.01
tensorboard_log_path: path to store the logs
"""
use_CNN = True
if use_CNN:
self.input_shape = ((32, 24, 1))
else:
self.input_shape = (
(768, )) # Size of sentences vector 长度是经过bert转化后的768
self.model = []
self.learning_rate = learning_rate
self.omniglot_loader = OmniglotLoader(
dataset_path=dataset_path,
use_augmentation=use_augmentation,
batch_size=batch_size)
self.summary_writer = tf.summary.FileWriter(tensorboard_log_path)
self._construct_siamese_architecture(learning_rate_multipliers,
l2_regularization_penalization)
def __init__(self, dataset_path, learning_rate, batch_size, use_augmentation,
learning_rate_multipliers, l2_regularization_penalization, tensorboard_log_path):
"""Inits SiameseNetwork with the provided values for the attributes.
It also constructs the siamese network architecture, creates a dataset
loader and opens the log file.
"""
self.input_shape = (1,128) # Size of images
self.model = []
self.learning_rate = learning_rate
self.omniglot_loader = OmniglotLoader(
dataset_path=dataset_path, use_augmentation=use_augmentation, batch_size=batch_size)
self.summary_writer = tf.summary.FileWriter(tensorboard_log_path)
self._construct_siamese_architecture(learning_rate_multipliers,
l2_regularization_penalization)
class SiameseNetwork:
"""Class that constructs the Siamese Net for training
This Class was constructed to create the siamese net and train it.
Attributes:
input_shape: image size
model: current siamese model
learning_rate: SGD learning rate
omniglot_loader: instance of OmniglotLoader
summary_writer: tensorflow writer to store the logs
"""
def __init__(self, dataset_path, learning_rate, batch_size,
use_augmentation, learning_rate_multipliers,
l2_regularization_penalization, tensorboard_log_path):
"""Inits SiameseNetwork with the provided values for the attributes.
It also constructs the siamese network architecture, creates a dataset
loader and opens the log file.
Arguments:
dataset_path: path of Omniglot dataset
learning_rate: SGD learning rate
batch_size: size of the batch to be used in training
use_augmentation: boolean that allows us to select if data augmentation
is used or not
learning_rate_multipliers: learning-rate multipliers (relative to the learning_rate
chosen) that will be applied to each fo the conv and dense layers
for example:
# Setting the Learning rate multipliers
LR_mult_dict = {}
LR_mult_dict['conv1']=1
LR_mult_dict['conv2']=1
LR_mult_dict['dense1']=2
LR_mult_dict['dense2']=2
l2_regularization_penalization: l2 penalization for each layer.
for example:
# Setting the Learning rate multipliers
L2_dictionary = {}
L2_dictionary['conv1']=0.1
L2_dictionary['conv2']=0.001
L2_dictionary['dense1']=0.001
L2_dictionary['dense2']=0.01
tensorboard_log_path: path to store the logs
"""
self.input_shape = (105, 105, 1) # Size of images
self.model = []
self.learning_rate = learning_rate
self.omniglot_loader = OmniglotLoader(
dataset_path=dataset_path,
use_augmentation=use_augmentation,
batch_size=batch_size)
self.summary_writer = tf.summary.FileWriter(tensorboard_log_path)
# self.summary_writer = tf.compat.v1.summary.FileWriter(tensorboard_log_path)
self._construct_siamese_architecture(learning_rate_multipliers,
l2_regularization_penalization)
def _construct_siamese_architecture(self, learning_rate_multipliers,
l2_regularization_penalization):
""" Constructs the siamese architecture and stores it in the class
Arguments:
learning_rate_multipliers
l2_regularization_penalization
"""
# Let's define the cnn architecture
convolutional_net = Sequential()
convolutional_net.add(
Conv2D(filters=64,
kernel_size=(10, 10),
activation='relu',
input_shape=self.input_shape,
kernel_regularizer=l2(
l2_regularization_penalization['Conv1']),
name='Conv1'))
convolutional_net.add(MaxPool2D())
convolutional_net.add(
Conv2D(filters=128,
kernel_size=(7, 7),
activation='relu',
kernel_regularizer=l2(
l2_regularization_penalization['Conv2']),
name='Conv2'))
convolutional_net.add(MaxPool2D())
convolutional_net.add(
Conv2D(filters=128,
kernel_size=(4, 4),
activation='relu',
kernel_regularizer=l2(
l2_regularization_penalization['Conv3']),
name='Conv3'))
convolutional_net.add(MaxPool2D())
convolutional_net.add(
Conv2D(filters=256,
kernel_size=(4, 4),
activation='relu',
kernel_regularizer=l2(
l2_regularization_penalization['Conv4']),
name='Conv4'))
convolutional_net.add(Flatten())
convolutional_net.add(
Dense(units=4096,
activation='sigmoid',
kernel_regularizer=l2(
l2_regularization_penalization['Dense1']),
name='Dense1'))
# Now the pairs of images
input_image_1 = Input(self.input_shape)
input_image_2 = Input(self.input_shape)
encoded_image_1 = convolutional_net(input_image_1)
encoded_image_2 = convolutional_net(input_image_2)
# L1 distance layer between the two encoded outputs
# One could use Subtract from Keras, but we want the absolute value
l1_distance_layer = Lambda(
lambda tensors: K.abs(tensors[0] - tensors[1]))
l1_distance = l1_distance_layer([encoded_image_1, encoded_image_2])
# Same class or not prediction
prediction = Dense(units=1, activation='sigmoid')(l1_distance)
self.model = Model(inputs=[input_image_1, input_image_2],
outputs=prediction)
# Define the optimizer and compile the model
optimizer = Modified_SGD(lr=self.learning_rate,
lr_multipliers=learning_rate_multipliers,
momentum=0.5)
self.model.compile(loss='binary_crossentropy',
metrics=['binary_accuracy'],
optimizer=optimizer)
def _write_logs_to_tensorboard(self, current_iteration, train_losses,
train_accuracies, validation_accuracy,
evaluate_each):
""" Writes the logs to a tensorflow log file
This allows us to see the loss curves and the metrics in tensorboard.
If we wrote every iteration, the training process would be slow, so
instead we write the logs every evaluate_each iteration.
Arguments:
current_iteration: iteration to be written in the log file
train_losses: contains the train losses from the last evaluate_each
iterations.
train_accuracies: the same as train_losses but with the accuracies
in the training set.
validation_accuracy: accuracy in the current one-shot task in the
validation set
evaluate each: number of iterations defined to evaluate the one-shot
tasks.
"""
summary = tf.Summary()
# Write to log file the values from the last evaluate_every iterations
for index in range(0, evaluate_each):
value = summary.value.add()
value.simple_value = train_losses[index]
value.tag = 'Train Loss'
value = summary.value.add()
value.simple_value = train_accuracies[index]
value.tag = 'Train Accuracy'
if index == (evaluate_each - 1):
value = summary.value.add()
value.simple_value = validation_accuracy
value.tag = 'One-Shot Validation Accuracy'
self.summary_writer.add_summary(
summary, current_iteration - evaluate_each + index + 1)
self.summary_writer.flush()
def train_siamese_network(self, number_of_iterations, support_set_size,
final_momentum, momentum_slope, evaluate_each,
model_name):
""" Train the Siamese net
This is the main function for training the siamese net.
In each every evaluate_each train iterations we evaluate one-shot tasks in
validation and evaluation set. We also write to the log file.
Arguments:
number_of_iterations: maximum number of iterations to train.
support_set_size: number of characters to use in the support set
in one-shot tasks.
final_momentum: mu_j in the paper. Each layer starts at 0.5 momentum
but evolves linearly to mu_j
momentum_slope: slope of the momentum evolution. In the paper we are
only told that this momentum evolves linearly. Because of that I
defined a slope to be passed to the training.
evaluate each: number of iterations defined to evaluate the one-shot
tasks.
model_name: save_name of the model
Returns:
Evaluation Accuracy
"""
# First of all let's divide randomly the 30 train alphabets in train
# and validation with 24 for training and 6 for validation
self.omniglot_loader.split_train_datasets()
# Variables that will store 100 iterations losses and accuracies
# after evaluate_each iterations these will be passed to tensorboard logs
train_losses = np.zeros(shape=(evaluate_each))
train_accuracies = np.zeros(shape=(evaluate_each))
count = 0
earrly_stop = 0
# Stop criteria variables
best_validation_accuracy = 0.0
best_accuracy_iteration = 0
validation_accuracy = 0.0
# Train loop
for iteration in range(number_of_iterations):
# train set
images, labels = self.omniglot_loader.get_train_batch()
train_loss, train_accuracy = self.model.train_on_batch(
images, labels)
# Decay learning rate 1 % per 500 iterations (in the paper the decay is
# 1% per epoch). Also update linearly the momentum (starting from 0.5 to 1)
if (iteration + 1) % 500 == 0:
K.set_value(self.model.optimizer.lr,
K.get_value(self.model.optimizer.lr) * 0.99)
if K.get_value(self.model.optimizer.momentum) < final_momentum:
K.set_value(
self.model.optimizer.momentum,
K.get_value(self.model.optimizer.momentum) +
momentum_slope)
train_losses[count] = train_loss
train_accuracies[count] = train_accuracy
# validation set
count += 1
print(
'Iteration %d/%d: Train loss: %f, Train Accuracy: %f, lr = %f'
% (iteration + 1, number_of_iterations, train_loss,
train_accuracy, K.get_value(self.model.optimizer.lr)))
# Each 100 iterations perform a one_shot_task and write to tensorboard the
# stored losses and accuracies
if (iteration + 1) % evaluate_each == 0:
number_of_runs_per_alphabet = 40
# use a support set size equal to the number of character in the alphabet
validation_accuracy = self.omniglot_loader.one_shot_test(
self.model,
support_set_size,
number_of_runs_per_alphabet,
is_validation=True)
self._write_logs_to_tensorboard(iteration, train_losses,
train_accuracies,
validation_accuracy,
evaluate_each)
count = 0
# Some hyperparameters lead to 100%, although the output is almost the same in
# all images.
if (validation_accuracy == 1.0 and train_accuracy == 0.5):
print('Early Stopping: Gradient Explosion')
print('Validation Accuracy = ' +
str(best_validation_accuracy))
return 0
elif train_accuracy == 0.0:
return 0
else:
# Save the model
if validation_accuracy > best_validation_accuracy:
best_validation_accuracy = validation_accuracy
best_accuracy_iteration = iteration
model_json = self.model.to_json()
if not os.path.exists('./models'):
os.makedirs('./models')
with open('models/' + model_name + '.json',
"w") as json_file:
json_file.write(model_json)
self.model.save_weights('models/' + model_name + '.h5')
# If accuracy does not improve for 10000 batches stop the training
if iteration - best_accuracy_iteration > 10000:
print(
'Early Stopping: validation accuracy did not increase for 10000 iterations'
)
print('Best Validation Accuracy = ' +
str(best_validation_accuracy))
print('Validation Accuracy = ' + str(best_validation_accuracy))
break
print('Trained Ended!')
return best_validation_accuracy
import matplotlib.patches as mpatches
import matplotlib.pyplot as plt
import numpy as np
import scikitplot as skplt
import sklearn
from omniglot_loader import OmniglotLoader
from siamese_network import SiameseNetwork
model_wo_tf = './trained_models/wo_transform/model.h5'
model_w_tf = './trained_models/w_transform/model.h5'
if __name__ == '__main__':
# First test case, training without transformations, testing without transformations
omg = OmniglotLoader(use_transformations=False)
network = SiameseNetwork(model_location=model_wo_tf)
y_pred, te_y = network.get_predictions(omg)
y_pred = np.array(y_pred)
te_y = np.array(te_y)
fpr, tpr, _ = skplt.metrics.roc_curve(te_y.flatten(), y_pred.flatten())
auc = sklearn.metrics.roc_auc_score(te_y.flatten(), y_pred.flatten())
# Second test case, training without transformations, testing with transformations
omg = OmniglotLoader(use_transformations=True)
y_pred, te_y = network.get_predictions(omg)
y_pred = np.array(y_pred)
te_y = np.array(te_y)
fpr_te_tf, tpr_te_tf, _ = skplt.metrics.roc_curve(te_y.flatten(),
y_pred.flatten())
run_start_time = datetime.today().strftime('%Y-%m-%d %H-%M-%S')
projector_dir = 'projector_data' + '/' + run_start_time
alphabet = 'Kannada'
model_w_tf = load_model('./models/2019-09-26 11-04-37/model.h5')
model_wo_tf = load_model('./models/2019-09-25 18-40-12/model.h5')
net_copy = copy_network(model_wo_tf)
# Test model w/o transformations on alphabet w/o transformations
log_dir = join(projector_dir, alphabet)
if not exists(log_dir):
makedirs(log_dir)
# Test three cases:
# 1) Original Training and Test Data (No Transformations)
omg = OmniglotLoader(use_transformations=False)
omg._current_alphabet_index = omg.get_alphabet_index(alphabet, False)
omg.set_training_evaluation_symbols(False)
write_output_and_metadata(model_wo_tf, omg, False, 'model', 'test',
log_dir, net_copy)
# 2) Training (No Transformation), Testing (With Transformations)
omg._tn_batches_done = False
omg._tp_batches_done = False
omg.use_transformations = True
write_output_and_metadata(model_wo_tf, omg, False, 'model', 'test_tf',
log_dir, net_copy)
# 3) Transformation in both Training and Testing
omg._tn_batches_done = False
omg._tp_batches_done = False
write_output_and_metadata(model_w_tf, omg, False, 'model_tf', 'test_tf',
log_dir, net_copy)
class OmniglotDataset:
def __init__(self, path="./data/"):
self.__path = path
self.__train_folder = "train_alphabets"
self.__test_folder = "test_alphabets"
self.__data_loader = OmniglotLoader(self.__path, self.__train_folder,
self.__test_folder)
self.train_set = ()
self.val_set = ()
self.test_set = ()
self.data_shape = ()
def load(self):
train_val_set, self.test_set = self.__data_loader.load_data()
_, _, height, width, channel = train_val_set[0].shape
start_index_val = train_val_set[1].tolist().index('Latin/character01')
self.train_set = (train_val_set[0][:start_index_val],
train_val_set[1][:start_index_val])
self.val_set = (train_val_set[0][start_index_val:],
train_val_set[1][start_index_val:])
self.data_shape = (height, width, channel)
def get_data_classes(self, num_classes, data_type='train'):
data = self.__get_data_type(data_type)
classes = np.random.choice(data.shape[0],
size=(num_classes, ),
replace=False)
return classes
def get_image_pair(self, class_value, data_type='train', same_class=False):
data = self.__get_data_type(data_type)
num_classes, num_images, _, _, _ = data.shape
image_indices = np.random.choice(num_images,
size=2,
replace=(not same_class))
first_image = data[class_value, image_indices[0]]
second_class = class_value if same_class else self.__get_different_value(
class_value, num_classes)
second_image = data[second_class, image_indices[1]]
return first_image, second_image
def __get_different_value(self, value, max_values):
values = list(range(max_values))
values.remove(value)
different_value = np.random.choice(values)
return different_value
def __get_data_type(self, data_type):
if (data_type == 'train'):
return self.train_set[0]
elif (data_type == 'val'):
return self.val_set[0]
else:
return self.test_set[0]
model = load_model('./trained_models/wo_transform/model.h5')
net_copy = copy_network(model)
# Test model w/o transformations on alphabet w/o transformations
log_dir = join(projector_dir, alphabet)
if not exists(log_dir):
makedirs(log_dir)
OutFunc = K.function([
model.layers[2].get_layer('Conv_layer_1').input,
[model.layers[0].input, model.layers[1].input]
], [
model.layers[2].get_layer('Dense_layer_1').output[1],
model.layers[4].output
])
omg = OmniglotLoader(use_transformations=False)
images, labels = omg.get_training_batch()
print(model.summary())
print(images[0, 0].dtype)
print(model.input)
K.clear_session()
out_val = OutFunc([[images[:, 0], images[:, 1]],
[images[:, 0], images[:, 1]]])
print(out_val[1])
#print(out_val)
print(len(out_val[0][0])) # activations
print(out_val[1][0]) # predictions
print(len(out_val[0][1])) # activations
print(out_val[1][1]) # predictions
print(model.layers[2].get_layer('Dense_layer_1').output[0])
#coding:utf-8
"""孪生网络
"""
from __future__ import print_function
import tensorflow as tf
import numpy as np
from omniglot_loader import OmniglotLoader
import sys
dataset_path = sys.argv[1]
omniglot_loader = OmniglotLoader(
dataset_path=dataset_path, use_augmentation=True, batch_size=32)
omniglot_loader.split_train_datasets()
images, labels = omniglot_loader.get_train_batch()
print([i.shape for i in images], labels.shape)
images, labels = omniglot_loader.get_one_shot_batch(32, False)
print([i.shape for i in images], labels.shape)
from keras.models import Sequential, Model
from keras.layers import Conv2D, Dense, MaxPool2D, GlobalAveragePooling2D, BatchNormalization, Activation, Input, Lambda
from keras.regularizers import l2
from keras.optimizers import Adam
import keras.backend as K
input_shape = (105, 105, 1)