def build_elu_cnn(input_shape, output_size):
"""Build a variation of the CNN implemented in the ELU paper.
https://arxiv.org/abs/1511.07289
"""
def layers(n, channels, kernel):
return sum(([
Convolution2D(channels, kernel_size=kernel, padding="same"),
ELU()
] for i in range(n)), [])
model = Sequential(
[
Convolution2D(
384, kernel_size=3, padding="same", input_shape=input_shape)
] + layers(1, 384, 3) + [MaxPooling2D(pool_size=(2, 2))] +
layers(1, 384, 1) + layers(1, 384, 2) + layers(2, 640, 2) +
[MaxPooling2D(pool_size=(2, 2))] + layers(1, 640, 1) +
layers(3, 768, 2) + [MaxPooling2D(pool_size=(2, 2))] +
layers(1, 768, 1) + layers(2, 896, 2) +
[MaxPooling2D(pool_size=(2, 2))] + layers(1, 896, 3) +
layers(2, 1024, 2) + [
MaxPooling2D(pool_size=(2, 2)),
Convolution2D(output_size, kernel_size=1, padding="same"),
GlobalAveragePooling2D(),
Activation("softmax")
])
model.compile(optimizer=SGD(momentum=0.9),
loss="categorical_crossentropy",
metrics=["accuracy"])
return model
def get_seq_model():
"""Define three channel input shape depending on image data format."""
if K.image_data_format() == 'channels_first':
input_shape = (3, img_width, img_height)
else:
input_shape = (img_width, img_height, 3)
# Initialize CNN by creating a sequential model.
model = Sequential()
model.add(Conv2D(32, (3, 3), input_shape=input_shape))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(32, (3, 3)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Conv2D(64, (3, 3)))
model.add(Activation('relu'))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
model.add(Dense(64))
model.add(Activation('relu'))
model.add(Dropout(0.5))
model.add(Dense(2))
model.add(Activation('sigmoid'))
model.compile(
loss='binary_crossentropy', optimizer='rmsprop', metrics=['accuracy'])
return model
def build_small_cnn(input_shape, output_size):
model = Sequential([
# conv1_*
Convolution2D(32,
kernel_size=3,
padding="same",
input_shape=input_shape),
Activation("relu"),
Convolution2D(32, kernel_size=3, padding="same"),
Activation("relu"),
MaxPooling2D(pool_size=(2, 2)),
# conv2_*
Convolution2D(64, kernel_size=3, padding="same"),
Activation("relu"),
Convolution2D(64, kernel_size=3, padding="same"),
Activation("relu"),
MaxPooling2D(pool_size=(2, 2)),
# Fully connected
Flatten(),
Dense(512),
Activation("relu"),
Dense(512),
Activation("relu"),
Dense(output_size),
Activation("softmax")
])
model.compile(loss="categorical_crossentropy",
optimizer="adam",
metrics=["accuracy"])
return model
def inception_block_1c(X):
X_3x3 = fr_utils.conv2d_bn(X,
layer='inception_3c_3x3',
cv1_out=128,
cv1_filter=(1, 1),
cv2_out=256,
cv2_filter=(3, 3),
cv2_strides=(2, 2),
padding=(1, 1))
X_5x5 = fr_utils.conv2d_bn(X,
layer='inception_3c_5x5',
cv1_out=32,
cv1_filter=(1, 1),
cv2_out=64,
cv2_filter=(5, 5),
cv2_strides=(2, 2),
padding=(2, 2))
X_pool = MaxPooling2D(pool_size=3, strides=2, data_format='channels_first')(X)
X_pool = ZeroPadding2D(padding=((0, 1), (0, 1)), data_format='channels_first')(X_pool)
inception = concatenate([X_3x3, X_5x5, X_pool], axis=1)
return inception
def inception_block_1a(X):
"""
Implementation of an inception block
"""
X_3x3 = Conv2D(96, (1, 1), data_format='channels_first', name='inception_3a_3x3_conv1')(X)
X_3x3 = BatchNormalization(axis=1, epsilon=0.00001, name='inception_3a_3x3_bn1')(X_3x3)
X_3x3 = Activation('relu')(X_3x3)
X_3x3 = ZeroPadding2D(padding=(1, 1), data_format='channels_first')(X_3x3)
X_3x3 = Conv2D(128, (3, 3), data_format='channels_first', name='inception_3a_3x3_conv2')(X_3x3)
X_3x3 = BatchNormalization(axis=1, epsilon=0.00001, name='inception_3a_3x3_bn2')(X_3x3)
X_3x3 = Activation('relu')(X_3x3)
X_5x5 = Conv2D(16, (1, 1), data_format='channels_first', name='inception_3a_5x5_conv1')(X)
X_5x5 = BatchNormalization(axis=1, epsilon=0.00001, name='inception_3a_5x5_bn1')(X_5x5)
X_5x5 = Activation('relu')(X_5x5)
X_5x5 = ZeroPadding2D(padding=(2, 2), data_format='channels_first')(X_5x5)
X_5x5 = Conv2D(32, (5, 5), data_format='channels_first', name='inception_3a_5x5_conv2')(X_5x5)
X_5x5 = BatchNormalization(axis=1, epsilon=0.00001, name='inception_3a_5x5_bn2')(X_5x5)
X_5x5 = Activation('relu')(X_5x5)
X_pool = MaxPooling2D(pool_size=3, strides=2, data_format='channels_first')(X)
X_pool = Conv2D(32, (1, 1), data_format='channels_first', name='inception_3a_pool_conv')(X_pool)
X_pool = BatchNormalization(axis=1, epsilon=0.00001, name='inception_3a_pool_bn')(X_pool)
X_pool = Activation('relu')(X_pool)
X_pool = ZeroPadding2D(padding=((3, 4), (3, 4)), data_format='channels_first')(X_pool)
X_1x1 = Conv2D(64, (1, 1), data_format='channels_first', name='inception_3a_1x1_conv')(X)
X_1x1 = BatchNormalization(axis=1, epsilon=0.00001, name='inception_3a_1x1_bn')(X_1x1)
X_1x1 = Activation('relu')(X_1x1)
# CONCAT
inception = concatenate([X_3x3, X_5x5, X_pool, X_1x1], axis=1)
return inception
def build_cnn(input_shape, output_size):
kwargs = {"kernel_size": 3, "activation": "relu", "padding": "same"}
model = Sequential([
# conv1_*
Convolution2D(64, input_shape=input_shape, **kwargs),
BatchRenormalization(),
Convolution2D(64, **kwargs),
BatchRenormalization(),
MaxPooling2D(pool_size=(2, 2)),
Dropout(0.25),
# conv2_*
Convolution2D(128, **kwargs),
BatchRenormalization(),
Convolution2D(128, **kwargs),
BatchRenormalization(),
MaxPooling2D(pool_size=(2, 2)),
Dropout(0.25),
# conv3_*
Convolution2D(256, **kwargs),
BatchRenormalization(),
Convolution2D(256, **kwargs),
BatchRenormalization(),
MaxPooling2D(pool_size=(2, 2)),
Dropout(0.25),
# Fully connected
Flatten(),
Dense(1024),
Activation("relu"),
Dropout(0.5),
Dense(512),
Activation("relu"),
Dropout(0.5),
Dense(output_size),
Activation("softmax")
])
model.compile(loss="categorical_crossentropy",
optimizer="adam",
metrics=["accuracy"])
return model
def Train(self):
# self.loadDataFeature()
self.loadDataTxt()
self.train_and_test_split(0.75)
# model
model = Sequential()
# model.add(Dense(392, activation='relu'))
# model.add(Dense(128, activation='relu'))
# model.add(Dense(36, activation='softmax'))
#cnn model
model.add(
Conv2D(64, (3, 3), activation='relu', input_shape=(28, 28, 1)))
model.add(Conv2D(64, (3, 3), activation='relu'))
model.add(MaxPooling2D((2, 2)))
model.add(Conv2D(64, (3, 3), activation='relu'))
model.add(Conv2D(64, (3, 3), activation='relu'))
model.add(MaxPooling2D((2, 2)))
model.add(Flatten())
model.add(Dense(128, activation='relu'))
model.add(Dense(128, activation='relu'))
model.add(Dense(36, activation='softmax'))
# model.compile(loss='mse', optimizer='adam', metrics=['accuracy'])
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
model.fit(
self.train_data['data'],
self.train_data['class_name'],
batch_size=25,
epochs=100,
verbose=1,
validation_data=(self.test_data['data'],
self.test_data['class_name']),
)
self.model = model
model.save('digit_classification_model1.h5')
# Y_pred = model.predict(self.test_data['data'])
# self.metric(self.test_data['class_name'], Y_pred, data_type='binary')
self.metric()
def neural_network(input_shape):
inputs = keras.Input(shape=input_shape)
#Layer 1
x = MaxPooling2D(pool_size=(2, 2), name='MaxPooling2D_1')(inputs)
x = Conv2D(32, kernel_size=(5, 5), padding='same')(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = MaxPooling2D(pool_size=(4, 4))(x)
#Layer 2
x = Conv2D(64, kernel_size=(5, 5), padding='same', name='Conv2D_2')(x)
x = BatchNormalization()(x)
x = LeakyReLU(alpha=0.1)(x)
x = MaxPooling2D(pool_size=(2, 2), name='MaxPooling2D_3')(x)
x = Flatten(name='Flatten')(x)
#Layer 3
#model.add(Dense(256,name = 'Dense_1'))
#model.add(BatchNormalization(name = 'BatchNormalization_2'))
#model.add(LeakyReLU(alpha=0.1))
#model.add(Dropout(0.5,name = 'Dropout_1'))
#Layer 4
x = Dense(128, name='Dense_2')(x)
x = BatchNormalization(name='BatchNormalization_3')(x)
x = LeakyReLU(alpha=0.1)(x)
x = Dropout(0.5, name='Dropout_2')(x)
#Layer 5
x = Dense(128, name='Dense_3')(x)
x = BatchNormalization(name='BatchNormalization_4')(x)
x = LeakyReLU(alpha=0.1)(x)
#model.add(Dropout(0.5,name = 'Dropout_3'))
outputs = Dense(1, activation='sigmoid', name='Dense_4')(x)
model = Model(inputs, outputs)
return model
def inception_block_3b(X):
X_3x3 = fr_utils.conv2d_bn(X,
layer='inception_5b_3x3',
cv1_out=96,
cv1_filter=(1, 1),
cv2_out=384,
cv2_filter=(3, 3),
cv2_strides=(1, 1),
padding=(1, 1))
X_pool = MaxPooling2D(pool_size=3, strides=2, data_format='channels_first')(X)
X_pool = fr_utils.conv2d_bn(X_pool,
layer='inception_5b_pool',
cv1_out=96,
cv1_filter=(1, 1))
X_pool = ZeroPadding2D(padding=(1, 1), data_format='channels_first')(X_pool)
X_1x1 = fr_utils.conv2d_bn(X,
layer='inception_5b_1x1',
cv1_out=256,
cv1_filter=(1, 1))
inception = concatenate([X_3x3, X_pool, X_1x1], axis=1)
return inception
def main(_):
kernel_posterior_fn = tfp_layers_util.default_mean_field_normal_fn(
untransformed_scale_initializer=tf.compat.v1.initializers.
random_normal(mean=FLAGS.var, stddev=0.1))
encoder_w0 = tf.keras.Sequential([
tfp.layers.Convolution2DReparameterization(
filters=32,
kernel_size=3,
strides=(2, 2),
activation='relu',
padding='SAME',
kernel_posterior_fn=kernel_posterior_fn),
tfp.layers.Convolution2DReparameterization(
filters=48,
kernel_size=3,
strides=(2, 2),
activation='relu',
padding='SAME',
kernel_posterior_fn=kernel_posterior_fn),
MaxPooling2D(pool_size=(2, 2)),
tfp.layers.Convolution2DReparameterization(
filters=64,
kernel_size=3,
strides=(2, 2),
activation='relu',
padding='SAME',
kernel_posterior_fn=kernel_posterior_fn),
tf.keras.layers.Flatten(),
tfp.layers.DenseReparameterization(
FLAGS.dim_w, kernel_posterior_fn=kernel_posterior_fn),
])
decoder0 = tf.keras.Sequential([
tf.keras.layers.Dense(100, activation=tf.nn.relu),
tf.keras.layers.Dense(100, activation=tf.nn.relu),
tf.keras.layers.Dense(FLAGS.dim_y),
])
dim_output = FLAGS.dim_y
dim_input = FLAGS.dim_im * FLAGS.dim_im * 1
exp_name = '%s.beta-%g.update_lr-%g.trial-%d' % (
'np_bbb', FLAGS.beta, FLAGS.update_lr, FLAGS.trial)
checkpoint_dir = os.path.join(FLAGS.logdir, exp_name)
x_train, y_train = pickle.load(
tf.io.gfile.GFile(os.path.join(get_data_dir(), FLAGS.data[0]), 'rb'))
x_val, y_val = pickle.load(
tf.io.gfile.GFile(os.path.join(get_data_dir(), FLAGS.data[1]), 'rb'))
x_train, y_train = np.array(x_train), np.array(y_train)
y_train = y_train[:, :, -1, None]
x_val, y_val = np.array(x_val), np.array(y_val)
y_val = y_val[:, :, -1, None]
ds_train = tf.data.Dataset.from_generator(
functools.partial(gen, x_train, y_train),
(tf.float32, tf.float32, tf.float32, tf.float32),
(tf.TensorShape(
[None, FLAGS.update_batch_size * FLAGS.num_classes, dim_input]),
tf.TensorShape(
[None, FLAGS.update_batch_size * FLAGS.num_classes, dim_output]),
tf.TensorShape(
[None, FLAGS.update_batch_size * FLAGS.num_classes, dim_input]),
tf.TensorShape(
[None, FLAGS.update_batch_size * FLAGS.num_classes, dim_output])))
ds_val = tf.data.Dataset.from_generator(
functools.partial(gen, x_val, y_val),
(tf.float32, tf.float32, tf.float32, tf.float32),
(tf.TensorShape(
[None, FLAGS.update_batch_size * FLAGS.num_classes, dim_input]),
tf.TensorShape(
[None, FLAGS.update_batch_size * FLAGS.num_classes, dim_output]),
tf.TensorShape(
[None, FLAGS.update_batch_size * FLAGS.num_classes, dim_input]),
tf.TensorShape(
[None, FLAGS.update_batch_size * FLAGS.num_classes, dim_output])))
inputa, labela, inputb, labelb = ds_train.make_one_shot_iterator(
).get_next()
input_tensors = {'inputa': inputa,\
'inputb': inputb,\
'labela': labela, 'labelb': labelb}
inputa_val, labela_val, inputb_val, labelb_val = ds_val.make_one_shot_iterator(
).get_next()
metaval_input_tensors = {'inputa': inputa_val,\
'inputb': inputb_val,\
'labela': labela_val, 'labelb': labelb_val}
loss, train_op, facto = construct_model(input_tensors,
encoder_w0,
decoder0,
prefix='metatrain_')
loss_val = construct_model(metaval_input_tensors,
encoder_w0,
decoder0,
prefix='metaval_')
###########
summ_op = tf.summary.merge_all()
sess = tf.InteractiveSession()
summary_writer = tf.summary.FileWriter(checkpoint_dir, sess.graph)
tf.global_variables_initializer().run()
PRINT_INTERVAL = 50 # pylint: disable=invalid-name
SUMMARY_INTERVAL = 5 # pylint: disable=invalid-name
prelosses, prelosses_val = [], []
old_time = time.time()
for itr in range(FLAGS.num_updates):
feed_dict = {facto: FLAGS.facto}
if itr % SUMMARY_INTERVAL == 0:
summary, cost, cost_val = sess.run([summ_op, loss, loss_val],
feed_dict)
summary_writer.add_summary(summary, itr)
prelosses.append(cost) # 0 step loss on training set
prelosses_val.append(
cost_val) # 0 step loss on meta_val training set
sess.run(train_op, feed_dict)
if (itr != 0) and itr % PRINT_INTERVAL == 0:
print('Iteration ' + str(itr) + ': ' + str(np.mean(prelosses)),
'time =',
time.time() - old_time)
prelosses = []
old_time = time.time()
print('Validation results: ' + str(np.mean(prelosses_val)))
prelosses_val = []
def main(_):
dim_output = FLAGS.dim_y
dim_input = FLAGS.dim_im * FLAGS.dim_im * 1
exp_name = '%s.beta-%g.meta_lr-%g.update_lr-%g.trial-%d' % (
'maml_bbb', FLAGS.beta, FLAGS.meta_lr, FLAGS.update_lr, FLAGS.trial)
checkpoint_dir = os.path.join(FLAGS.logdir, exp_name)
x_train, y_train = pickle.load(
tf.io.gfile.GFile(os.path.join(get_data_dir(), FLAGS.data[0]), 'rb'))
x_val, y_val = pickle.load(
tf.io.gfile.GFile(os.path.join(get_data_dir(), FLAGS.data[1]), 'rb'))
x_train, y_train = np.array(x_train), np.array(y_train)
y_train = y_train[:, :, -1, None]
x_val, y_val = np.array(x_val), np.array(y_val)
y_val = y_val[:, :, -1, None]
ds_train = tf.data.Dataset.from_generator(
functools.partial(gen, x_train, y_train),
(tf.float32, tf.float32, tf.float32, tf.float32),
(tf.TensorShape(
[None, FLAGS.update_batch_size * FLAGS.num_classes, dim_input]),
tf.TensorShape(
[None, FLAGS.update_batch_size * FLAGS.num_classes, dim_output]),
tf.TensorShape(
[None, FLAGS.update_batch_size * FLAGS.num_classes, dim_input]),
tf.TensorShape(
[None, FLAGS.update_batch_size * FLAGS.num_classes, dim_output])))
ds_val = tf.data.Dataset.from_generator(
functools.partial(gen, x_val, y_val),
(tf.float32, tf.float32, tf.float32, tf.float32),
(tf.TensorShape(
[None, FLAGS.update_batch_size * FLAGS.num_classes, dim_input]),
tf.TensorShape(
[None, FLAGS.update_batch_size * FLAGS.num_classes, dim_output]),
tf.TensorShape(
[None, FLAGS.update_batch_size * FLAGS.num_classes, dim_input]),
tf.TensorShape(
[None, FLAGS.update_batch_size * FLAGS.num_classes, dim_output])))
kernel_posterior_fn = tfp_layers_util.default_mean_field_normal_fn(
untransformed_scale_initializer=tf.compat.v1.initializers.
random_normal(mean=FLAGS.var, stddev=0.1))
encoder_w = tf.keras.Sequential([
tfp.layers.Convolution2DReparameterization(
filters=32,
kernel_size=3,
strides=(2, 2),
activation='relu',
padding='SAME',
kernel_posterior_fn=kernel_posterior_fn),
tfp.layers.Convolution2DReparameterization(
filters=48,
kernel_size=3,
strides=(2, 2),
activation='relu',
padding='SAME',
kernel_posterior_fn=kernel_posterior_fn),
MaxPooling2D(pool_size=(2, 2)),
tfp.layers.Convolution2DReparameterization(
filters=64,
kernel_size=3,
strides=(2, 2),
activation='relu',
padding='SAME',
kernel_posterior_fn=kernel_posterior_fn),
tf.keras.layers.Flatten(),
tfp.layers.DenseReparameterization(
FLAGS.dim_w, kernel_posterior_fn=kernel_posterior_fn),
])
xa, labela, xb, labelb = ds_train.make_one_shot_iterator().get_next()
xa = tf.reshape(xa, [-1, 128, 128, 1])
xb = tf.reshape(xb, [-1, 128, 128, 1])
with tf.variable_scope('encoder'):
inputa = encoder_w(xa)
inputa = tf.reshape(
inputa, [-1, FLAGS.update_batch_size * FLAGS.num_classes, FLAGS.dim_w])
inputb = encoder_w(xb)
inputb = tf.reshape(
inputb, [-1, FLAGS.update_batch_size * FLAGS.num_classes, FLAGS.dim_w])
input_tensors = {'inputa': inputa,\
'inputb': inputb, \
'labela': labela, 'labelb': labelb}
# n_task * n_im_per_task * dim_w
xa_val, labela_val, xb_val, labelb_val = ds_val.make_one_shot_iterator(
).get_next()
xa_val = tf.reshape(xa_val, [-1, 128, 128, 1])
xb_val = tf.reshape(xb_val, [-1, 128, 128, 1])
inputa_val = encoder_w(xa_val)
inputa_val = tf.reshape(
inputa_val,
[-1, FLAGS.update_batch_size * FLAGS.num_classes, FLAGS.dim_w])
inputb_val = encoder_w(xb_val)
inputb_val = tf.reshape(
inputb_val,
[-1, FLAGS.update_batch_size * FLAGS.num_classes, FLAGS.dim_w])
metaval_input_tensors = {'inputa': inputa_val,\
'inputb': inputb_val, \
'labela': labela_val, 'labelb': labelb_val}
# num_updates = max(self.test_num_updates, FLAGS.num_updates)
model = MAML(encoder_w, FLAGS.dim_w, dim_output)
model.construct_model(input_tensors=input_tensors, prefix='metatrain_')
model.construct_model(input_tensors=metaval_input_tensors,
prefix='metaval_',
test_num_updates=FLAGS.test_num_updates)
# model.construct_model(input_tensors=input_tensors, prefix='metaval_')
model.summ_op = tf.summary.merge_all()
sess = tf.InteractiveSession()
tf.global_variables_initializer().run()
if FLAGS.train:
train(model, sess, checkpoint_dir, exp_name)
NN2 = Sequential()
NN2.add(InputLayer(input_shape=(Nmaturities, Nstrikes, 1)))
NN2.add(
Conv2D(64, (3, 3),
use_bias=True,
padding='valid',
strides=(1, 1),
activation='tanh'))
NN2.add(
Conv2D(64, (2, 2),
use_bias=True,
padding='valid',
strides=(1, 1),
activation='tanh'))
NN2.add(MaxPooling2D(pool_size=(2, 2)))
NN2.add(
Conv2D(64, (2, 2),
use_bias=True,
padding='valid',
strides=(1, 1),
activation='tanh'))
NN2.add(ZeroPadding2D(padding=(1, 1)))
NN2.add(
Conv2D(64, (2, 2),
use_bias=True,
padding='valid',
strides=(1, 1),
activation='tanh'))
NN2.add(ZeroPadding2D(padding=(1, 1)))
NN2.add(
layer_sizes = [4, 8, 16]
conv_layers = [1, 2]
for dense_layer in dense_layers:
for layer_size in layer_sizes:
for conv_layer in conv_layers:
NAME = f'Pneumonia-{IMG_SIZE}px-{NUM_SAMPLES}samples-{conv_layer}conv-{layer_size}nodes-{dense_layer}dense-{int(time.time())}'
tensorboard = TensorBoard(log_dir=f'logs/{NAME}')
print(NAME)
model = Sequential()
# format: Num of filters, window/step, dimensions
model.add(Conv2D(layer_size, (3, 3),
input_shape=x_train.shape[1:]))
model.add(Activation("relu"))
model.add(MaxPooling2D(pool_size=(2, 2)))
print('Layer 0 generated')
for i in range(conv_layer - 1):
print(f'Layer {i + 1} generated.')
model.add(Conv2D(layer_size, (3, 3)))
model.add(Activation("relu"))
model.add(MaxPooling2D(pool_size=(2, 2)))
model.add(Flatten())
for l in range(dense_layer):
model.add(Dense(layer_size))
model.add(Activation("relu"))
model.add(Dense(1))
# Final activation function
def stack_layers(inputs, layers, kernel_initializer='glorot_uniform'):
'''
Builds the architecture of the network by applying each layer specified in layers to inputs.
inputs: a dict containing input_types and input_placeholders for each key and value pair, respecively.
for spectralnet, this means the input_types 'Unlabeled' and 'Orthonorm'*
layers: a list of dicts containing all layers to be used in the network, where each dict describes
one such layer. each dict requires the key 'type'. all other keys are dependent on the layer
type
kernel_initializer: initialization configuration passed to keras (see keras initializers)
returns: outputs, a dict formatted in much the same way as inputs. it contains input_types and
output_tensors for each key and value pair, respectively, where output_tensors are
the outputs of the input_placeholders in inputs after each layer in layers is applied
* this is necessary since spectralnet takes multiple inputs and performs special computations on the
orthonorm layer
'''
outputs = dict()
for key in inputs:
outputs[key] = inputs[key]
for layer in layers:
# check for l2_reg argument
l2_reg = layer.get('l2_reg')
if l2_reg:
l2_reg = l2(layer['l2_reg'])
# create the layer
if layer['type'] == 'softplus_reg':
l = Dense(layer['size'],
activation='softplus',
kernel_initializer=kernel_initializer,
kernel_regularizer=l2(0.001),
name=layer.get('name'))
elif layer['type'] == 'softplus':
l = Dense(layer['size'],
activation='softplus',
kernel_initializer=kernel_initializer,
kernel_regularizer=l2_reg,
name=layer.get('name'))
elif layer['type'] == 'softmax':
l = Dense(layer['size'],
activation='softmax',
kernel_initializer=kernel_initializer,
kernel_regularizer=l2_reg,
name=layer.get('name'))
elif layer['type'] == 'tanh':
l = Dense(layer['size'],
activation='tanh',
kernel_initializer=kernel_initializer,
kernel_regularizer=l2_reg,
name=layer.get('name'))
elif layer['type'] == 'relu':
l = Dense(layer['size'],
activation='relu',
kernel_initializer=kernel_initializer,
kernel_regularizer=l2_reg,
name=layer.get('name'))
elif layer['type'] == 'selu':
l = Dense(layer['size'],
activation='selu',
kernel_initializer=kernel_initializer,
kernel_regularizer=l2_reg,
name=layer.get('name'))
elif layer['type'] == 'Conv2D':
l = Conv2D(layer['channels'],
kernel_size=layer['kernel'],
activation='relu',
data_format='channels_last',
kernel_regularizer=l2_reg,
name=layer.get('name'))
elif layer['type'] == 'BatchNormalization':
l = BatchNormalization(name=layer.get('name'))
elif layer['type'] == 'MaxPooling2D':
l = MaxPooling2D(pool_size=layer['pool_size'],
data_format='channels_first',
name=layer.get('name'))
elif layer['type'] == 'Dropout':
l = Dropout(layer['rate'], name=layer.get('name'))
elif layer['type'] == 'Flatten':
l = Flatten(name=layer.get('name'))
elif layer['type'] == 'Orthonorm':
l = Orthonorm(outputs['Orthonorm'], name=layer.get('name'))
else:
raise ValueError("Invalid layer type '{}'".format(layer['type']))
# apply the layer to each input in inputs
for k in outputs:
outputs[k] = l(outputs[k])
return outputs
def main(_):
encoder_w0 = tf.keras.Sequential([
Conv2D(filters=32,
kernel_size=3,
strides=(2, 2),
activation='relu',
padding='same'),
Conv2D(filters=48,
kernel_size=3,
strides=(2, 2),
activation='relu',
padding='same'),
MaxPooling2D(pool_size=(2, 2)),
Conv2D(filters=64,
kernel_size=3,
strides=(2, 2),
activation='relu',
padding='same'),
tf.keras.layers.Flatten(),
tf.keras.layers.Dense(FLAGS.dim_w),
])
decoder0 = tf.keras.Sequential([
tf.keras.layers.Dense(100, activation=tf.nn.relu),
tf.keras.layers.Dense(100, activation=tf.nn.relu),
tf.keras.layers.Dense(FLAGS.dim_y),
])
dim_output = FLAGS.dim_y
dim_input = FLAGS.dim_im * FLAGS.dim_im * 1
exp_basename = 'np_vanilla'
if FLAGS.noise_scale:
exp_basename = 'np_vanilla_noised_scale' + str(FLAGS.noise_scale)
elif FLAGS.num_noise > 0:
exp_basename = 'np_vanilla_noise' + str(FLAGS.num_noise)
if FLAGS.weight_decay:
exp_name = '%s.update_lr-%g.beta-%g.trial-%d' % (
exp_basename, FLAGS.update_lr, FLAGS.beta, FLAGS.trial)
else:
exp_name = '%s.update_lr-%g.trial-%d' % (exp_basename, FLAGS.update_lr,
FLAGS.trial)
if FLAGS.testing:
exp_name += "-test"
checkpoint_dir = os.path.join(FLAGS.logdir, exp_name)
if FLAGS.testing:
data = [FLAGS.data[0], FLAGS.data[1]]
else:
data = [FLAGS.data[0], FLAGS.data[0]]
x_train, y_train = pickle.load(
tf.io.gfile.GFile(os.path.join(get_data_dir(), data[0]), 'rb'))
x_val, y_val = pickle.load(
tf.io.gfile.GFile(os.path.join(get_data_dir(), data[1]), 'rb'))
if not FLAGS.testing:
# Split the train dataset into val and train
x_train, y_train = x_train[:-5], y_train[:-5]
x_val, y_val = x_val[-5:], y_val[-5:]
x_train, y_train = np.array(x_train), np.array(y_train)
y_train = y_train[:, :, -1, None]
x_val, y_val = np.array(x_val), np.array(y_val)
y_val = y_val[:, :, -1, None]
ds_train = tf.data.Dataset.from_generator(
functools.partial(gen, x_train, y_train, True),
(tf.float32, tf.float32, tf.float32, tf.float32),
(tf.TensorShape(
[None, FLAGS.update_batch_size * FLAGS.num_classes, dim_input]),
tf.TensorShape(
[None, FLAGS.update_batch_size * FLAGS.num_classes, dim_output]),
tf.TensorShape(
[None, FLAGS.update_batch_size * FLAGS.num_classes, dim_input]),
tf.TensorShape(
[None, FLAGS.update_batch_size * FLAGS.num_classes, dim_output])))
ds_val = tf.data.Dataset.from_generator(
functools.partial(gen, x_val, y_val, False),
(tf.float32, tf.float32, tf.float32, tf.float32),
(tf.TensorShape(
[None, FLAGS.update_batch_size * FLAGS.num_classes, dim_input]),
tf.TensorShape(
[None, FLAGS.update_batch_size * FLAGS.num_classes, dim_output]),
tf.TensorShape(
[None, FLAGS.update_batch_size * FLAGS.num_classes, dim_input]),
tf.TensorShape(
[None, FLAGS.update_batch_size * FLAGS.num_classes, dim_output])))
inputa, labela, inputb, labelb = ds_train.make_one_shot_iterator(
).get_next()
input_tensors = {'inputa': inputa,\
'inputb': inputb,\
'labela': labela, 'labelb': labelb}
inputa_val, labela_val, inputb_val, labelb_val = ds_val.make_one_shot_iterator(
).get_next()
metaval_input_tensors = {'inputa': inputa_val,\
'inputb': inputb_val,\
'labela': labela_val, 'labelb': labelb_val}
loss, train_op, facto = construct_model(input_tensors,
encoder_w0,
decoder0,
prefix='metatrain_')
loss_val = construct_model(metaval_input_tensors,
encoder_w0,
decoder0,
prefix='metaval_')
###########
summ_op = tf.summary.merge_all()
sess = tf.InteractiveSession()
summary_writer = tf.summary.FileWriter(checkpoint_dir, sess.graph)
tf.global_variables_initializer().run()
PRINT_INTERVAL = 50 # pylint: disable=invalid-name
SUMMARY_INTERVAL = 5 # pylint: disable=invalid-name
val_step = []
train_k, val_k = [], []
# scratch buffers
prelosses, prelosses_val = [], []
old_time = time.time()
for itr in range(FLAGS.num_updates):
feed_dict = {facto: FLAGS.facto}
if itr % SUMMARY_INTERVAL == 0:
summary, cost, cost_val = sess.run([summ_op, loss, loss_val],
feed_dict)
summary_writer.add_summary(summary, itr)
prelosses.append(cost) # 0 step loss on training set
prelosses_val.append(
cost_val) # 0 step loss on meta_val training set
sess.run(train_op, feed_dict)
if (itr != 0) and itr % PRINT_INTERVAL == 0:
val_step.append(itr)
print('Iteration ' + str(itr) + ': ' + str(np.mean(prelosses)),
'time =',
time.time() - old_time)
prelosses = []
old_time = time.time()
print('Validation results: ' + str(np.mean(prelosses_val)))
# Dump (theres no inner loss?)
train_k.append(np.mean(prelosses))
val_k.append(np.mean(prelosses_val))
all_ = (val_step, train_k, val_k)
pickle.dump(all_,
open(os.path.join(checkpoint_dir, 'results.p'), 'wb'))
prelosses_val = []
prelosses = []
def InceptionModel(input_shape):
"""
Implementation of the Inception model used for FaceNet
Arguments:
input_shape -- shape of the images of the dataset
Returns:
model -- a Model() instance in Keras
"""
# Define the input as a tensor with shape input_shape
X_input = Input(input_shape)
# Zero-Padding
X = ZeroPadding2D((3, 3))(X_input)
# First Block
X = Conv2D(64, (7, 7), strides=(2, 2), name='conv1')(X)
X = BatchNormalization(axis=1, name='bn1')(X)
X = Activation('relu')(X)
# Zero-Padding + MAXPOOL
X = ZeroPadding2D((1, 1))(X)
X = MaxPooling2D((3, 3), strides=2)(X)
# Second Block
X = Conv2D(64, (1, 1), strides=(1, 1), name='conv2')(X)
X = BatchNormalization(axis=1, epsilon=0.00001, name='bn2')(X)
X = Activation('relu')(X)
# Zero-Padding + MAXPOOL
X = ZeroPadding2D((1, 1))(X)
# Second Block
X = Conv2D(192, (3, 3), strides=(1, 1), name='conv3')(X)
X = BatchNormalization(axis=1, epsilon=0.00001, name='bn3')(X)
X = Activation('relu')(X)
# Zero-Padding + MAXPOOL
X = ZeroPadding2D((1, 1))(X)
X = MaxPooling2D(pool_size=3, strides=2)(X)
# Inception 1: a/b/c
X = inception_block_1a(X)
X = inception_block_1b(X)
X = inception_block_1c(X)
# Inception 2: a/b
X = inception_block_2a(X)
X = inception_block_2b(X)
# Inception 3: a/b
X = inception_block_3a(X)
X = inception_block_3b(X)
# Top layer
X = AveragePooling2D(pool_size=(3, 3), strides=(1, 1), data_format='channels_first')(X)
X = Flatten()(X)
X = Dense(128, name='dense_layer')(X)
# L2 normalization
X = Lambda(lambda x: K.l2_normalize(x, axis=1))(X)
# Create model instance
model = Model(inputs=X_input, outputs=X, name='FaceRecoModel')
return model
