def optimizer_class():
from keras.optimizers import SGD, Adam, RMSprop, Nadam, TFOptimizer
# TODO(KGF): check this fix; lr, tf were undefined in neglected
# serial runner.py, since mpi_runner.py has been the main tool
import tensorflow as tf
if conf['model']['optimizer'] == 'sgd':
return SGD(lr=conf['model']['lr'], clipnorm=conf['model']['clipnorm'])
elif conf['model']['optimizer'] == 'momentum_sgd':
return SGD(lr=conf['model']['lr'],
clipnorm=conf['model']['clipnorm'],
decay=1e-6,
momentum=0.9)
elif conf['model']['optimizer'] == 'tf_momentum_sgd':
return TFOptimizer(
tf.train.MomentumOptimizer(learning_rate=conf['model']['lr'],
momentum=0.9))
elif conf['model']['optimizer'] == 'adam':
return Adam(lr=conf['model']['lr'], clipnorm=conf['model']['clipnorm'])
elif conf['model']['optimizer'] == 'tf_adam':
return TFOptimizer(
tf.train.AdamOptimizer(learning_rate=conf['model']['lr']))
elif conf['model']['optimizer'] == 'rmsprop':
return RMSprop(lr=conf['model']['lr'],
clipnorm=conf['model']['clipnorm'])
elif conf['model']['optimizer'] == 'nadam':
return Nadam(lr=conf['model']['lr'],
clipnorm=conf['model']['clipnorm'])
else:
print("Optimizer not implemented yet")
exit(1)
def compile(self, optimizer, clipnorm, loss='mse'):
# TODO(KGF): check the following import taken from runner.py
# Was not in this file, originally.
from keras.optimizers import SGD, Adam, RMSprop, Nadam, TFOptimizer
if optimizer == 'sgd':
optimizer_class = SGD(lr=self.DUMMY_LR, clipnorm=clipnorm)
elif optimizer == 'momentum_sgd':
optimizer_class = SGD(lr=self.DUMMY_LR,
clipnorm=clipnorm,
decay=1e-6,
momentum=0.9)
elif optimizer == 'tf_momentum_sgd':
optimizer_class = TFOptimizer(
tf.train.MomentumOptimizer(learning_rate=self.DUMMY_LR,
momentum=0.9))
elif optimizer == 'adam':
optimizer_class = Adam(lr=self.DUMMY_LR, clipnorm=clipnorm)
elif optimizer == 'tf_adam':
optimizer_class = TFOptimizer(
tf.train.AdamOptimizer(learning_rate=self.DUMMY_LR))
elif optimizer == 'rmsprop':
optimizer_class = RMSprop(lr=self.DUMMY_LR, clipnorm=clipnorm)
elif optimizer == 'nadam':
optimizer_class = Nadam(lr=self.DUMMY_LR, clipnorm=clipnorm)
else:
print("Optimizer not implemented yet")
exit(1)
self.model.compile(optimizer=optimizer_class, loss=loss)
self.ensure_equal_weights()
def optimizer_class():
from keras.optimizers import SGD, Adam, RMSprop, Nadam, TFOptimizer
if conf['model']['optimizer'] == 'sgd':
return SGD(lr=conf['model']['lr'], clipnorm=conf['model']['clipnorm'])
elif conf['model']['optimizer'] == 'momentum_sgd':
return SGD(lr=conf['model']['lr'],
clipnorm=conf['model']['clipnorm'],
decay=1e-6,
momentum=0.9)
elif conf['model']['optimizer'] == 'tf_momentum_sgd':
return TFOptimizer(
tf.train.MomentumOptimizer(learning_rate=conf['model']['lr'],
momentum=0.9))
elif conf['model']['optimizer'] == 'adam':
return Adam(lr=conf['model']['lr'], clipnorm=conf['model']['clipnorm'])
elif conf['model']['optimizer'] == 'tf_adam':
return TFOptimizer(
tf.train.AdamOptimizer(learning_rate=conf['model']['lr']))
elif conf['model']['optimizer'] == 'rmsprop':
return RMSprop(lr=conf['model']['lr'],
clipnorm=conf['model']['clipnorm'])
elif conf['model']['optimizer'] == 'nadam':
return Nadam(lr=conf['model']['lr'],
clipnorm=conf['model']['clipnorm'])
else:
print("Optimizer not implemented yet")
exit(1)
def train_model(self):
"""Train a model according to options using generator.
This saves results after each epoch and attempts to resume training
from the last saved point.
# Arguments
run_path: the path to the files for a run
model: a Keras model
options: RunOptions object that specifies the run
generator: a Generator object to generate the training and
validation data
"""
print(self.model.summary())
train_gen = self.generator.make_split_generator(
TRAIN, target_size=self.options.target_size,
batch_size=self.options.batch_size,
shuffle=True, augment_methods=self.options.augment_methods,
normalize=True, only_xy=True)
validation_gen = self.generator.make_split_generator(
VALIDATION, target_size=self.options.target_size,
batch_size=self.options.batch_size,
shuffle=True, augment_methods=self.options.augment_methods,
normalize=True, only_xy=True)
if self.options.optimizer == ADAM:
optimizer = Adam(lr=self.options.init_lr,
decay=self.options.lr_step_decay)
elif self.options.optimizer == RMS_PROP:
optimizer = RMSprop(lr=self.options.init_lr,
rho=self.options.rho,
epsilon=self.options.epsilon,
decay=self.options.lr_step_decay)
elif self.options.optimizer == 'sgd':
optimizer = SGD(
lr=self.options.init_lr,
momentum=self.options.momentum,
nesterov=self.options.nesterov
)
elif self.options.optimizer == YELLOWFIN:
optimizer = TFOptimizer(YFOptimizer(
learning_rate=self.options.init_lr,
momentum=self.options.momentum))
self.model.compile(
optimizer, self.loss_function, metrics=self.metrics)
initial_epoch = self.get_initial_epoch()
callbacks = self.make_callbacks()
self.model.fit_generator(
train_gen,
initial_epoch=initial_epoch,
steps_per_epoch=self.options.steps_per_epoch,
epochs=self.options.epochs,
validation_data=validation_gen,
validation_steps=self.options.validation_steps,
callbacks=callbacks)
def parse_optimizer(string_input):
# motivation for tf optimizers https://github.com/fchollet/keras/issues/1021#issuecomment-270566219
if string_input.startswith("tf."):
from keras.optimizers import TFOptimizer
import tensorflow as tf
print("> using tensorflow optimizer")
if string_input == "tf.sgd":
return TFOptimizer(
tf.train.GradientDescentOptimizer(learning_rate=0.1))
if string_input == "tf.rmsprop":
# lr taken from keras standard impl
return TFOptimizer(tf.train.RMSPropOptimizer(learning_rate=0.001))
else:
return string_input
def get(identifier):
"""Retrieves a Keras Optimizer instance.
# Arguments
identifier: Optimizer identifier, one of
- String: name of an optimizer
- Dictionary: configuration dictionary.
- Keras Optimizer instance (it will be returned unchanged).
- TensorFlow Optimizer instance
(it will be wrapped as a Keras Optimizer).
# Returns
A Keras Optimizer instance.
# Raises
ValueError: If `identifier` cannot be interpreted.
"""
if K.backend() == 'tensorflow':
# Wrap TF optimizer instances
if isinstance(identifier, tf.train.Optimizer):
return TFOptimizer(identifier)
if isinstance(identifier, dict):
return deserialize(identifier)
elif isinstance(identifier, six.string_types):
config = {'class_name': str(identifier), 'config': {}}
return deserialize(config)
if isinstance(identifier, Optimizer):
return identifier
else:
raise ValueError('Could not interpret optimizer identifier:',
identifier)
def __init__(self, featurizer, opts):
self.featurizer = featurizer
self.dropout_p = opts.dropout_p
self.l2_lambda = opts.l2_lambda
self.l1_lambda = opts.l1_lambda
self.n_features = opts.n_features
self.dense_dims = opts.dense_dims
self.epochs = opts.epochs
self.steps_per_epoch = opts.steps_per_epoch
embeddings = BaseWordEmbeddings(self.dense_dims, self.n_features, self.l1_lambda, self.l2_lambda,
self.dropout_p)
self.nn_rank_model = dict()
self.title_embedding = embeddings
self.abstract_embedding = embeddings
self.title_embedding_multiplier = LambdaScalarMultiplier(name='title-scalar-multiplier')
self.abstract_embedding_multiplier = LambdaScalarMultiplier(name='abstract-scalar-multiplier')
self.model, self.embedding_model = self.__compile_dense_model()
optimizer = TFOptimizer(tf.contrib.opt.LazyAdamOptimizer(learning_rate=opts.learning_rate))
self.model.compile(optimizer=optimizer, loss=triplet_loss,
metrics=['accuracy'])
self.callbacks = [
SaveModelWeights([('dense', self.model), ('embedding', self.embedding_model)], opts.weights_directory,
opts.checkpoint_frequency)
]
def best_model_evaluate(train_x,
train_y,
test_x,
test_y,
file_output_path,
file_name,
batch_size=256):
"""
Evaluate accuracy on training and test set.
Parameters
----------
train_x : np.array(float)
Training `x` set (training features).
train_y : np.array(int)
Training `y` set (training labels).
test_x : np.array(float)
Test `x` set (test features).
test_y : np.array(int)
Test `y` set (test labels).
file_output_path : str
Path to store h5 files generated by custom BestAccs callback.
file_name : str
Name of file that contains weights
batch_size : int, optional
Size of batch (amount of samples) for model fitting.
Returns
-------
acc_train : float
Accuracy on training set.
acc_test : float
Accuracy on test set.
"""
initializer = 'zeros'
lr = 0.01
optimizer = TFOptimizer(tf.train.GradientDescentOptimizer(lr))
model = Sequential()
model.add(
Dense(1,
kernel_initializer=initializer,
bias_initializer=initializer,
input_dim=train_x.shape[1],
activation='sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
file_output_path = 'samplingHypersurface'
model.load_weights(os.path.join(file_output_path, file_name))
metrics_train = model.evaluate(train_x, train_y, batch_size=256, verbose=0)
acc_train = metrics_train[1]
metrics_test = model.evaluate(test_x, test_y, batch_size=256, verbose=0)
acc_test = metrics_test[1]
return acc_train, acc_test
def get_optimizer():
'''
Get optimizer for learning the network
@return: optimizing function
'''
optimizer = TFOptimizer(tf.train.RMSPropOptimizer(learning_rate=LEARNING_RATE,
decay=DECAY,
momentum=0.9)) # Tensorflow RMSProp
return optimizer
def best_learning_rate(train_x, train_y, test_x, test_y, lr, batch_size=256):
"""
Perform 10 random initializations of model, then compile and fit it.
Model is initialized randomly (random_uniform), compiled and fitted.
Operation is repeated 10 times. Then all histories are returned as a list.
Parameters
----------
train_x : np.array(float)
Training `x` set (training features).
train_y : np.array(int)
Training `y` set (training labels).
test_x : np.array(float)
Test `x` set (test features).
test_y : np.array(int)
Test `y` set (test labels).
lr : float
Learning rate.
batch_size : int, optional
Size of batch (amount of samples) for model fitting.
Returns
-------
history_set : list(keras.callbacks.History object)
History of loss, accuracy, validation loss and validation
accuracy during model fitting.
"""
optimizer = TFOptimizer(tf.train.GradientDescentOptimizer(lr))
history_set = []
for i in range(10):
model = Sequential()
initializer = RandomUniform(minval=-1.0, maxval=1.0, seed=None)
model.add(
Dense(1,
kernel_initializer=initializer,
bias_initializer=initializer,
input_dim=train_x.shape[1],
activation='sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
history = model.fit(train_x,
train_y,
epochs=1000,
validation_data=(test_x, test_y),
batch_size=batch_size,
verbose=0)
history_set.append(history)
return history_set
def best_batch_size(train_x, train_y, epochs=200):
"""
Determine optimal batch size.
Parameters
----------
train_x : np.array(float)
Training `x` set (training features).
train_y : np.array(int)
Training `y` set (training labels).
epochs : int, optional
Number of epochs.
Returns
-------
batch_size : int
Size of most efficient (fastest) batch size.
batch_exe_time: list
List of execution time for given range of batch size.
"""
lr = 0.005
optimizer = TFOptimizer(tf.train.GradientDescentOptimizer(lr))
batch_size_limit = int(np.log2(train_x.shape[0])) + 1
batch_size_set = [2**x for x in range(5, batch_size_limit + 1)]
model = Sequential()
model.add(
Dense(1,
kernel_initializer='zeros',
bias_initializer='zeros',
input_dim=train_x.shape[1],
activation='sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
# Measeure of execution time for each batch_size
batches_exe_time = []
for batch_size in batch_size_set:
time_start = time()
model.fit(train_x,
train_y,
epochs=epochs,
batch_size=batch_size,
verbose=0)
time_end = time() - time_start
batches_exe_time.append([time_end, batch_size])
batches_exe_time.sort()
batch_size = batches_exe_time[0][1]
return batch_size, batches_exe_time
def __init__(self, data_shape, choke_shape, filename=None):
self.data_shape = data_shape
self.choke_shape = choke_shape
self.__define()
self.load_file(self.model_ae, filename)
optimizer = TFOptimizer(tf.train.AdadeltaOptimizer())
self.model_encoder.compile(optimizer=optimizer, loss=self.rmsle, metrics=[self.rmsle])
self.model_decoder.compile(optimizer=optimizer, loss=self.rmsle, metrics=[self.rmsle])
self.model_ae.compile(optimizer=optimizer, loss=self.rmsle, metrics=[self.rmsle])
def get_tf_model(self, ground_truth, method = config.model_type):
global_step = tf.Variable(0, trainable=False)
learn_rate = tf.train.cosine_decay_restarts(learning_rate=0.001, global_step=global_step, first_decay_steps=100)
crispr_model = getattr(self, "_{!s}__cas9_{!s}_model".format(self.__class__.__name__, method),
self.__cas9_rnn_model)()
loss = tf.losses.mean_squared_error(ground_truth, crispr_model.output)
rmsprop_optimizer = tf.train.RMSPropOptimizer(learning_rate=learn_rate).minimize(loss, global_step=global_step)
custimized_rmsprop = TFOptimizer(rmsprop_optimizer)
crispr_model.compile(optimizer=custimized_rmsprop, loss='mse', metrics=[utils.revised_mse_loss, 'mse'])
return crispr_model
def build_model(self, existing_embedding_weights,
existing_context_weights):
stddev = 1.0 / self.vector_dim
initializer = RandomNormal(mean=0.0, stddev=stddev, seed=None)
target_input = Input(shape=(1, ), name="target_input")
target_layer = Embedding(
input_dim=self.vocabulary_size,
output_dim=self.vector_dim,
input_length=1,
name="target_layer",
embeddings_initializer=initializer)(target_input)
context_input = Input(shape=(1, ), name="context_input")
context_layer = Embedding(
input_dim=self.vocabulary_size,
output_dim=self.vector_dim,
input_length=1,
name="context_layer",
embeddings_initializer=initializer)(context_input)
merged = dot([target_layer, context_layer],
axes=2,
normalize=False,
name="dot")
merged = Flatten()(merged)
output = Dense(1, activation='sigmoid', name="output")(merged)
optimizer = TFOptimizer(tf.train.AdagradOptimizer(0.1))
model = Model(inputs=[target_input, context_input], outputs=output)
model.compile(loss="binary_crossentropy",
optimizer=optimizer,
metrics=[binary_accuracy])
embedding_weights = model.get_layer("target_layer").get_weights()
context_weights = model.get_layer("context_layer").get_weights()
for i in range(0, len(existing_embedding_weights)):
embedding_weights[0][i] = existing_embedding_weights[i]
for i in range(0, len(existing_context_weights)):
context_weights[0][i] = existing_context_weights[i]
model.get_layer("target_layer").set_weights(embedding_weights)
model.get_layer("context_layer").set_weights(context_weights)
model.summary()
self.model = model
def create_optim(self, **args):
if self.optim == 'yellowfin':
filtered_args = filter_args(YFOptimizer, args)
print(filtered_args)
opt = TFOptimizer(YFOptimizer(**filtered_args))
elif self.optim == 'ftml':
filtered_args = filter_args(FTML, args)
opt = FTML(**filtered_args)
else:
for name, optim_class in inspect.getmembers(optimizers):
if inspect.isclass(optim_class) and name[0] == name[0].upper():
filtered_args = filter_args(optim_class, args)
return optim_class(**filtered_args)
return opt
def get_optimizer(opt_params, lr):
"""Helper to get optimizer from text params
Parameters
----------
opt_params: dict
Dictionary containing optimization function name and learning rate decay
lr: float
Initial learning rate
Return
------
opt_function: Keras optimizer
"""
if opt_params['opt_func'] == 'sgd':
return SGD(lr=lr, momentum=opt_params['momentum'])
elif opt_params['opt_func'] == 'adam':
return Adam(lr=lr)
elif opt_params['opt_func'] == 'rmsprop':
return rmsprop(lr=lr)
elif opt_params['opt_func'] == 'powersign':
from tensorflow.contrib.opt.python.training import sign_decay as sd
d_steps = opt_params['decay_steps']
# Define the decay function (if specified)
if opt_params['decay_func'] == 'lin':
decay_func = sd.get_linear_decay_fn(d_steps)
elif opt_params['decay_func'] == 'cos':
decay_func = sd.get_consine_decay_fn(d_steps)
elif opt_params['decay_func'] == 'res':
decay_func = sd.get_restart_decay_fn(
d_steps, num_periods=opt_params['decay_periods'])
elif opt_params['decay_func'] is None:
decay_func = None
else:
raise ValueError('decay function not specified correctly')
# Use decay function in TF optimizer
return TFOptimizer(
PowerSignOptimizer(learning_rate=lr, sign_decay_fn=decay_func))
else:
raise ValueError('Optimizer specification not understood')
def create_model(self):
(iw, emb_in, vv_iw) = self.create_emb_layer()
self.emb_layer = emb_in
ow = Input(shape=(1, ), dtype='int32', name="outputword")
nws = []
for i in range(self.negative_sample_num):
nws.append(
Input(shape=(1, ), dtype='int32', name="neg_sam" + str(i)))
inputs = iw
inputs.append(ow)
inputs.extend(nws)
emb_ov = embeddings.Embedding(output_dim=self.vector_size,
input_dim=self.voc_size,
init="uniform")
vv_ov = emb_ov(ow)
vv_nss = []
for i in range(self.negative_sample_num):
vv_nss.append(emb_ov(nws[i]))
cos_pos = merge([vv_iw, vv_ov], mode='dot', dot_axes=(2, 2))
cos_negs = []
for i in range(self.negative_sample_num):
cos_neg = merge([vv_iw, vv_nss[i]], mode='dot', dot_axes=(2, 2))
cos_negs.append(cos_neg)
ress = []
ress.append(Lambda(lambda x: K.sigmoid(x))(cos_pos))
for i in range(self.negative_sample_num):
ress.append(Lambda(lambda x: K.sigmoid(-x))(cos_negs[i]))
self.keras_model = Model(input=inputs, output=ress)
self.keras_model.compile(loss='binary_crossentropy',
optimizer=TFOptimizer(
tf.train.GradientDescentOptimizer(
self.learning_rate)))
def build(self, vector_dim, vocab_size, learn_rate):
""" returns a word2vec model """
logging.info("Building keras model")
stddev = 1.0 / vector_dim
logging.info("Setting initializer standard deviation to: %.4f", stddev)
initializer = RandomNormal(mean=0.0, stddev=stddev, seed=None)
word_input = Input(shape=(1,), name="word_input")
word = Embedding(input_dim=vocab_size, output_dim=vector_dim, input_length=1,
name="word_embedding", embeddings_initializer=initializer)(word_input)
context_input = Input(shape=(1,), name="context_input")
context = Embedding(input_dim=vocab_size, output_dim=vector_dim, input_length=1,
name="context_embedding", embeddings_initializer=initializer)(context_input)
merged = dot([word, context], axes=2, normalize=False, name="dot")
merged = Flatten()(merged)
output = Dense(1, activation='sigmoid', name="output")(merged)
optimizer = TFOptimizer(tf.train.AdagradOptimizer(learn_rate))
model = Model(inputs=[word_input, context_input], outputs=output)
model.compile(loss="binary_crossentropy", optimizer=optimizer)
self.model = model
def isensee2017_model_dil3(input_shape=(4, 128, 128, 128),
n_base_filters=16,
depth=5,
dropout_rate=0.3,
n_segmentation_levels=3,
n_labels=4,
optimizer="Adam",
initial_learning_rate=5e-4,
loss_function=weighted_dice_coefficient_loss,
activation_name="sigmoid"):
"""
This function builds a model proposed by Isensee et al. for the BRATS 2017 competition:
https://www.cbica.upenn.edu/sbia/Spyridon.Bakas/MICCAI_BraTS/MICCAI_BraTS_2017_proceedings_shortPapers.pdf
This network is highly similar to the model proposed by Kayalibay et al. "CNN-based Segmentation of Medical
Imaging Data", 2017: https://arxiv.org/pdf/1701.03056.pdf
:param input_shape:
:param n_base_filters:
:param depth:
:param dropout_rate:
:param n_segmentation_levels:
:param n_labels:
:param optimizer:
:param initial_learning_rate:
:param loss_function:
:param activation_name:
:return:
"""
inputs = Input(input_shape)
current_layer = inputs
level_output_layers = list()
level_filters = list()
for level_number in range(depth):
n_level_filters = (2**level_number) * n_base_filters
level_filters.append(n_level_filters)
if current_layer is inputs:
in_conv = create_convolution_block(current_layer, n_level_filters)
else:
in_conv = create_convolution_block(current_layer,
n_level_filters,
strides=(2, 2, 2),
dilation_rate=(1, 1, 1))
context_output_layer = create_context_module(in_conv,
n_level_filters,
dropout_rate=dropout_rate)
summation_layer = Add()([in_conv, context_output_layer])
level_output_layers.append(summation_layer)
current_layer = summation_layer
segmentation_layers = list()
for level_number in range(depth - 2, -1, -1):
up_sampling = create_up_sampling_module(current_layer,
level_filters[level_number])
concatenation_layer = concatenate(
[level_output_layers[level_number], up_sampling], axis=1)
localization_output = create_localization_module(
concatenation_layer, level_filters[level_number])
current_layer = localization_output
if level_number < n_segmentation_levels:
segmentation_layers.insert(
0,
create_convolution_block(current_layer,
n_filters=n_labels,
kernel=(1, 1, 1)))
output_layer = None
for level_number in reversed(range(n_segmentation_levels)):
segmentation_layer = segmentation_layers[level_number]
if output_layer is None:
output_layer = segmentation_layer
else:
output_layer = Add()([output_layer, segmentation_layer])
if level_number > 0:
output_layer = UpSampling3D(size=(2, 2, 2))(output_layer)
activation_block = Activation(activation_name)(output_layer)
model = Model(inputs=inputs, outputs=activation_block)
if optimizer == "YellowFin":
# import and instantiate yellowFin optimizer
from .yellowfinkeras.yellowfin import YFOptimizer
from keras.optimizers import TFOptimizer
# define your optimizer
optimizer = TFOptimizer(YFOptimizer())
model.compile(optimizer=optimizer, loss=loss_function)
elif optimizer == "Adam":
from keras.optimizers import Adam
optimizer = Adam
model.compile(optimizer=optimizer(lr=initial_learning_rate),
loss=loss_function)
else:
raise ValueError(
str(error) + "\n\nYou can use only Adam or YellowFin optimizer\n")
return model
from ensemble.metrics import map
from ensemble.np_functions import jrer_fusion as jrer_fusion_np
from ensemble.np_functions import avg_fusion as avg_fusion_np
from ensemble.preprocessing import randomizer
from ensemble.tf_functions import jrer_fusion, avg_fusion
from ensemble.optimizers import YFOptimizer
from keras.optimizers import TFOptimizer
def freeze(model):
for layer in model.layers:
layer.trainable = False
# define your optimizer
opt = TFOptimizer(YFOptimizer())
dataset = Dataset(multilabel=True)
# replace 5000 with dataset.labels.shape[0] to use all the data (slow and mem intensive).
sample_number = dataset.labels.shape[0]
# random indexer to retrieve 30% of the test data for validation
pick = randomizer(sample_number // 2, validation_split=1.0)
# loading the train datasets.
_,\
y=next(dataset.generator(source='cnn', samples=sample_number, load_window=1, train=1))
cnn_x,\
_=np.load("inputs/%s_output_dnn_.npy"%'cnn'),y
def sampling_hypersurface(train_x,
train_y,
test_x,
test_y,
file_output_path,
suffix='',
batch_size=256,
iterations=1000,
verbose=1):
"""
Sampling hypersurface by different random initialization.
Parameters
----------
train_x : np.array(float)
Training `x` set (training features).
train_y : np.array(int)
Training `y` set (training labels).
test_x : np.array(float)
Test `x` set (test features).
test_y : np.array(int)
Test `y` set (test labels).
file_output_path : str
Path to store h5 files generated by custom BestAccs callback.
suffix: str, optional
File name suffix.
batch_size : int, optional
Size of batch (amount of samples) for model fitting.
iterations : int, optional
Defines how many times defining, compilation and fitting
procedure has to be executed.
verbose : int, optional
Verbosity level: 0 or 1. 0 means quite run, 1 means progress
will be shown.
"""
if not os.path.exists(file_output_path):
os.makedirs(file_output_path)
# Define form of h5 file name prefix
metrics = '{:.3f}-{acc:.3f}-{val_acc:.3f}'
initializer = RandomUniform(minval=-1.0, maxval=1.0, seed=None)
lr = 0.1
optimizer = TFOptimizer(tf.train.GradientDescentOptimizer(lr))
for iteration in range(iterations):
filename = metrics + '-iteration-' + str(iteration) + suffix + '.h5'
path_and_filename = os.path.join(file_output_path, '')
path_and_filename += filename # workaround of os.path.join issue
best_accs = BestAccs(path_and_filename, min_accs=0.88, diff=0.04)
model = Sequential()
model.add(
Dense(1,
kernel_initializer=initializer,
bias_initializer=initializer,
input_dim=train_x.shape[1],
activation='sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
model.fit(train_x,
train_y,
epochs=1000,
callbacks=[best_accs],
validation_data=(test_x, test_y),
batch_size=batch_size,
verbose=0)
if verbose == 1:
if (iteration + 1) % 100 == 0:
end = '\n'
else:
end = ''
if (iteration + 1) % 10 == 0:
print('|', end=end)
else:
print('.', end=end)
def __init__(self, optimizer, gdev_list=None):
TFOptimizer.__init__(self, optimizer)
self._gdev_list = gdev_list
def course_assignment(train_x,
train_y,
test_x,
test_y,
file_output_path,
initializer='zeros',
batch_size=256):
"""
First Coursera Deep Learning course programming assignment in Keras.
Training of single neuron with sigmoid activation function and set
of images. Our goal is to teach that neuron to recognize images with cat.
Parameters
----------
train_x : np.array(float)
Training `x` set (training features).
train_y : np.array(int)
Training `y` set (training labels).
test_x : np.array(float)
Test `x` set (test features).
test_y : np.array(int)
Test `y` set (test labels).
file_output_path : str
Path to store h5 files generated by custom BestAccs callback.
initializer : str, optional
Type of kernel and bias initializer.
batch_size : int, optional
Size of batch (amount of samples) for model fitting.
Returns
-------
history : keras.callbacks.History object
History of loss, accuracy, validation loss and validation
accuracy during model fitting.
"""
if not os.path.exists(file_output_path):
os.makedirs(file_output_path)
# Define form of h5 file name prefix
metrics = '{:.2f}-{acc:.2f}-{val_acc:.2f}'
filename = metrics + '-' + initializer + '.h5'
path_and_filename = os.path.join(file_output_path, '')
path_and_filename += filename # workaround of os.path.join issue
best_accs = BestAccs(path_and_filename, min_accs=0.7, diff=0.02)
lr = 0.005
optimizer = TFOptimizer(tf.train.GradientDescentOptimizer(lr))
model = Sequential()
model.add(
Dense(1,
kernel_initializer=initializer,
bias_initializer=initializer,
input_dim=train_x.shape[1],
activation='sigmoid'))
model.compile(loss='binary_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
history = model.fit(train_x,
train_y,
epochs=2000,
callbacks=[best_accs],
validation_data=(test_x, test_y),
batch_size=batch_size,
verbose=0)
return history
print("Tokenizing reviews...")
train_X = preprocess_reviews(train_X, tokenizer)
test_X = preprocess_reviews(test_X, tokenizer)
embedding_matrix = create_embedding_matrix(tokenizer, all_X, load=True)
# vectorize labels
train_Y = np.delete(to_categorical(train_Y), 0, axis=1)
test_Y = np.delete(to_categorical(test_Y), 0, axis=1)
model = models.Sequential()
model.add(layers.embeddings.Embedding(NUM_WORDS, VECTOR_DIM, input_length=MAX_LENGTH, weights=[embedding_matrix], trainable=True))
model.add(layers.LSTM(32))
model.add(layers.Dense(5, activation='softmax'))
model.summary()
model.compile(optimizer=TFOptimizer(tf.optimizers.Adam()),
loss='categorical_crossentropy',
metrics=['accuracy'])
history = model.fit(train_X,
train_Y,
epochs=0,
batch_size=32,
validation_split=0.1)
model.save('amazon.h5')
pred_Y = model.predict_classes(test_X)
val_Y = [np.argmax(x) for x in test_Y]
confusion_matrix = tf.math.confusion_matrix(labels=val_Y, predictions=pred_Y).numpy()
def train_text_model(
corpus: Corpus,
featurizer: Featurizer,
model_options: ModelOptions,
models_ann_dir=None,
debug=False,
tensorboard_dir=None,
):
"""
Utility function for training citeomatic models.
"""
# load pretrained embeddings
if model_options.use_pretrained:
dp = DatasetPaths()
pretrained_embeddings_file = dp.embeddings_weights_for_corpus('shared')
with h5py.File(pretrained_embeddings_file, 'r') as f:
pretrained_embeddings = f['embedding'][...]
else:
pretrained_embeddings = None
create_model = import_from(
'citeomatic.models.%s' % model_options.model_name, 'create_model')
models = create_model(model_options, pretrained_embeddings)
model, embedding_model = models['citeomatic'], models['embedding']
logging.info(model.summary())
if model_options.train_for_test_set:
paper_ids_for_training = corpus.train_ids + corpus.valid_ids
candidates_for_training = corpus.train_ids + corpus.valid_ids + corpus.test_ids
else:
paper_ids_for_training = corpus.train_ids
candidates_for_training = corpus.train_ids + corpus.valid_ids
training_dg = DataGenerator(
corpus=corpus,
featurizer=featurizer,
margin_multiplier=model_options.margin_multiplier,
use_variable_margin=model_options.use_variable_margin)
training_generator = training_dg.triplet_generator(
paper_ids=paper_ids_for_training,
candidate_ids=candidates_for_training,
batch_size=model_options.batch_size,
neg_to_pos_ratio=model_options.neg_to_pos_ratio)
validation_dg = DataGenerator(
corpus=corpus,
featurizer=featurizer,
margin_multiplier=model_options.margin_multiplier,
use_variable_margin=model_options.use_variable_margin)
validation_generator = validation_dg.triplet_generator(
paper_ids=corpus.valid_ids,
candidate_ids=corpus.train_ids + corpus.valid_ids,
batch_size=1024,
neg_to_pos_ratio=model_options.neg_to_pos_ratio)
if model_options.optimizer == 'tfopt':
optimizer = TFOptimizer(
tf.contrib.opt.LazyAdamOptimizer(learning_rate=model_options.lr))
else:
optimizer = import_from('keras.optimizers',
model_options.optimizer)(lr=model_options.lr)
model.compile(optimizer=optimizer, loss=layers.triplet_loss)
# training calculation
model_options.samples_per_epoch = int(
np.minimum(model_options.samples_per_epoch,
model_options.total_samples))
epochs = int(
np.ceil(model_options.total_samples / model_options.samples_per_epoch))
steps_per_epoch = int(model_options.samples_per_epoch /
model_options.batch_size)
# callbacks
callbacks_list = []
if debug:
callbacks_list.append(MemoryUsageCallback())
if model_options.tb_dir is not None:
callbacks_list.append(
TensorBoard(log_dir=model_options.tb_dir,
histogram_freq=1,
write_graph=True))
if model_options.reduce_lr_flag:
if model_options.optimizer != 'tfopt':
callbacks_list.append(
ReduceLROnPlateau(verbose=1,
patience=2,
epsilon=0.01,
min_lr=1e-6,
factor=0.5))
if models_ann_dir is None:
ann_featurizer = featurizer
paper_embedding_model = embedding_model
embed_at_epoch_end = True
embed_at_train_begin = False
else:
ann_featurizer, ann_models = model_from_directory(models_ann_dir,
on_cpu=True)
paper_embedding_model = ann_models['embedding']
paper_embedding_model._make_predict_function()
embed_at_epoch_end = False
embed_at_train_begin = True
callbacks_list.append(
UpdateANN(corpus, ann_featurizer, paper_embedding_model, training_dg,
validation_dg, embed_at_epoch_end, embed_at_train_begin))
if model_options.tb_dir is None:
validation_data = validation_generator
else:
validation_data = next(validation_generator)
# logic
model.fit_generator(generator=training_generator,
steps_per_epoch=steps_per_epoch,
epochs=epochs,
callbacks=callbacks_list,
validation_data=validation_generator,
validation_steps=10)
return model, embedding_model
for identity_id in range(len(identities) - 1):
welcome_message += '%s, ' % identities[identity_id]
welcome_message += 'and %s, ' % identities[-1]
welcome_message += 'have a nice day!'
windows10_voice_interface.Speak(welcome_message)
# Allow the program to start detecting identities again
ready_to_detect_identity = True
if __name__ == "__main__":
windows10_voice_interface = wincl.Dispatch("SAPI.SpVoice")
FRmodel = faceRecoModel(input_shape=(3, 96, 96))
FRmodel.compile(optimizer=TFOptimizer(
tf.train.GradientDescentOptimizer(0.1)),
loss=triplet_loss,
metrics=['accuracy'])
load_weights_from_FaceNet(FRmodel)
database = prepare_database()
webcam_face_recognizer(database)
# ### References:
#
# - Florian Schroff, Dmitry Kalenichenko, James Philbin (2015). [FaceNet: A Unified Embedding for Face Recognition and Clustering](https://arxiv.org/pdf/1503.03832.pdf)
# - Yaniv Taigman, Ming Yang, Marc'Aurelio Ranzato, Lior Wolf (2014). [DeepFace: Closing the gap to human-level performance in face verification](https://research.fb.com/wp-content/uploads/2016/11/deepface-closing-the-gap-to-human-level-performance-in-face-verification.pdf)
# - The pretrained model we use is inspired by Victor Sy Wang's implementation and was loaded using his code: https://github.com/iwantooxxoox/Keras-OpenFace.
# - Our implementation also took a lot of inspiration from the official FaceNet github repository: https://github.com/davidsandberg/facenet
#
target_size=(224, 224),
color_mode='grayscale',
classes = class_names,
class_mode='categorical',
batch_size=256,
shuffle=True)
with K.tf.device('/cpu:0'):
kernel_model = MobileNet(
input_shape=(224, 224, 1),
alpha=1.0,
include_top=True,
weights=None,
classes=len(class_names))
parallel_model = multi_gpu_model(kernel_model, gpus=8)
optimizer = TFOptimizer(tf.train.RMSPropOptimizer(learning_rate=LEARNING_RATE,
decay=DECAY,
momentum=0.9)) # Tensorflow RMSProp
parallel_model.compile(
optimizer=optimizer,
loss='categorical_crossentropy', # for multiclass classification
metrics=['accuracy'])
parallel_model.fit_generator(
train_generator,
steps_per_epoch=25000,
epochs=10,
verbose=1,
max_queue_size=100,
use_multiprocessing=True,
workers=cpu_count(),
callbacks=[
ModelCheckpoint(
def main(argv=None):
'''
'''
main.__doc__ = __doc__
argv = sys.argv if argv is None else sys.argv.extend(argv)
desc = main.__doc__ # .format(os.path.basename(__file__))
# CLI parser
args = parser_(desc)
nranks_per_gpu = args.nranks_per_gpu
local_rank = hvd.local_rank()
gpu_local_rank = local_rank // nranks_per_gpu
print('local_rank, GPU_LOCAL_RANK: {}, {}'.format(local_rank,
gpu_local_rank))
# Pin GPU to be used to process local rank (one GPU per process)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
# config.gpu_options.visible_device_list = str(hvd.local_rank())
config.gpu_options.visible_device_list = str(gpu_local_rank)
K.set_session(tf.Session(config=config))
# input image dimensions
img_rows, img_cols, img_chns = 28, 28, 1
# number of convolutional filters to use
filters = 64
# convolution kernel size
num_conv = 3
hvdsize = hvd.size()
batch_size = 128 # 100
if K.image_data_format() == 'channels_first':
original_img_size = (img_chns, img_rows, img_cols)
else:
original_img_size = (img_rows, img_cols, img_chns)
latent_dim = 2
intermediate_dim = 128
epsilon_std = 1.0
epochs = args.epochs # 5
# train the VAE on MNIST digits
(x_train, _), (x_test, y_test) = mnist.load_data()
x_train = x_train.astype('float32') / 255.
x_train = x_train.reshape((x_train.shape[0], ) + original_img_size)
x_test = x_test.astype('float32') / 255.
x_test = x_test.reshape((x_test.shape[0], ) + original_img_size)
if hvd.rank() == 0:
print('x_train.shape:', x_train.shape)
train_samples = x_train.shape[0]
# steps_per_epoch = train_samples // batch_size // hvdsize
speedupopt = args.speedup
if speedupopt == SpeedupOpts.imgspersec:
steps_per_epoch = train_samples // batch_size
else:
steps_per_epoch = int(
round(float(train_samples) / batch_size / hvdsize + 0.5))
# Create the dataset and its associated one-shot iterator.
buffer_size = 10000
dataset = Dataset.from_tensor_slices(x_train)
dataset = dataset.repeat()
dataset = dataset.shuffle(buffer_size)
dataset = dataset.batch(batch_size)
iterator = dataset.make_one_shot_iterator()
x_train_batch = iterator.get_next()
ldict = make_shared_layers_dict(img_chns, img_rows, img_cols, batch_size,
filters, num_conv, intermediate_dim,
latent_dim, epsilon_std)
# ldict is a dictionary that holds all layers. Since these layers are
# instantiated once, they are shared amongs vae, encoder, and generator.
x = Input(tensor=x_train_batch)
vae = make_vae(ldict, x)
# : :type vae: Model
lr = 0.001 # * hvdsize
opt = tf.train.RMSPropOptimizer(lr)
# Add Horovod Distributed Optimizer.
opt = hvd.DistributedOptimizer(opt) # , use_locking=True)
opt = TFOptimizer(opt)
# opt = RMSprop(lr)
# Add Horovod Distributed Optimizer.
# opt = hvd_keras.DistributedOptimizer(opt) # , use_locking=True)
vae.compile(optimizer=opt, loss=None)
if hvd.rank() == 0:
vae.summary()
callbacks = []
if hvd.rank() == 0:
callbacks += [BatchTiming(), SamplesPerSec(batch_size * hvdsize)]
sess = K.get_session()
sess.run(hvd.broadcast_global_variables(0))
# Fit the model using data from the TF data tensors.
vae.fit(steps_per_epoch=steps_per_epoch,
epochs=epochs,
callbacks=callbacks)
if hvd.rank() == 0:
x = Input(shape=original_img_size)
vae_val = make_vae(ldict, x)
vae_val.compile(optimizer=opt, loss=None)
loss = vae_val.evaluate(x=x_test, y=None, batch_size=batch_size)
print('\n\nVAE VALIDATION LOSS: {}'.format(loss))
x = Input(shape=original_img_size)
z_mean, _ = get_encoded(ldict, x)
encoder = Model(x, z_mean)
# : :type encoder: Model
decoder_input = Input(shape=(latent_dim, ))
x_decoded_mean_squash = get_decoded(ldict, decoder_input)
generator = Model(decoder_input, x_decoded_mean_squash)
# : :type generator: Model
# display a 2D plot of the digit classes in the latent space
x_test_encoded = encoder.predict(x_test, batch_size=batch_size)
plt.figure(figsize=(6, 6))
plt.scatter(x_test_encoded[:, 0], x_test_encoded[:, 1], c=y_test)
plt.colorbar()
# plt.show()
plt.savefig('vae_scatter.ps')
plt.close()
# display a 2D manifold of the digits
n = 15 # figure with 15x15 digits
digit_size = 28
figure = np.zeros((digit_size * n, digit_size * n))
# Linearly spaced coordinates on the unit square were transformed
# through the inverse CDF (ppf) of the Gaussian
# To produce values of the latent variables z, since the prior of the
# latent space is Gaussian
grid_x = norm.ppf(np.linspace(0.05, 0.95, n))
grid_y = norm.ppf(np.linspace(0.05, 0.95, n))
for i, yi in enumerate(grid_x):
for j, xi in enumerate(grid_y):
z_sample = np.array([[xi, yi]])
z_sample = np.tile(z_sample, batch_size).reshape(batch_size, 2)
x_decoded = generator.predict(z_sample, batch_size=batch_size)
digit = x_decoded[0].reshape(digit_size, digit_size)
figure[i * digit_size:(i + 1) * digit_size,
j * digit_size:(j + 1) * digit_size] = digit
plt.figure(figsize=(10, 10))
plt.imshow(figure, cmap='Greys_r')
# plt.show()
plt.savefig('vae_digit.ps')
plt.close()
K.clear_session()
def main(argv=None):
'''
'''
main.__doc__ = __doc__
argv = sys.argv if argv is None else sys.argv.extend(argv)
desc = main.__doc__ # .format(os.path.basename(__file__))
# CLI parser
args = parser_(desc)
nranks_per_gpu = args.nranks_per_gpu
local_rank = hvd.local_rank()
gpu_local_rank = local_rank // nranks_per_gpu
print('local_rank, GPU_LOCAL_RANK: {}, {}'.format(local_rank,
gpu_local_rank))
# Pin GPU to be used to process local rank (one GPU per process)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
# config.gpu_options.visible_device_list = str(hvd.local_rank())
config.gpu_options.visible_device_list = str(gpu_local_rank)
K.set_session(tf.Session(config=config))
# input image dimensions
img_rows, img_cols, img_chns = 28, 28, 1
# number of convolutional filters to use
filters = 64
# convolution kernel size
num_conv = 3
hvdsize = hvd.size()
batch_size = 128 # 100
if K.image_data_format() == 'channels_first':
original_img_size = (img_chns, img_rows, img_cols)
else:
original_img_size = (img_rows, img_cols, img_chns)
latent_dim = 2
intermediate_dim = 128
epsilon_std = 1.0
epochs = args.epochs # 5
# train the VAE on MNIST digits
(x_train, _), (x_test, y_test) = mnist.load_data()
# Data split if going for reduction in each iteration step. Using
# tf-queue or dataset is better to preserve uniform random sampling.
# nsamples = x_train.shape[0]
# mysamples = nsamples // hvdsize
# start_sam = hvd.local_rank() * mysamples
# stop_sam = min((hvd.local_rank() + 1) * mysamples, nsamples)
# x_train = x_train[start_sam:stop_sam, ...]
x_train = x_train.astype('float32') / 255.
x_train = x_train.reshape((x_train.shape[0], ) + original_img_size)
x_test = x_test.astype('float32') / 255.
x_test = x_test.reshape((x_test.shape[0], ) + original_img_size)
if hvd.rank() == 0:
print('x_train.shape:', x_train.shape)
vae, encoder, generator = make_vae_and_codec(original_img_size, img_chns,
img_rows, img_cols,
batch_size, filters, num_conv,
intermediate_dim, latent_dim,
epsilon_std)
# : :type vae: Model
lr = 0.001 # * hvdsize
opt = tf.train.RMSPropOptimizer(lr)
# Add Horovod Distributed Optimizer.
opt = hvd.DistributedOptimizer(opt) # , use_locking=True)
opt = TFOptimizer(opt)
vae.compile(optimizer=opt, loss=None)
if hvd.rank() == 0:
vae.summary()
callbacks = []
if hvd.rank() == 0:
callbacks += [BatchTiming(), SamplesPerSec(batch_size * hvdsize)]
sess = K.get_session()
sess.run(hvd.broadcast_global_variables(0))
vae.fit(x_train,
shuffle=True,
epochs=epochs,
batch_size=batch_size,
validation_data=(x_test, None),
callbacks=callbacks)
if hvd.rank() == 0:
vae_val = vae
loss = vae_val.evaluate(x=x_test, y=None, batch_size=batch_size)
print('\n\nVAE VALIDATION LOSS: {}'.format(loss))
# display a 2D plot of the digit classes in the latent space
x_test_encoded = encoder.predict(x_test, batch_size=batch_size)
plt.figure(figsize=(6, 6))
plt.scatter(x_test_encoded[:, 0], x_test_encoded[:, 1], c=y_test)
plt.colorbar()
# plt.show()
plt.savefig('vae_scatter.ps')
plt.close()
# display a 2D manifold of the digits
n = 15 # figure with 15x15 digits
digit_size = 28
figure = np.zeros((digit_size * n, digit_size * n))
# Linearly spaced coordinates on the unit square were transformed
# through the inverse CDF (ppf) of the Gaussian
# To produce values of the latent variables z, since the prior of the
# latent space is Gaussian
grid_x = norm.ppf(np.linspace(0.05, 0.95, n))
grid_y = norm.ppf(np.linspace(0.05, 0.95, n))
for i, yi in enumerate(grid_x):
for j, xi in enumerate(grid_y):
z_sample = np.array([[xi, yi]])
z_sample = np.tile(z_sample, batch_size).reshape(batch_size, 2)
x_decoded = generator.predict(z_sample, batch_size=batch_size)
digit = x_decoded[0].reshape(digit_size, digit_size)
figure[i * digit_size:(i + 1) * digit_size,
j * digit_size:(j + 1) * digit_size] = digit
plt.figure(figsize=(10, 10))
plt.imshow(figure, cmap='Greys_r')
# plt.show()
plt.savefig('vae_digit.ps')
plt.close()
K.clear_session()
activation = 'selu'
layer_size = 128
n_layers = 2
drop_rate = 0.01
for n in range(n_layers):
x = Dense(layer_size)(x)
x = Activation(activation)(x)
x = AlphaDropout(rate=drop_rate)(x)
# main output for loss calculation #2
main_pred = Dense(1, name='main_out')(x)
# define inputs / outputs
model = Model(inputs=[main_input, aux_input], outputs=[main_pred, gru_pred])
# weight losses and compile model
model.compile(optimizer=TFOptimizer(YFOptimizer()),
loss='mean_squared_error',
loss_weights=[.5, .5])
model.fit([X1, X2], [y, y],
epochs=50,
validation_split=0.5,
batch_size=256,
callbacks=[EarlyStopping(patience=8)])
# clear graph
K.clear_session()
