def optimizer_class(): from keras.optimizers import SGD, Adam, RMSprop, Nadam, TFOptimizer # TODO(KGF): check this fix; lr, tf were undefined in neglected # serial runner.py, since mpi_runner.py has been the main tool import tensorflow as tf if conf['model']['optimizer'] == 'sgd': return SGD(lr=conf['model']['lr'], clipnorm=conf['model']['clipnorm']) elif conf['model']['optimizer'] == 'momentum_sgd': return SGD(lr=conf['model']['lr'], clipnorm=conf['model']['clipnorm'], decay=1e-6, momentum=0.9) elif conf['model']['optimizer'] == 'tf_momentum_sgd': return TFOptimizer( tf.train.MomentumOptimizer(learning_rate=conf['model']['lr'], momentum=0.9)) elif conf['model']['optimizer'] == 'adam': return Adam(lr=conf['model']['lr'], clipnorm=conf['model']['clipnorm']) elif conf['model']['optimizer'] == 'tf_adam': return TFOptimizer( tf.train.AdamOptimizer(learning_rate=conf['model']['lr'])) elif conf['model']['optimizer'] == 'rmsprop': return RMSprop(lr=conf['model']['lr'], clipnorm=conf['model']['clipnorm']) elif conf['model']['optimizer'] == 'nadam': return Nadam(lr=conf['model']['lr'], clipnorm=conf['model']['clipnorm']) else: print("Optimizer not implemented yet") exit(1)
def compile(self, optimizer, clipnorm, loss='mse'): # TODO(KGF): check the following import taken from runner.py # Was not in this file, originally. from keras.optimizers import SGD, Adam, RMSprop, Nadam, TFOptimizer if optimizer == 'sgd': optimizer_class = SGD(lr=self.DUMMY_LR, clipnorm=clipnorm) elif optimizer == 'momentum_sgd': optimizer_class = SGD(lr=self.DUMMY_LR, clipnorm=clipnorm, decay=1e-6, momentum=0.9) elif optimizer == 'tf_momentum_sgd': optimizer_class = TFOptimizer( tf.train.MomentumOptimizer(learning_rate=self.DUMMY_LR, momentum=0.9)) elif optimizer == 'adam': optimizer_class = Adam(lr=self.DUMMY_LR, clipnorm=clipnorm) elif optimizer == 'tf_adam': optimizer_class = TFOptimizer( tf.train.AdamOptimizer(learning_rate=self.DUMMY_LR)) elif optimizer == 'rmsprop': optimizer_class = RMSprop(lr=self.DUMMY_LR, clipnorm=clipnorm) elif optimizer == 'nadam': optimizer_class = Nadam(lr=self.DUMMY_LR, clipnorm=clipnorm) else: print("Optimizer not implemented yet") exit(1) self.model.compile(optimizer=optimizer_class, loss=loss) self.ensure_equal_weights()
def optimizer_class(): from keras.optimizers import SGD, Adam, RMSprop, Nadam, TFOptimizer if conf['model']['optimizer'] == 'sgd': return SGD(lr=conf['model']['lr'], clipnorm=conf['model']['clipnorm']) elif conf['model']['optimizer'] == 'momentum_sgd': return SGD(lr=conf['model']['lr'], clipnorm=conf['model']['clipnorm'], decay=1e-6, momentum=0.9) elif conf['model']['optimizer'] == 'tf_momentum_sgd': return TFOptimizer( tf.train.MomentumOptimizer(learning_rate=conf['model']['lr'], momentum=0.9)) elif conf['model']['optimizer'] == 'adam': return Adam(lr=conf['model']['lr'], clipnorm=conf['model']['clipnorm']) elif conf['model']['optimizer'] == 'tf_adam': return TFOptimizer( tf.train.AdamOptimizer(learning_rate=conf['model']['lr'])) elif conf['model']['optimizer'] == 'rmsprop': return RMSprop(lr=conf['model']['lr'], clipnorm=conf['model']['clipnorm']) elif conf['model']['optimizer'] == 'nadam': return Nadam(lr=conf['model']['lr'], clipnorm=conf['model']['clipnorm']) else: print("Optimizer not implemented yet") exit(1)
def train_model(self): """Train a model according to options using generator. This saves results after each epoch and attempts to resume training from the last saved point. # Arguments run_path: the path to the files for a run model: a Keras model options: RunOptions object that specifies the run generator: a Generator object to generate the training and validation data """ print(self.model.summary()) train_gen = self.generator.make_split_generator( TRAIN, target_size=self.options.target_size, batch_size=self.options.batch_size, shuffle=True, augment_methods=self.options.augment_methods, normalize=True, only_xy=True) validation_gen = self.generator.make_split_generator( VALIDATION, target_size=self.options.target_size, batch_size=self.options.batch_size, shuffle=True, augment_methods=self.options.augment_methods, normalize=True, only_xy=True) if self.options.optimizer == ADAM: optimizer = Adam(lr=self.options.init_lr, decay=self.options.lr_step_decay) elif self.options.optimizer == RMS_PROP: optimizer = RMSprop(lr=self.options.init_lr, rho=self.options.rho, epsilon=self.options.epsilon, decay=self.options.lr_step_decay) elif self.options.optimizer == 'sgd': optimizer = SGD( lr=self.options.init_lr, momentum=self.options.momentum, nesterov=self.options.nesterov ) elif self.options.optimizer == YELLOWFIN: optimizer = TFOptimizer(YFOptimizer( learning_rate=self.options.init_lr, momentum=self.options.momentum)) self.model.compile( optimizer, self.loss_function, metrics=self.metrics) initial_epoch = self.get_initial_epoch() callbacks = self.make_callbacks() self.model.fit_generator( train_gen, initial_epoch=initial_epoch, steps_per_epoch=self.options.steps_per_epoch, epochs=self.options.epochs, validation_data=validation_gen, validation_steps=self.options.validation_steps, callbacks=callbacks)
def parse_optimizer(string_input): # motivation for tf optimizers https://github.com/fchollet/keras/issues/1021#issuecomment-270566219 if string_input.startswith("tf."): from keras.optimizers import TFOptimizer import tensorflow as tf print("> using tensorflow optimizer") if string_input == "tf.sgd": return TFOptimizer( tf.train.GradientDescentOptimizer(learning_rate=0.1)) if string_input == "tf.rmsprop": # lr taken from keras standard impl return TFOptimizer(tf.train.RMSPropOptimizer(learning_rate=0.001)) else: return string_input
def get(identifier): """Retrieves a Keras Optimizer instance. # Arguments identifier: Optimizer identifier, one of - String: name of an optimizer - Dictionary: configuration dictionary. - Keras Optimizer instance (it will be returned unchanged). - TensorFlow Optimizer instance (it will be wrapped as a Keras Optimizer). # Returns A Keras Optimizer instance. # Raises ValueError: If `identifier` cannot be interpreted. """ if K.backend() == 'tensorflow': # Wrap TF optimizer instances if isinstance(identifier, tf.train.Optimizer): return TFOptimizer(identifier) if isinstance(identifier, dict): return deserialize(identifier) elif isinstance(identifier, six.string_types): config = {'class_name': str(identifier), 'config': {}} return deserialize(config) if isinstance(identifier, Optimizer): return identifier else: raise ValueError('Could not interpret optimizer identifier:', identifier)
def __init__(self, featurizer, opts): self.featurizer = featurizer self.dropout_p = opts.dropout_p self.l2_lambda = opts.l2_lambda self.l1_lambda = opts.l1_lambda self.n_features = opts.n_features self.dense_dims = opts.dense_dims self.epochs = opts.epochs self.steps_per_epoch = opts.steps_per_epoch embeddings = BaseWordEmbeddings(self.dense_dims, self.n_features, self.l1_lambda, self.l2_lambda, self.dropout_p) self.nn_rank_model = dict() self.title_embedding = embeddings self.abstract_embedding = embeddings self.title_embedding_multiplier = LambdaScalarMultiplier(name='title-scalar-multiplier') self.abstract_embedding_multiplier = LambdaScalarMultiplier(name='abstract-scalar-multiplier') self.model, self.embedding_model = self.__compile_dense_model() optimizer = TFOptimizer(tf.contrib.opt.LazyAdamOptimizer(learning_rate=opts.learning_rate)) self.model.compile(optimizer=optimizer, loss=triplet_loss, metrics=['accuracy']) self.callbacks = [ SaveModelWeights([('dense', self.model), ('embedding', self.embedding_model)], opts.weights_directory, opts.checkpoint_frequency) ]
def best_model_evaluate(train_x, train_y, test_x, test_y, file_output_path, file_name, batch_size=256): """ Evaluate accuracy on training and test set. Parameters ---------- train_x : np.array(float) Training `x` set (training features). train_y : np.array(int) Training `y` set (training labels). test_x : np.array(float) Test `x` set (test features). test_y : np.array(int) Test `y` set (test labels). file_output_path : str Path to store h5 files generated by custom BestAccs callback. file_name : str Name of file that contains weights batch_size : int, optional Size of batch (amount of samples) for model fitting. Returns ------- acc_train : float Accuracy on training set. acc_test : float Accuracy on test set. """ initializer = 'zeros' lr = 0.01 optimizer = TFOptimizer(tf.train.GradientDescentOptimizer(lr)) model = Sequential() model.add( Dense(1, kernel_initializer=initializer, bias_initializer=initializer, input_dim=train_x.shape[1], activation='sigmoid')) model.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy']) file_output_path = 'samplingHypersurface' model.load_weights(os.path.join(file_output_path, file_name)) metrics_train = model.evaluate(train_x, train_y, batch_size=256, verbose=0) acc_train = metrics_train[1] metrics_test = model.evaluate(test_x, test_y, batch_size=256, verbose=0) acc_test = metrics_test[1] return acc_train, acc_test
def get_optimizer(): ''' Get optimizer for learning the network @return: optimizing function ''' optimizer = TFOptimizer(tf.train.RMSPropOptimizer(learning_rate=LEARNING_RATE, decay=DECAY, momentum=0.9)) # Tensorflow RMSProp return optimizer
def best_learning_rate(train_x, train_y, test_x, test_y, lr, batch_size=256): """ Perform 10 random initializations of model, then compile and fit it. Model is initialized randomly (random_uniform), compiled and fitted. Operation is repeated 10 times. Then all histories are returned as a list. Parameters ---------- train_x : np.array(float) Training `x` set (training features). train_y : np.array(int) Training `y` set (training labels). test_x : np.array(float) Test `x` set (test features). test_y : np.array(int) Test `y` set (test labels). lr : float Learning rate. batch_size : int, optional Size of batch (amount of samples) for model fitting. Returns ------- history_set : list(keras.callbacks.History object) History of loss, accuracy, validation loss and validation accuracy during model fitting. """ optimizer = TFOptimizer(tf.train.GradientDescentOptimizer(lr)) history_set = [] for i in range(10): model = Sequential() initializer = RandomUniform(minval=-1.0, maxval=1.0, seed=None) model.add( Dense(1, kernel_initializer=initializer, bias_initializer=initializer, input_dim=train_x.shape[1], activation='sigmoid')) model.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy']) history = model.fit(train_x, train_y, epochs=1000, validation_data=(test_x, test_y), batch_size=batch_size, verbose=0) history_set.append(history) return history_set
def best_batch_size(train_x, train_y, epochs=200): """ Determine optimal batch size. Parameters ---------- train_x : np.array(float) Training `x` set (training features). train_y : np.array(int) Training `y` set (training labels). epochs : int, optional Number of epochs. Returns ------- batch_size : int Size of most efficient (fastest) batch size. batch_exe_time: list List of execution time for given range of batch size. """ lr = 0.005 optimizer = TFOptimizer(tf.train.GradientDescentOptimizer(lr)) batch_size_limit = int(np.log2(train_x.shape[0])) + 1 batch_size_set = [2**x for x in range(5, batch_size_limit + 1)] model = Sequential() model.add( Dense(1, kernel_initializer='zeros', bias_initializer='zeros', input_dim=train_x.shape[1], activation='sigmoid')) model.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy']) # Measeure of execution time for each batch_size batches_exe_time = [] for batch_size in batch_size_set: time_start = time() model.fit(train_x, train_y, epochs=epochs, batch_size=batch_size, verbose=0) time_end = time() - time_start batches_exe_time.append([time_end, batch_size]) batches_exe_time.sort() batch_size = batches_exe_time[0][1] return batch_size, batches_exe_time
def __init__(self, data_shape, choke_shape, filename=None): self.data_shape = data_shape self.choke_shape = choke_shape self.__define() self.load_file(self.model_ae, filename) optimizer = TFOptimizer(tf.train.AdadeltaOptimizer()) self.model_encoder.compile(optimizer=optimizer, loss=self.rmsle, metrics=[self.rmsle]) self.model_decoder.compile(optimizer=optimizer, loss=self.rmsle, metrics=[self.rmsle]) self.model_ae.compile(optimizer=optimizer, loss=self.rmsle, metrics=[self.rmsle])
def get_tf_model(self, ground_truth, method = config.model_type): global_step = tf.Variable(0, trainable=False) learn_rate = tf.train.cosine_decay_restarts(learning_rate=0.001, global_step=global_step, first_decay_steps=100) crispr_model = getattr(self, "_{!s}__cas9_{!s}_model".format(self.__class__.__name__, method), self.__cas9_rnn_model)() loss = tf.losses.mean_squared_error(ground_truth, crispr_model.output) rmsprop_optimizer = tf.train.RMSPropOptimizer(learning_rate=learn_rate).minimize(loss, global_step=global_step) custimized_rmsprop = TFOptimizer(rmsprop_optimizer) crispr_model.compile(optimizer=custimized_rmsprop, loss='mse', metrics=[utils.revised_mse_loss, 'mse']) return crispr_model
def build_model(self, existing_embedding_weights, existing_context_weights): stddev = 1.0 / self.vector_dim initializer = RandomNormal(mean=0.0, stddev=stddev, seed=None) target_input = Input(shape=(1, ), name="target_input") target_layer = Embedding( input_dim=self.vocabulary_size, output_dim=self.vector_dim, input_length=1, name="target_layer", embeddings_initializer=initializer)(target_input) context_input = Input(shape=(1, ), name="context_input") context_layer = Embedding( input_dim=self.vocabulary_size, output_dim=self.vector_dim, input_length=1, name="context_layer", embeddings_initializer=initializer)(context_input) merged = dot([target_layer, context_layer], axes=2, normalize=False, name="dot") merged = Flatten()(merged) output = Dense(1, activation='sigmoid', name="output")(merged) optimizer = TFOptimizer(tf.train.AdagradOptimizer(0.1)) model = Model(inputs=[target_input, context_input], outputs=output) model.compile(loss="binary_crossentropy", optimizer=optimizer, metrics=[binary_accuracy]) embedding_weights = model.get_layer("target_layer").get_weights() context_weights = model.get_layer("context_layer").get_weights() for i in range(0, len(existing_embedding_weights)): embedding_weights[0][i] = existing_embedding_weights[i] for i in range(0, len(existing_context_weights)): context_weights[0][i] = existing_context_weights[i] model.get_layer("target_layer").set_weights(embedding_weights) model.get_layer("context_layer").set_weights(context_weights) model.summary() self.model = model
def create_optim(self, **args): if self.optim == 'yellowfin': filtered_args = filter_args(YFOptimizer, args) print(filtered_args) opt = TFOptimizer(YFOptimizer(**filtered_args)) elif self.optim == 'ftml': filtered_args = filter_args(FTML, args) opt = FTML(**filtered_args) else: for name, optim_class in inspect.getmembers(optimizers): if inspect.isclass(optim_class) and name[0] == name[0].upper(): filtered_args = filter_args(optim_class, args) return optim_class(**filtered_args) return opt
def get_optimizer(opt_params, lr): """Helper to get optimizer from text params Parameters ---------- opt_params: dict Dictionary containing optimization function name and learning rate decay lr: float Initial learning rate Return ------ opt_function: Keras optimizer """ if opt_params['opt_func'] == 'sgd': return SGD(lr=lr, momentum=opt_params['momentum']) elif opt_params['opt_func'] == 'adam': return Adam(lr=lr) elif opt_params['opt_func'] == 'rmsprop': return rmsprop(lr=lr) elif opt_params['opt_func'] == 'powersign': from tensorflow.contrib.opt.python.training import sign_decay as sd d_steps = opt_params['decay_steps'] # Define the decay function (if specified) if opt_params['decay_func'] == 'lin': decay_func = sd.get_linear_decay_fn(d_steps) elif opt_params['decay_func'] == 'cos': decay_func = sd.get_consine_decay_fn(d_steps) elif opt_params['decay_func'] == 'res': decay_func = sd.get_restart_decay_fn( d_steps, num_periods=opt_params['decay_periods']) elif opt_params['decay_func'] is None: decay_func = None else: raise ValueError('decay function not specified correctly') # Use decay function in TF optimizer return TFOptimizer( PowerSignOptimizer(learning_rate=lr, sign_decay_fn=decay_func)) else: raise ValueError('Optimizer specification not understood')
def create_model(self): (iw, emb_in, vv_iw) = self.create_emb_layer() self.emb_layer = emb_in ow = Input(shape=(1, ), dtype='int32', name="outputword") nws = [] for i in range(self.negative_sample_num): nws.append( Input(shape=(1, ), dtype='int32', name="neg_sam" + str(i))) inputs = iw inputs.append(ow) inputs.extend(nws) emb_ov = embeddings.Embedding(output_dim=self.vector_size, input_dim=self.voc_size, init="uniform") vv_ov = emb_ov(ow) vv_nss = [] for i in range(self.negative_sample_num): vv_nss.append(emb_ov(nws[i])) cos_pos = merge([vv_iw, vv_ov], mode='dot', dot_axes=(2, 2)) cos_negs = [] for i in range(self.negative_sample_num): cos_neg = merge([vv_iw, vv_nss[i]], mode='dot', dot_axes=(2, 2)) cos_negs.append(cos_neg) ress = [] ress.append(Lambda(lambda x: K.sigmoid(x))(cos_pos)) for i in range(self.negative_sample_num): ress.append(Lambda(lambda x: K.sigmoid(-x))(cos_negs[i])) self.keras_model = Model(input=inputs, output=ress) self.keras_model.compile(loss='binary_crossentropy', optimizer=TFOptimizer( tf.train.GradientDescentOptimizer( self.learning_rate)))
def build(self, vector_dim, vocab_size, learn_rate): """ returns a word2vec model """ logging.info("Building keras model") stddev = 1.0 / vector_dim logging.info("Setting initializer standard deviation to: %.4f", stddev) initializer = RandomNormal(mean=0.0, stddev=stddev, seed=None) word_input = Input(shape=(1,), name="word_input") word = Embedding(input_dim=vocab_size, output_dim=vector_dim, input_length=1, name="word_embedding", embeddings_initializer=initializer)(word_input) context_input = Input(shape=(1,), name="context_input") context = Embedding(input_dim=vocab_size, output_dim=vector_dim, input_length=1, name="context_embedding", embeddings_initializer=initializer)(context_input) merged = dot([word, context], axes=2, normalize=False, name="dot") merged = Flatten()(merged) output = Dense(1, activation='sigmoid', name="output")(merged) optimizer = TFOptimizer(tf.train.AdagradOptimizer(learn_rate)) model = Model(inputs=[word_input, context_input], outputs=output) model.compile(loss="binary_crossentropy", optimizer=optimizer) self.model = model
def isensee2017_model_dil3(input_shape=(4, 128, 128, 128), n_base_filters=16, depth=5, dropout_rate=0.3, n_segmentation_levels=3, n_labels=4, optimizer="Adam", initial_learning_rate=5e-4, loss_function=weighted_dice_coefficient_loss, activation_name="sigmoid"): """ This function builds a model proposed by Isensee et al. for the BRATS 2017 competition: https://www.cbica.upenn.edu/sbia/Spyridon.Bakas/MICCAI_BraTS/MICCAI_BraTS_2017_proceedings_shortPapers.pdf This network is highly similar to the model proposed by Kayalibay et al. "CNN-based Segmentation of Medical Imaging Data", 2017: https://arxiv.org/pdf/1701.03056.pdf :param input_shape: :param n_base_filters: :param depth: :param dropout_rate: :param n_segmentation_levels: :param n_labels: :param optimizer: :param initial_learning_rate: :param loss_function: :param activation_name: :return: """ inputs = Input(input_shape) current_layer = inputs level_output_layers = list() level_filters = list() for level_number in range(depth): n_level_filters = (2**level_number) * n_base_filters level_filters.append(n_level_filters) if current_layer is inputs: in_conv = create_convolution_block(current_layer, n_level_filters) else: in_conv = create_convolution_block(current_layer, n_level_filters, strides=(2, 2, 2), dilation_rate=(1, 1, 1)) context_output_layer = create_context_module(in_conv, n_level_filters, dropout_rate=dropout_rate) summation_layer = Add()([in_conv, context_output_layer]) level_output_layers.append(summation_layer) current_layer = summation_layer segmentation_layers = list() for level_number in range(depth - 2, -1, -1): up_sampling = create_up_sampling_module(current_layer, level_filters[level_number]) concatenation_layer = concatenate( [level_output_layers[level_number], up_sampling], axis=1) localization_output = create_localization_module( concatenation_layer, level_filters[level_number]) current_layer = localization_output if level_number < n_segmentation_levels: segmentation_layers.insert( 0, create_convolution_block(current_layer, n_filters=n_labels, kernel=(1, 1, 1))) output_layer = None for level_number in reversed(range(n_segmentation_levels)): segmentation_layer = segmentation_layers[level_number] if output_layer is None: output_layer = segmentation_layer else: output_layer = Add()([output_layer, segmentation_layer]) if level_number > 0: output_layer = UpSampling3D(size=(2, 2, 2))(output_layer) activation_block = Activation(activation_name)(output_layer) model = Model(inputs=inputs, outputs=activation_block) if optimizer == "YellowFin": # import and instantiate yellowFin optimizer from .yellowfinkeras.yellowfin import YFOptimizer from keras.optimizers import TFOptimizer # define your optimizer optimizer = TFOptimizer(YFOptimizer()) model.compile(optimizer=optimizer, loss=loss_function) elif optimizer == "Adam": from keras.optimizers import Adam optimizer = Adam model.compile(optimizer=optimizer(lr=initial_learning_rate), loss=loss_function) else: raise ValueError( str(error) + "\n\nYou can use only Adam or YellowFin optimizer\n") return model
from ensemble.metrics import map from ensemble.np_functions import jrer_fusion as jrer_fusion_np from ensemble.np_functions import avg_fusion as avg_fusion_np from ensemble.preprocessing import randomizer from ensemble.tf_functions import jrer_fusion, avg_fusion from ensemble.optimizers import YFOptimizer from keras.optimizers import TFOptimizer def freeze(model): for layer in model.layers: layer.trainable = False # define your optimizer opt = TFOptimizer(YFOptimizer()) dataset = Dataset(multilabel=True) # replace 5000 with dataset.labels.shape[0] to use all the data (slow and mem intensive). sample_number = dataset.labels.shape[0] # random indexer to retrieve 30% of the test data for validation pick = randomizer(sample_number // 2, validation_split=1.0) # loading the train datasets. _,\ y=next(dataset.generator(source='cnn', samples=sample_number, load_window=1, train=1)) cnn_x,\ _=np.load("inputs/%s_output_dnn_.npy"%'cnn'),y
def sampling_hypersurface(train_x, train_y, test_x, test_y, file_output_path, suffix='', batch_size=256, iterations=1000, verbose=1): """ Sampling hypersurface by different random initialization. Parameters ---------- train_x : np.array(float) Training `x` set (training features). train_y : np.array(int) Training `y` set (training labels). test_x : np.array(float) Test `x` set (test features). test_y : np.array(int) Test `y` set (test labels). file_output_path : str Path to store h5 files generated by custom BestAccs callback. suffix: str, optional File name suffix. batch_size : int, optional Size of batch (amount of samples) for model fitting. iterations : int, optional Defines how many times defining, compilation and fitting procedure has to be executed. verbose : int, optional Verbosity level: 0 or 1. 0 means quite run, 1 means progress will be shown. """ if not os.path.exists(file_output_path): os.makedirs(file_output_path) # Define form of h5 file name prefix metrics = '{:.3f}-{acc:.3f}-{val_acc:.3f}' initializer = RandomUniform(minval=-1.0, maxval=1.0, seed=None) lr = 0.1 optimizer = TFOptimizer(tf.train.GradientDescentOptimizer(lr)) for iteration in range(iterations): filename = metrics + '-iteration-' + str(iteration) + suffix + '.h5' path_and_filename = os.path.join(file_output_path, '') path_and_filename += filename # workaround of os.path.join issue best_accs = BestAccs(path_and_filename, min_accs=0.88, diff=0.04) model = Sequential() model.add( Dense(1, kernel_initializer=initializer, bias_initializer=initializer, input_dim=train_x.shape[1], activation='sigmoid')) model.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy']) model.fit(train_x, train_y, epochs=1000, callbacks=[best_accs], validation_data=(test_x, test_y), batch_size=batch_size, verbose=0) if verbose == 1: if (iteration + 1) % 100 == 0: end = '\n' else: end = '' if (iteration + 1) % 10 == 0: print('|', end=end) else: print('.', end=end)
def __init__(self, optimizer, gdev_list=None): TFOptimizer.__init__(self, optimizer) self._gdev_list = gdev_list
def course_assignment(train_x, train_y, test_x, test_y, file_output_path, initializer='zeros', batch_size=256): """ First Coursera Deep Learning course programming assignment in Keras. Training of single neuron with sigmoid activation function and set of images. Our goal is to teach that neuron to recognize images with cat. Parameters ---------- train_x : np.array(float) Training `x` set (training features). train_y : np.array(int) Training `y` set (training labels). test_x : np.array(float) Test `x` set (test features). test_y : np.array(int) Test `y` set (test labels). file_output_path : str Path to store h5 files generated by custom BestAccs callback. initializer : str, optional Type of kernel and bias initializer. batch_size : int, optional Size of batch (amount of samples) for model fitting. Returns ------- history : keras.callbacks.History object History of loss, accuracy, validation loss and validation accuracy during model fitting. """ if not os.path.exists(file_output_path): os.makedirs(file_output_path) # Define form of h5 file name prefix metrics = '{:.2f}-{acc:.2f}-{val_acc:.2f}' filename = metrics + '-' + initializer + '.h5' path_and_filename = os.path.join(file_output_path, '') path_and_filename += filename # workaround of os.path.join issue best_accs = BestAccs(path_and_filename, min_accs=0.7, diff=0.02) lr = 0.005 optimizer = TFOptimizer(tf.train.GradientDescentOptimizer(lr)) model = Sequential() model.add( Dense(1, kernel_initializer=initializer, bias_initializer=initializer, input_dim=train_x.shape[1], activation='sigmoid')) model.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy']) history = model.fit(train_x, train_y, epochs=2000, callbacks=[best_accs], validation_data=(test_x, test_y), batch_size=batch_size, verbose=0) return history
print("Tokenizing reviews...") train_X = preprocess_reviews(train_X, tokenizer) test_X = preprocess_reviews(test_X, tokenizer) embedding_matrix = create_embedding_matrix(tokenizer, all_X, load=True) # vectorize labels train_Y = np.delete(to_categorical(train_Y), 0, axis=1) test_Y = np.delete(to_categorical(test_Y), 0, axis=1) model = models.Sequential() model.add(layers.embeddings.Embedding(NUM_WORDS, VECTOR_DIM, input_length=MAX_LENGTH, weights=[embedding_matrix], trainable=True)) model.add(layers.LSTM(32)) model.add(layers.Dense(5, activation='softmax')) model.summary() model.compile(optimizer=TFOptimizer(tf.optimizers.Adam()), loss='categorical_crossentropy', metrics=['accuracy']) history = model.fit(train_X, train_Y, epochs=0, batch_size=32, validation_split=0.1) model.save('amazon.h5') pred_Y = model.predict_classes(test_X) val_Y = [np.argmax(x) for x in test_Y] confusion_matrix = tf.math.confusion_matrix(labels=val_Y, predictions=pred_Y).numpy()
def train_text_model( corpus: Corpus, featurizer: Featurizer, model_options: ModelOptions, models_ann_dir=None, debug=False, tensorboard_dir=None, ): """ Utility function for training citeomatic models. """ # load pretrained embeddings if model_options.use_pretrained: dp = DatasetPaths() pretrained_embeddings_file = dp.embeddings_weights_for_corpus('shared') with h5py.File(pretrained_embeddings_file, 'r') as f: pretrained_embeddings = f['embedding'][...] else: pretrained_embeddings = None create_model = import_from( 'citeomatic.models.%s' % model_options.model_name, 'create_model') models = create_model(model_options, pretrained_embeddings) model, embedding_model = models['citeomatic'], models['embedding'] logging.info(model.summary()) if model_options.train_for_test_set: paper_ids_for_training = corpus.train_ids + corpus.valid_ids candidates_for_training = corpus.train_ids + corpus.valid_ids + corpus.test_ids else: paper_ids_for_training = corpus.train_ids candidates_for_training = corpus.train_ids + corpus.valid_ids training_dg = DataGenerator( corpus=corpus, featurizer=featurizer, margin_multiplier=model_options.margin_multiplier, use_variable_margin=model_options.use_variable_margin) training_generator = training_dg.triplet_generator( paper_ids=paper_ids_for_training, candidate_ids=candidates_for_training, batch_size=model_options.batch_size, neg_to_pos_ratio=model_options.neg_to_pos_ratio) validation_dg = DataGenerator( corpus=corpus, featurizer=featurizer, margin_multiplier=model_options.margin_multiplier, use_variable_margin=model_options.use_variable_margin) validation_generator = validation_dg.triplet_generator( paper_ids=corpus.valid_ids, candidate_ids=corpus.train_ids + corpus.valid_ids, batch_size=1024, neg_to_pos_ratio=model_options.neg_to_pos_ratio) if model_options.optimizer == 'tfopt': optimizer = TFOptimizer( tf.contrib.opt.LazyAdamOptimizer(learning_rate=model_options.lr)) else: optimizer = import_from('keras.optimizers', model_options.optimizer)(lr=model_options.lr) model.compile(optimizer=optimizer, loss=layers.triplet_loss) # training calculation model_options.samples_per_epoch = int( np.minimum(model_options.samples_per_epoch, model_options.total_samples)) epochs = int( np.ceil(model_options.total_samples / model_options.samples_per_epoch)) steps_per_epoch = int(model_options.samples_per_epoch / model_options.batch_size) # callbacks callbacks_list = [] if debug: callbacks_list.append(MemoryUsageCallback()) if model_options.tb_dir is not None: callbacks_list.append( TensorBoard(log_dir=model_options.tb_dir, histogram_freq=1, write_graph=True)) if model_options.reduce_lr_flag: if model_options.optimizer != 'tfopt': callbacks_list.append( ReduceLROnPlateau(verbose=1, patience=2, epsilon=0.01, min_lr=1e-6, factor=0.5)) if models_ann_dir is None: ann_featurizer = featurizer paper_embedding_model = embedding_model embed_at_epoch_end = True embed_at_train_begin = False else: ann_featurizer, ann_models = model_from_directory(models_ann_dir, on_cpu=True) paper_embedding_model = ann_models['embedding'] paper_embedding_model._make_predict_function() embed_at_epoch_end = False embed_at_train_begin = True callbacks_list.append( UpdateANN(corpus, ann_featurizer, paper_embedding_model, training_dg, validation_dg, embed_at_epoch_end, embed_at_train_begin)) if model_options.tb_dir is None: validation_data = validation_generator else: validation_data = next(validation_generator) # logic model.fit_generator(generator=training_generator, steps_per_epoch=steps_per_epoch, epochs=epochs, callbacks=callbacks_list, validation_data=validation_generator, validation_steps=10) return model, embedding_model
for identity_id in range(len(identities) - 1): welcome_message += '%s, ' % identities[identity_id] welcome_message += 'and %s, ' % identities[-1] welcome_message += 'have a nice day!' windows10_voice_interface.Speak(welcome_message) # Allow the program to start detecting identities again ready_to_detect_identity = True if __name__ == "__main__": windows10_voice_interface = wincl.Dispatch("SAPI.SpVoice") FRmodel = faceRecoModel(input_shape=(3, 96, 96)) FRmodel.compile(optimizer=TFOptimizer( tf.train.GradientDescentOptimizer(0.1)), loss=triplet_loss, metrics=['accuracy']) load_weights_from_FaceNet(FRmodel) database = prepare_database() webcam_face_recognizer(database) # ### References: # # - Florian Schroff, Dmitry Kalenichenko, James Philbin (2015). [FaceNet: A Unified Embedding for Face Recognition and Clustering](https://arxiv.org/pdf/1503.03832.pdf) # - Yaniv Taigman, Ming Yang, Marc'Aurelio Ranzato, Lior Wolf (2014). [DeepFace: Closing the gap to human-level performance in face verification](https://research.fb.com/wp-content/uploads/2016/11/deepface-closing-the-gap-to-human-level-performance-in-face-verification.pdf) # - The pretrained model we use is inspired by Victor Sy Wang's implementation and was loaded using his code: https://github.com/iwantooxxoox/Keras-OpenFace. # - Our implementation also took a lot of inspiration from the official FaceNet github repository: https://github.com/davidsandberg/facenet #
target_size=(224, 224), color_mode='grayscale', classes = class_names, class_mode='categorical', batch_size=256, shuffle=True) with K.tf.device('/cpu:0'): kernel_model = MobileNet( input_shape=(224, 224, 1), alpha=1.0, include_top=True, weights=None, classes=len(class_names)) parallel_model = multi_gpu_model(kernel_model, gpus=8) optimizer = TFOptimizer(tf.train.RMSPropOptimizer(learning_rate=LEARNING_RATE, decay=DECAY, momentum=0.9)) # Tensorflow RMSProp parallel_model.compile( optimizer=optimizer, loss='categorical_crossentropy', # for multiclass classification metrics=['accuracy']) parallel_model.fit_generator( train_generator, steps_per_epoch=25000, epochs=10, verbose=1, max_queue_size=100, use_multiprocessing=True, workers=cpu_count(), callbacks=[ ModelCheckpoint(
def main(argv=None): ''' ''' main.__doc__ = __doc__ argv = sys.argv if argv is None else sys.argv.extend(argv) desc = main.__doc__ # .format(os.path.basename(__file__)) # CLI parser args = parser_(desc) nranks_per_gpu = args.nranks_per_gpu local_rank = hvd.local_rank() gpu_local_rank = local_rank // nranks_per_gpu print('local_rank, GPU_LOCAL_RANK: {}, {}'.format(local_rank, gpu_local_rank)) # Pin GPU to be used to process local rank (one GPU per process) config = tf.ConfigProto() config.gpu_options.allow_growth = True # config.gpu_options.visible_device_list = str(hvd.local_rank()) config.gpu_options.visible_device_list = str(gpu_local_rank) K.set_session(tf.Session(config=config)) # input image dimensions img_rows, img_cols, img_chns = 28, 28, 1 # number of convolutional filters to use filters = 64 # convolution kernel size num_conv = 3 hvdsize = hvd.size() batch_size = 128 # 100 if K.image_data_format() == 'channels_first': original_img_size = (img_chns, img_rows, img_cols) else: original_img_size = (img_rows, img_cols, img_chns) latent_dim = 2 intermediate_dim = 128 epsilon_std = 1.0 epochs = args.epochs # 5 # train the VAE on MNIST digits (x_train, _), (x_test, y_test) = mnist.load_data() x_train = x_train.astype('float32') / 255. x_train = x_train.reshape((x_train.shape[0], ) + original_img_size) x_test = x_test.astype('float32') / 255. x_test = x_test.reshape((x_test.shape[0], ) + original_img_size) if hvd.rank() == 0: print('x_train.shape:', x_train.shape) train_samples = x_train.shape[0] # steps_per_epoch = train_samples // batch_size // hvdsize speedupopt = args.speedup if speedupopt == SpeedupOpts.imgspersec: steps_per_epoch = train_samples // batch_size else: steps_per_epoch = int( round(float(train_samples) / batch_size / hvdsize + 0.5)) # Create the dataset and its associated one-shot iterator. buffer_size = 10000 dataset = Dataset.from_tensor_slices(x_train) dataset = dataset.repeat() dataset = dataset.shuffle(buffer_size) dataset = dataset.batch(batch_size) iterator = dataset.make_one_shot_iterator() x_train_batch = iterator.get_next() ldict = make_shared_layers_dict(img_chns, img_rows, img_cols, batch_size, filters, num_conv, intermediate_dim, latent_dim, epsilon_std) # ldict is a dictionary that holds all layers. Since these layers are # instantiated once, they are shared amongs vae, encoder, and generator. x = Input(tensor=x_train_batch) vae = make_vae(ldict, x) # : :type vae: Model lr = 0.001 # * hvdsize opt = tf.train.RMSPropOptimizer(lr) # Add Horovod Distributed Optimizer. opt = hvd.DistributedOptimizer(opt) # , use_locking=True) opt = TFOptimizer(opt) # opt = RMSprop(lr) # Add Horovod Distributed Optimizer. # opt = hvd_keras.DistributedOptimizer(opt) # , use_locking=True) vae.compile(optimizer=opt, loss=None) if hvd.rank() == 0: vae.summary() callbacks = [] if hvd.rank() == 0: callbacks += [BatchTiming(), SamplesPerSec(batch_size * hvdsize)] sess = K.get_session() sess.run(hvd.broadcast_global_variables(0)) # Fit the model using data from the TF data tensors. vae.fit(steps_per_epoch=steps_per_epoch, epochs=epochs, callbacks=callbacks) if hvd.rank() == 0: x = Input(shape=original_img_size) vae_val = make_vae(ldict, x) vae_val.compile(optimizer=opt, loss=None) loss = vae_val.evaluate(x=x_test, y=None, batch_size=batch_size) print('\n\nVAE VALIDATION LOSS: {}'.format(loss)) x = Input(shape=original_img_size) z_mean, _ = get_encoded(ldict, x) encoder = Model(x, z_mean) # : :type encoder: Model decoder_input = Input(shape=(latent_dim, )) x_decoded_mean_squash = get_decoded(ldict, decoder_input) generator = Model(decoder_input, x_decoded_mean_squash) # : :type generator: Model # display a 2D plot of the digit classes in the latent space x_test_encoded = encoder.predict(x_test, batch_size=batch_size) plt.figure(figsize=(6, 6)) plt.scatter(x_test_encoded[:, 0], x_test_encoded[:, 1], c=y_test) plt.colorbar() # plt.show() plt.savefig('vae_scatter.ps') plt.close() # display a 2D manifold of the digits n = 15 # figure with 15x15 digits digit_size = 28 figure = np.zeros((digit_size * n, digit_size * n)) # Linearly spaced coordinates on the unit square were transformed # through the inverse CDF (ppf) of the Gaussian # To produce values of the latent variables z, since the prior of the # latent space is Gaussian grid_x = norm.ppf(np.linspace(0.05, 0.95, n)) grid_y = norm.ppf(np.linspace(0.05, 0.95, n)) for i, yi in enumerate(grid_x): for j, xi in enumerate(grid_y): z_sample = np.array([[xi, yi]]) z_sample = np.tile(z_sample, batch_size).reshape(batch_size, 2) x_decoded = generator.predict(z_sample, batch_size=batch_size) digit = x_decoded[0].reshape(digit_size, digit_size) figure[i * digit_size:(i + 1) * digit_size, j * digit_size:(j + 1) * digit_size] = digit plt.figure(figsize=(10, 10)) plt.imshow(figure, cmap='Greys_r') # plt.show() plt.savefig('vae_digit.ps') plt.close() K.clear_session()
def main(argv=None): ''' ''' main.__doc__ = __doc__ argv = sys.argv if argv is None else sys.argv.extend(argv) desc = main.__doc__ # .format(os.path.basename(__file__)) # CLI parser args = parser_(desc) nranks_per_gpu = args.nranks_per_gpu local_rank = hvd.local_rank() gpu_local_rank = local_rank // nranks_per_gpu print('local_rank, GPU_LOCAL_RANK: {}, {}'.format(local_rank, gpu_local_rank)) # Pin GPU to be used to process local rank (one GPU per process) config = tf.ConfigProto() config.gpu_options.allow_growth = True # config.gpu_options.visible_device_list = str(hvd.local_rank()) config.gpu_options.visible_device_list = str(gpu_local_rank) K.set_session(tf.Session(config=config)) # input image dimensions img_rows, img_cols, img_chns = 28, 28, 1 # number of convolutional filters to use filters = 64 # convolution kernel size num_conv = 3 hvdsize = hvd.size() batch_size = 128 # 100 if K.image_data_format() == 'channels_first': original_img_size = (img_chns, img_rows, img_cols) else: original_img_size = (img_rows, img_cols, img_chns) latent_dim = 2 intermediate_dim = 128 epsilon_std = 1.0 epochs = args.epochs # 5 # train the VAE on MNIST digits (x_train, _), (x_test, y_test) = mnist.load_data() # Data split if going for reduction in each iteration step. Using # tf-queue or dataset is better to preserve uniform random sampling. # nsamples = x_train.shape[0] # mysamples = nsamples // hvdsize # start_sam = hvd.local_rank() * mysamples # stop_sam = min((hvd.local_rank() + 1) * mysamples, nsamples) # x_train = x_train[start_sam:stop_sam, ...] x_train = x_train.astype('float32') / 255. x_train = x_train.reshape((x_train.shape[0], ) + original_img_size) x_test = x_test.astype('float32') / 255. x_test = x_test.reshape((x_test.shape[0], ) + original_img_size) if hvd.rank() == 0: print('x_train.shape:', x_train.shape) vae, encoder, generator = make_vae_and_codec(original_img_size, img_chns, img_rows, img_cols, batch_size, filters, num_conv, intermediate_dim, latent_dim, epsilon_std) # : :type vae: Model lr = 0.001 # * hvdsize opt = tf.train.RMSPropOptimizer(lr) # Add Horovod Distributed Optimizer. opt = hvd.DistributedOptimizer(opt) # , use_locking=True) opt = TFOptimizer(opt) vae.compile(optimizer=opt, loss=None) if hvd.rank() == 0: vae.summary() callbacks = [] if hvd.rank() == 0: callbacks += [BatchTiming(), SamplesPerSec(batch_size * hvdsize)] sess = K.get_session() sess.run(hvd.broadcast_global_variables(0)) vae.fit(x_train, shuffle=True, epochs=epochs, batch_size=batch_size, validation_data=(x_test, None), callbacks=callbacks) if hvd.rank() == 0: vae_val = vae loss = vae_val.evaluate(x=x_test, y=None, batch_size=batch_size) print('\n\nVAE VALIDATION LOSS: {}'.format(loss)) # display a 2D plot of the digit classes in the latent space x_test_encoded = encoder.predict(x_test, batch_size=batch_size) plt.figure(figsize=(6, 6)) plt.scatter(x_test_encoded[:, 0], x_test_encoded[:, 1], c=y_test) plt.colorbar() # plt.show() plt.savefig('vae_scatter.ps') plt.close() # display a 2D manifold of the digits n = 15 # figure with 15x15 digits digit_size = 28 figure = np.zeros((digit_size * n, digit_size * n)) # Linearly spaced coordinates on the unit square were transformed # through the inverse CDF (ppf) of the Gaussian # To produce values of the latent variables z, since the prior of the # latent space is Gaussian grid_x = norm.ppf(np.linspace(0.05, 0.95, n)) grid_y = norm.ppf(np.linspace(0.05, 0.95, n)) for i, yi in enumerate(grid_x): for j, xi in enumerate(grid_y): z_sample = np.array([[xi, yi]]) z_sample = np.tile(z_sample, batch_size).reshape(batch_size, 2) x_decoded = generator.predict(z_sample, batch_size=batch_size) digit = x_decoded[0].reshape(digit_size, digit_size) figure[i * digit_size:(i + 1) * digit_size, j * digit_size:(j + 1) * digit_size] = digit plt.figure(figsize=(10, 10)) plt.imshow(figure, cmap='Greys_r') # plt.show() plt.savefig('vae_digit.ps') plt.close() K.clear_session()
activation = 'selu' layer_size = 128 n_layers = 2 drop_rate = 0.01 for n in range(n_layers): x = Dense(layer_size)(x) x = Activation(activation)(x) x = AlphaDropout(rate=drop_rate)(x) # main output for loss calculation #2 main_pred = Dense(1, name='main_out')(x) # define inputs / outputs model = Model(inputs=[main_input, aux_input], outputs=[main_pred, gru_pred]) # weight losses and compile model model.compile(optimizer=TFOptimizer(YFOptimizer()), loss='mean_squared_error', loss_weights=[.5, .5]) model.fit([X1, X2], [y, y], epochs=50, validation_split=0.5, batch_size=256, callbacks=[EarlyStopping(patience=8)]) # clear graph K.clear_session()