# make a new resnet without the softmax x = resnet_.layers[-2].output W, b = resnet_.layers[-1].get_weights() y = Dense(1000)(x) resnet = Model(resnet_.input, y) resnet.layers[-1].set_weights([W, b]) # you can determine the correct layer # by looking at resnet.layers in the console partial_model = Model(inputs=resnet.input, outputs=resnet.layers[175].output) # maybe useful when building your model # to look at the layers you're trying to copy print(partial_model.summary()) # create an instance of our own model my_partial_resnet = TFResNet() # make a fake image X = np.random.random((1, 224, 224, 3)) # get keras output keras_output = partial_model.predict(X) ### get my model output ### # init only the variables in our net init = tf.variables_initializer(my_partial_resnet.get_params())
# Depthwise Convolution x = DepthwiseConv2D((3, 3), padding='same')(input) x = BatchNormalization()(x) x = ReLU()(x) # Pointwise Convolution x = Conv2D(128, (1, 1))(x) x = BatchNormalization()(x) x = ReLU()(x) return x # %% ''' ##### 建構模型 ''' # %% input = Input((64, 64, 3)) output = SeparableConv(input) model = Model(inputs=input, outputs=output) model.summary() # %% ''' 更多相關連接參考: https://github.com/keras-team/keras-applications/blob/master/keras_applications/mobilenet.py#L364 ''' # %%
def train(self, dataset): # Transform data into format to be fed into model # Below code is more suitable for run mode than train mode ''' (X, Y, X_valid, Y_valid) = dataset.load_as_list() X = self.trainable_model.encode_input(X) Y = self.trainable_model.encode_output(Y) X_valid = self.trainable_model.encode_input(X_valid) Y_valid = self.trainable_model.encode_output(Y_valid) ''' # If using multi-gpu, then we save model/log files in other directory than normal one dir_suffix = '' gpu_count = len(self.get_available_gpus()) if self.multi_gpu: gpu_count = len(self.get_available_gpus()) # Changed to save multi-gpu model at the same path as single gpu model #if gpu_count > 1: # dir_suffix = '_' + str(gpu_count) + 'gpus' print('Training on ' + str(gpu_count) + ' GPU(s)') # In case of train mode, we can load data in the wqay that we can utilize caching feature. # We separate call between input and output because they are use different transformation approach. (X, Y, X_valid, Y_valid) = self.trainable_model.load_encoded_data(dataset) print(len(X[0])) print(len(Y)) print(len(X_valid[0])) print(len(Y_valid)) ''' xx = X[0:5] yy = Y[0:5] print('xx') print(xx) print('yy') print(yy) ''' training_data_count = 0 if self.input_transform.get_data_dimension() > 1: training_data_count = X[0].shape[0] else: training_data_count = X.shape[0] print('Training data count = ' + str(training_data_count)) batch_count = int(training_data_count / self.training_config['batch_size']) print('Batch count = ' + str(batch_count)) training_data_count = int(batch_count * self.training_config['batch_size']) print('Training data used = ' + str(training_data_count)) epochs_count = int(self.training_config['epochs']) if 'final_epochs' in self.training_config: # Federated learning will have this vale overidden epochs_count = int(self.training_config['final_epochs']) training_steps = int(batch_count) * epochs_count training_batch_count = batch_count validation_data_count = 0 if self.input_transform.get_data_dimension() > 1: validation_data_count = X_valid[0].shape[0] else: validation_data_count = X_valid.shape[0] print('Validation data count = ' + str(validation_data_count)) batch_count = int(validation_data_count / self.training_config['batch_size']) print('Batch count = ' + str(batch_count)) validation_data_count = int(batch_count * self.training_config['batch_size']) print('Validation data used = ' + str(validation_data_count)) if self.input_transform.get_data_dimension() > 1: X = [a[0:training_data_count] for a in X] X_valid = [a[0:validation_data_count] for a in X_valid] print('>>> X len = ' + str(len(X[0]))) print('>>> X_valid len = ' + str(len(X_valid[0]))) else: X = X[0:training_data_count] X_valid = X_valid[0:validation_data_count] print('>>>> X len = ' + str(X.shape[0])) print('>>>> X_valid len = ' + str(X_valid.shape[0])) if self.output_transform.get_data_dimension() > 1: Y = [a[0:training_data_count] for a in Y] Y_valid = [a[0:validation_data_count] for a in Y_valid] print('>>> Y len = ' + str(len(X[0]))) print('>>> Y_valid len = ' + str(len(X_valid[0]))) else: Y = Y[0:training_data_count] Y_valid = Y_valid[0:validation_data_count] print('>>>> Y len = ' + str(Y.shape[0])) print('>>>> Y_valid len = ' + str(Y_valid.shape[0])) # If multi-model, wrap it as Data Parallel trainable model if gpu_count > 1: with tf.device('/cpu'): [input_tensors, output_tensors] = self.trainable_model.get_forward_tensors() print("=== INPUT_TENSOR ===") print(input_tensors) print("=== OUTPUT_TENSOR ===") print(output_tensors) model = Model(input_tensors, output_tensors) print("=== CPU TEMPLATE MODEL ===") model.summary() single_gpu_model = model # For saving weight model = multi_gpu_model(model, gpus=gpu_count) print("=== MULTI-GPU MODEL ===") model.summary() elif gpu_count == 1: with tf.device('/gpu'): [input_tensors, output_tensors] = self.trainable_model.get_forward_tensors() model = Model(input_tensors, output_tensors) single_gpu_model = model elif gpu_count == 0: with tf.device('/cpu'): [input_tensors, output_tensors] = self.trainable_model.get_forward_tensors() model = Model(input_tensors, output_tensors) single_gpu_model = model current_epoch_wrapper = LogCurrentEpochWrapper(self.training_config, dir_suffix) initial_epoch = 0 if 'resume_if_possible' in self.training_config and self.training_config[ 'resume_if_possible'] == True: initial_epoch = current_epoch_wrapper.get_current_epoch() # Home of output directory (support multi-OS) output_dir = os.path.join( *re.split('/|\\\\', self.training_config['output_dir'])) if not os.path.exists(output_dir): os.makedirs(output_dir) optimizer = self.training_config['optimizer'] if optimizer == 'adam': optimizer_params = self.training_config['optimizer_params'] optimizer = Adam(optimizer_params[0], optimizer_params[1], optimizer_params[2], epsilon=optimizer_params[3]) elif optimizer == 'bert_adam': optimizer_params = self.training_config['optimizer_params'] # Calculate total step and set it to decay_steps (learning rate reachs 0 in the every end) total_steps = batch_count * self.training_config['epochs'] print('[INFO] Training with BERT Optimizer with decay_steps = ' + str(total_steps)) from NLP_LIB.optimizer.bert_optimizer import BERTOptimizer optimizer = BERTOptimizer( decay_steps=total_steps, # 100000, warmup_steps=optimizer_params[2], # 10000, learning_rate=optimizer_params[0], # 1e-4, weight_decay=optimizer_params[1], # 0.01, weight_decay_pattern=[ 'embeddings', 'kernel', 'W1', 'W2', 'Wk', 'Wq', 'Wv', 'Wo' ], ) elif optimizer == 'bert': optimizer_params = self.training_config['optimizer_params'] from NLP_LIB.ext.bert.optimization import AdamWeightDecayOptimizer print('initial_epoch = ' + str(initial_epoch)) print('training_batch_count = ' + str(training_batch_count)) initial_step = initial_epoch * training_batch_count print('initial_step = ' + str(initial_step)) optimizer = AdamWeightDecayOptimizer( initial_step= initial_step, # Start from current epoch to keep model running with correct LR learning_rate=optimizer_params[0], # 0.0001, num_train_steps=training_steps, # 100, warmup_steps=optimizer_params[4], # 10, lr_decay_power=optimizer_params[5], weight_decay_rate=optimizer_params[6], beta_1=optimizer_params[1], # 0.9, beta_2=optimizer_params[2], # 0.999, epsilon=optimizer_params[3], # 1e-6, exclude_from_weight_decay=["LayerNorm", "layer_norm", "bias"]) # Add model metric names and tensors to tracking list metric_names = self.trainable_model.get_metric_names() metric_funcs = self.trainable_model.get_metric_functions() ''' metric_names = self.trainable_model.get_metric_names() metric_tensors = self.trainable_model.get_metric_tensors() for metric_name, metric_tensor in zip(metric_names, metric_tensors): print('Add Metric: ' + metric_name) model.metrics_names.append(metric_name) model.metrics_tensors.append(metric_tensor) ''' model.compile(optimizer=optimizer, loss=self.trainable_model.get_loss_function(), metrics=metric_funcs) model.summary() if self.input_transform.get_data_dimension() > 1: x_feed = X x_valid_feed = X_valid else: x_feed = [X] x_valid_feed = [X_valid] #exit(0) if self.output_transform.get_data_dimension() > 1: y_feed = Y y_valid_feed = Y_valid else: y_feed = [Y] y_valid_feed = [Y_valid] # If model is sequence model, we have to feed prev_output too. # TODO: Can we embed the flow to generate input list into the data transformation class? if isinstance(self.trainable_model, SequenceModelWrapper): print('OH NOOO!!!') #exit(0) x_feed.append(Y) x_valid_feed.append(Y_valid) # Also, if we are running Sequence Model, output will be logits but label will be sparse value. # Keras loss function need label and output to be in same dimension, thus we need to convert label to dense value too. # The converson to Dense is done in custom loss funciton in the model, but be need to "prepare" addition dimension to sparse label. y_feed = [np.expand_dims(Y, axis=2)] y_valid_feed = [np.expand_dims(Y_valid, axis=2)] class CustomTensorBoard(TensorBoard): def __init__( self, log_dir, **kwargs): # add other arguments to __init__ if you need super().__init__(log_dir=log_dir, **kwargs) def on_epoch_end(self, epoch, logs=None): logs = logs or {} # If there is learning_rate_tensor in the optimizer, we want to log it too. if hasattr(optimizer, 'learning_rate_tensor'): logs.update({ 'learning_rate': K.eval(optimizer.learning_rate_tensor) }) ''' # Also add gradient norm as a default metric # Get a "l2 norm of gradients" tensor def get_gradient_norm(model): with K.name_scope('gradient_norm'): grads = K.gradients(model.total_loss, model.trainable_weights) norm = K.sqrt(sum([K.sum(K.square(g)) for g in grads])) return norm logs.update({'gradient_norm': K.eval(get_gradient_norm(model))}) ''' super().on_epoch_end(epoch, logs) # Tensorboard log directory tboard_log_dir = os.path.join(output_dir, 'tboard_log' + dir_suffix) if not os.path.exists(tboard_log_dir): os.makedirs(tboard_log_dir) tboard_log_saver = CustomTensorBoard(tboard_log_dir, write_graph=False, write_images=False) # For saving weight history along with accuracy in each epoch (May use a lot of disk) verbose_model_saver = None if self.training_config['save_weight_history']: verbose_log_dir = os.path.join(output_dir, 'weight_history' + dir_suffix) if not os.path.exists(verbose_log_dir): os.makedirs(verbose_log_dir) verbose_weight_history_filepath = os.path.join( verbose_log_dir, 'weights.{epoch:02d}-{' + self.training_config['watch_metric'] + ':.4f}.h5') # If there is option to specified number of eopch to be saved if 'save_weight_every' in self.training_config: save_weight_every = self.training_config['save_weight_every'] print('[INFO] Save weight every = ' + str(save_weight_every)) verbose_model_saver = RefModelCheckpoint( verbose_weight_history_filepath, single_gpu_model, save_best_only=False, save_weights_only=True, period=save_weight_every) else: verbose_model_saver = RefModelCheckpoint( verbose_weight_history_filepath, single_gpu_model, save_best_only=False, save_weights_only=True) model.summary() # Initialize all variables, including local variables created by metrics calculations and optimizers. sess = K.get_session() init = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer()) sess.run(init) ##### ## DEBUG Print some training variable before loading checkpoint #global_vars = tf.global_variables() #print('[DEBUG]: First Weight Name = ' + str(global_vars[0].name)) #print('[DEBUG]: First Weight = ' + str(sess.run(global_vars[0]))) # Callback to model after finish variable initialization, init_from_checkpoint is loaded here. self.trainable_model.on_after_init(single_gpu_model) # If resume training, load latest checkpoint # Checkpoint saving directory checkpoint_dir = os.path.join(output_dir, 'checkpoint') if not os.path.exists(checkpoint_dir): os.makedirs(checkpoint_dir) last_checkpoint_filepath = os.path.join( checkpoint_dir, 'last_weight' + dir_suffix + '.h5') if 'resume_if_possible' in self.training_config and self.training_config[ 'resume_if_possible'] == True: print('Init model ' + str(self) + ' from epoch: ' + str(initial_epoch)) if os.path.exists(last_checkpoint_filepath): print('Init model ' + str(self) + ' from checkpoint: ' + last_checkpoint_filepath) single_gpu_model.load_weights(last_checkpoint_filepath) self.training_config['initial_epoch'] = initial_epoch checkpoint_filepath = os.path.join(checkpoint_dir, 'best_weight' + dir_suffix + '.h5') model_saver = RefModelCheckpoint(checkpoint_filepath, single_gpu_model, save_best_only=True, save_weights_only=True) # Also always save lastest model for continue training last_model_saver = RefModelCheckpoint(last_checkpoint_filepath, single_gpu_model, save_best_only=False, save_weights_only=True) # Construct all training callbacks training_callbacks = [model_saver, last_model_saver, tboard_log_saver] if verbose_model_saver is not None: training_callbacks.append(verbose_model_saver) if self.callback_list is not None: for callback in self.callback_list: training_callbacks.append(callback.get_keras_callback()) # Save current epoch training_callbacks.append(current_epoch_wrapper.get_keras_callback()) ##### ## DEBUG Print some training variable before after checkpoint #global_vars = tf.global_variables() #print('[DEBUG]: First Weight Name = ' + str(global_vars[0].name)) #print('[DEBUG]: First Weight = ' + str(sess.run(global_vars[0]))) print('Start training.') ''' with tf.Session(config = tf.ConfigProto(log_device_placement = False, allow_soft_placement=False)) as sess: init = tf.global_variables_initializer() sess.run(init) model.fit(x=x_feed, y=y_feed, batch_size=self.training_config['batch_size'], epochs=self.training_config['epochs'], validation_data=(x_valid_feed, y_valid_feed), callbacks=training_callbacks, initial_epoch=initial_epoch ) ''' # print(model.trainable_weights) model.fit(x=x_feed, y=y_feed, batch_size=self.training_config['batch_size'], epochs=self.training_config['epochs'], validation_data=(x_valid_feed, y_valid_feed), callbacks=training_callbacks, initial_epoch=initial_epoch) print('Finished training.') # Return trained model (single_gpu_model) and validation set as output. # They are used for further benchmarking like in federated training. return (single_gpu_model, x_valid_feed, y_valid_feed)
class seq2seq_train: def __init__(self, cfg): self.cfg = cfg self.enc_inp = None self.enc_outp = None self.dec_inp = None self.dec_outp = None self.enc_model = None self.model = None self.__get_model__() def __get_model__(self): self.enc_inp = Input(shape=(self.cfg.input_seq_len(), ), name="Encoder-Input") embd = Embedding(self.cfg.num_input_tokens(), self.cfg.latent_dim(), name='Encoder-Embedding', mask_zero=False) embd_outp = embd(self.enc_inp) x = BatchNormalization(name='Encoder-Batchnorm-1')(embd_outp) _, state_h = GRU(self.cfg.latent_dim(), return_state=True, name='Encoder-Last-GRU')(x) self.enc_model = Model(inputs=self.enc_inp, outputs=state_h, name='Encoder-Model') self.enc_outp = self.enc_model(self.enc_inp) self.cfg.logger.info("********** Encoder Model summary **************") self.cfg.logger.info(self.enc_model.summary()) # get the decoder self.dec_inp = Input(shape=(None, ), name='Decoder-Input') dec_emb = Embedding(self.cfg.num_output_tokens(), self.cfg.latent_dim(), name='Decoder-Embedding', mask_zero=False)(self.dec_inp) dec_bn = BatchNormalization(name='Decoder-Batchnorm-1')(dec_emb) decoder_gru = GRU(self.cfg.latent_dim(), return_state=True, return_sequences=True, name='Decoder-GRU') decoder_gru_output, _ = decoder_gru(dec_bn, initial_state=self.enc_outp) x = BatchNormalization(name='Decoder-Batchnorm-2')(decoder_gru_output) dec_dense = Dense(self.cfg.num_output_tokens(), activation='softmax', name='Final-Output-Dense') self.dec_outp = dec_dense(x) model_inp = [self.enc_inp, self.dec_inp] self.model = Model(model_inp, self.dec_outp) self.cfg.logger.info("********** Full Model summary **************") self.cfg.logger.info(str(self.model.summary())) plot_model(self.model, to_file=self.cfg.scratch_dir() + os.sep + "seq2seq.png") def fit_model(self, input_vecs, output_vecs): input_data = [input_vecs, output_vecs[:, :-1]] output_data = output_vecs[:, 1:] self.model.compile(optimizer=optimizers.Nadam(lr=0.001), loss='sparse_categorical_crossentropy', metrics=['accuracy']) model_checkpoint = ModelCheckpoint(self.cfg.output_dir() + os.sep + 'model.hdf5', monitor='val_loss', save_best_only=True, period=1) csv_logger = CSVLogger(self.cfg.log_dir() + os.sep + 'history.csv') tb_dir = self.cfg.log_dir() + os.sep + "tensorboard" if os.path.isfile(tb_dir): rmtree(tb_dir) tensorboard = TensorBoard(log_dir=tb_dir, histogram_freq=10, batch_size=self.cfg.batch_size(), write_graph=True, write_grads=False, write_images=False, embeddings_freq=0, embeddings_layer_names=None, embeddings_metadata=None, embeddings_data=None) history = self.model.fit( input_data, np.expand_dims(output_data, -1), batch_size=self.cfg.batch_size(), epochs=self.cfg.nepochs(), validation_split=self.cfg.validation_split(), callbacks=[csv_logger, model_checkpoint, tensorboard]) return (history)
def create_serving_model(self): # Create Keras serving model and return the model object # If using multi-gpu, then we save model/log files in other directory than normal one dir_suffix = '' gpu_count = len(self.get_available_gpus()) if self.multi_gpu: gpu_count = len(self.get_available_gpus()) # Changed to save multi-gpu model at the same path as single gpu model #if gpu_count > 1: # dir_suffix = '_' + str(gpu_count) + 'gpus' print('Running on ' + str(gpu_count) + ' GPU(s)') # If multi-model, wrap it as Data Parallel trainable model if gpu_count > 1: with tf.device('/cpu'): [input_tensors, output_tensors] = self.trainable_model.get_forward_tensors() print("=== INPUT_TENSOR ===") print(input_tensors) print("=== OUTPUT_TENSOR ===") print(output_tensors) model = Model(input_tensors, output_tensors) print("=== CPU TEMPLATE MODEL ===") model.summary() single_gpu_model = model # For saving weight model = multi_gpu_model(model, gpus=gpu_count) print("=== MULTI-GPU MODEL ===") model.summary() elif gpu_count == 1: with tf.device('/gpu'): [input_tensors, output_tensors] = self.trainable_model.get_forward_tensors() model = Model(input_tensors, output_tensors) single_gpu_model = model elif gpu_count == 0: with tf.device('/cpu'): [input_tensors, output_tensors] = self.trainable_model.get_forward_tensors() model = Model(input_tensors, output_tensors) single_gpu_model = model # Home of output directory (support multi-OS) output_dir = os.path.join( *re.split('/|\\\\', self.training_config['output_dir'])) if not os.path.exists(output_dir): os.makedirs(output_dir) # Construct all training callbacks training_callbacks = [] if self.callback_list is not None: for callback in self.callback_list: training_callbacks.append(callback.get_keras_callback()) # Callback to model after finish variable initialization, init_from_checkpoint is loaded here. self.trainable_model.on_after_init(single_gpu_model) # If load best weight from the training folder. checkpoint_dir = os.path.join(output_dir, 'checkpoint') if not os.path.exists(checkpoint_dir): os.makedirs(checkpoint_dir) best_checkpoint_filepath = os.path.join( checkpoint_dir, 'best_weight' + dir_suffix + '.h5') if os.path.exists(best_checkpoint_filepath): print('Init model ' + str(self) + ' from checkpoint: ' + best_checkpoint_filepath) single_gpu_model.load_weights(best_checkpoint_filepath) model.summary() return model