# make a new resnet without the softmax
x = resnet_.layers[-2].output
W, b = resnet_.layers[-1].get_weights()
y = Dense(1000)(x)
resnet = Model(resnet_.input, y)
resnet.layers[-1].set_weights([W, b])
# you can determine the correct layer
# by looking at resnet.layers in the console
partial_model = Model(inputs=resnet.input,
outputs=resnet.layers[175].output)
# maybe useful when building your model
# to look at the layers you're trying to copy
print(partial_model.summary())
# create an instance of our own model
my_partial_resnet = TFResNet()
# make a fake image
X = np.random.random((1, 224, 224, 3))
# get keras output
keras_output = partial_model.predict(X)
### get my model output ###
# init only the variables in our net
init = tf.variables_initializer(my_partial_resnet.get_params())
# Depthwise Convolution
x = DepthwiseConv2D((3, 3), padding='same')(input)
x = BatchNormalization()(x)
x = ReLU()(x)
# Pointwise Convolution
x = Conv2D(128, (1, 1))(x)
x = BatchNormalization()(x)
x = ReLU()(x)
return x
# %%
'''
##### 建構模型
'''
# %%
input = Input((64, 64, 3))
output = SeparableConv(input)
model = Model(inputs=input, outputs=output)
model.summary()
# %%
'''
更多相關連接參考: https://github.com/keras-team/keras-applications/blob/master/keras_applications/mobilenet.py#L364
'''
# %%
def train(self, dataset):
# Transform data into format to be fed into model
# Below code is more suitable for run mode than train mode
'''
(X, Y, X_valid, Y_valid) = dataset.load_as_list()
X = self.trainable_model.encode_input(X)
Y = self.trainable_model.encode_output(Y)
X_valid = self.trainable_model.encode_input(X_valid)
Y_valid = self.trainable_model.encode_output(Y_valid)
'''
# If using multi-gpu, then we save model/log files in other directory than normal one
dir_suffix = ''
gpu_count = len(self.get_available_gpus())
if self.multi_gpu:
gpu_count = len(self.get_available_gpus())
# Changed to save multi-gpu model at the same path as single gpu model
#if gpu_count > 1:
# dir_suffix = '_' + str(gpu_count) + 'gpus'
print('Training on ' + str(gpu_count) + ' GPU(s)')
# In case of train mode, we can load data in the wqay that we can utilize caching feature.
# We separate call between input and output because they are use different transformation approach.
(X, Y, X_valid,
Y_valid) = self.trainable_model.load_encoded_data(dataset)
print(len(X[0]))
print(len(Y))
print(len(X_valid[0]))
print(len(Y_valid))
'''
xx = X[0:5]
yy = Y[0:5]
print('xx')
print(xx)
print('yy')
print(yy)
'''
training_data_count = 0
if self.input_transform.get_data_dimension() > 1:
training_data_count = X[0].shape[0]
else:
training_data_count = X.shape[0]
print('Training data count = ' + str(training_data_count))
batch_count = int(training_data_count /
self.training_config['batch_size'])
print('Batch count = ' + str(batch_count))
training_data_count = int(batch_count *
self.training_config['batch_size'])
print('Training data used = ' + str(training_data_count))
epochs_count = int(self.training_config['epochs'])
if 'final_epochs' in self.training_config: # Federated learning will have this vale overidden
epochs_count = int(self.training_config['final_epochs'])
training_steps = int(batch_count) * epochs_count
training_batch_count = batch_count
validation_data_count = 0
if self.input_transform.get_data_dimension() > 1:
validation_data_count = X_valid[0].shape[0]
else:
validation_data_count = X_valid.shape[0]
print('Validation data count = ' + str(validation_data_count))
batch_count = int(validation_data_count /
self.training_config['batch_size'])
print('Batch count = ' + str(batch_count))
validation_data_count = int(batch_count *
self.training_config['batch_size'])
print('Validation data used = ' + str(validation_data_count))
if self.input_transform.get_data_dimension() > 1:
X = [a[0:training_data_count] for a in X]
X_valid = [a[0:validation_data_count] for a in X_valid]
print('>>> X len = ' + str(len(X[0])))
print('>>> X_valid len = ' + str(len(X_valid[0])))
else:
X = X[0:training_data_count]
X_valid = X_valid[0:validation_data_count]
print('>>>> X len = ' + str(X.shape[0]))
print('>>>> X_valid len = ' + str(X_valid.shape[0]))
if self.output_transform.get_data_dimension() > 1:
Y = [a[0:training_data_count] for a in Y]
Y_valid = [a[0:validation_data_count] for a in Y_valid]
print('>>> Y len = ' + str(len(X[0])))
print('>>> Y_valid len = ' + str(len(X_valid[0])))
else:
Y = Y[0:training_data_count]
Y_valid = Y_valid[0:validation_data_count]
print('>>>> Y len = ' + str(Y.shape[0]))
print('>>>> Y_valid len = ' + str(Y_valid.shape[0]))
# If multi-model, wrap it as Data Parallel trainable model
if gpu_count > 1:
with tf.device('/cpu'):
[input_tensors,
output_tensors] = self.trainable_model.get_forward_tensors()
print("=== INPUT_TENSOR ===")
print(input_tensors)
print("=== OUTPUT_TENSOR ===")
print(output_tensors)
model = Model(input_tensors, output_tensors)
print("=== CPU TEMPLATE MODEL ===")
model.summary()
single_gpu_model = model # For saving weight
model = multi_gpu_model(model, gpus=gpu_count)
print("=== MULTI-GPU MODEL ===")
model.summary()
elif gpu_count == 1:
with tf.device('/gpu'):
[input_tensors,
output_tensors] = self.trainable_model.get_forward_tensors()
model = Model(input_tensors, output_tensors)
single_gpu_model = model
elif gpu_count == 0:
with tf.device('/cpu'):
[input_tensors,
output_tensors] = self.trainable_model.get_forward_tensors()
model = Model(input_tensors, output_tensors)
single_gpu_model = model
current_epoch_wrapper = LogCurrentEpochWrapper(self.training_config,
dir_suffix)
initial_epoch = 0
if 'resume_if_possible' in self.training_config and self.training_config[
'resume_if_possible'] == True:
initial_epoch = current_epoch_wrapper.get_current_epoch()
# Home of output directory (support multi-OS)
output_dir = os.path.join(
*re.split('/|\\\\', self.training_config['output_dir']))
if not os.path.exists(output_dir):
os.makedirs(output_dir)
optimizer = self.training_config['optimizer']
if optimizer == 'adam':
optimizer_params = self.training_config['optimizer_params']
optimizer = Adam(optimizer_params[0],
optimizer_params[1],
optimizer_params[2],
epsilon=optimizer_params[3])
elif optimizer == 'bert_adam':
optimizer_params = self.training_config['optimizer_params']
# Calculate total step and set it to decay_steps (learning rate reachs 0 in the every end)
total_steps = batch_count * self.training_config['epochs']
print('[INFO] Training with BERT Optimizer with decay_steps = ' +
str(total_steps))
from NLP_LIB.optimizer.bert_optimizer import BERTOptimizer
optimizer = BERTOptimizer(
decay_steps=total_steps, # 100000,
warmup_steps=optimizer_params[2], # 10000,
learning_rate=optimizer_params[0], # 1e-4,
weight_decay=optimizer_params[1], # 0.01,
weight_decay_pattern=[
'embeddings', 'kernel', 'W1', 'W2', 'Wk', 'Wq', 'Wv', 'Wo'
],
)
elif optimizer == 'bert':
optimizer_params = self.training_config['optimizer_params']
from NLP_LIB.ext.bert.optimization import AdamWeightDecayOptimizer
print('initial_epoch = ' + str(initial_epoch))
print('training_batch_count = ' + str(training_batch_count))
initial_step = initial_epoch * training_batch_count
print('initial_step = ' + str(initial_step))
optimizer = AdamWeightDecayOptimizer(
initial_step=
initial_step, # Start from current epoch to keep model running with correct LR
learning_rate=optimizer_params[0], # 0.0001,
num_train_steps=training_steps, # 100,
warmup_steps=optimizer_params[4], # 10,
lr_decay_power=optimizer_params[5],
weight_decay_rate=optimizer_params[6],
beta_1=optimizer_params[1], # 0.9,
beta_2=optimizer_params[2], # 0.999,
epsilon=optimizer_params[3], # 1e-6,
exclude_from_weight_decay=["LayerNorm", "layer_norm", "bias"])
# Add model metric names and tensors to tracking list
metric_names = self.trainable_model.get_metric_names()
metric_funcs = self.trainable_model.get_metric_functions()
'''
metric_names = self.trainable_model.get_metric_names()
metric_tensors = self.trainable_model.get_metric_tensors()
for metric_name, metric_tensor in zip(metric_names, metric_tensors):
print('Add Metric: ' + metric_name)
model.metrics_names.append(metric_name)
model.metrics_tensors.append(metric_tensor)
'''
model.compile(optimizer=optimizer,
loss=self.trainable_model.get_loss_function(),
metrics=metric_funcs)
model.summary()
if self.input_transform.get_data_dimension() > 1:
x_feed = X
x_valid_feed = X_valid
else:
x_feed = [X]
x_valid_feed = [X_valid]
#exit(0)
if self.output_transform.get_data_dimension() > 1:
y_feed = Y
y_valid_feed = Y_valid
else:
y_feed = [Y]
y_valid_feed = [Y_valid]
# If model is sequence model, we have to feed prev_output too.
# TODO: Can we embed the flow to generate input list into the data transformation class?
if isinstance(self.trainable_model, SequenceModelWrapper):
print('OH NOOO!!!')
#exit(0)
x_feed.append(Y)
x_valid_feed.append(Y_valid)
# Also, if we are running Sequence Model, output will be logits but label will be sparse value.
# Keras loss function need label and output to be in same dimension, thus we need to convert label to dense value too.
# The converson to Dense is done in custom loss funciton in the model, but be need to "prepare" addition dimension to sparse label.
y_feed = [np.expand_dims(Y, axis=2)]
y_valid_feed = [np.expand_dims(Y_valid, axis=2)]
class CustomTensorBoard(TensorBoard):
def __init__(
self, log_dir,
**kwargs): # add other arguments to __init__ if you need
super().__init__(log_dir=log_dir, **kwargs)
def on_epoch_end(self, epoch, logs=None):
logs = logs or {}
# If there is learning_rate_tensor in the optimizer, we want to log it too.
if hasattr(optimizer, 'learning_rate_tensor'):
logs.update({
'learning_rate':
K.eval(optimizer.learning_rate_tensor)
})
'''
# Also add gradient norm as a default metric
# Get a "l2 norm of gradients" tensor
def get_gradient_norm(model):
with K.name_scope('gradient_norm'):
grads = K.gradients(model.total_loss, model.trainable_weights)
norm = K.sqrt(sum([K.sum(K.square(g)) for g in grads]))
return norm
logs.update({'gradient_norm': K.eval(get_gradient_norm(model))})
'''
super().on_epoch_end(epoch, logs)
# Tensorboard log directory
tboard_log_dir = os.path.join(output_dir, 'tboard_log' + dir_suffix)
if not os.path.exists(tboard_log_dir):
os.makedirs(tboard_log_dir)
tboard_log_saver = CustomTensorBoard(tboard_log_dir,
write_graph=False,
write_images=False)
# For saving weight history along with accuracy in each epoch (May use a lot of disk)
verbose_model_saver = None
if self.training_config['save_weight_history']:
verbose_log_dir = os.path.join(output_dir,
'weight_history' + dir_suffix)
if not os.path.exists(verbose_log_dir):
os.makedirs(verbose_log_dir)
verbose_weight_history_filepath = os.path.join(
verbose_log_dir, 'weights.{epoch:02d}-{' +
self.training_config['watch_metric'] + ':.4f}.h5')
# If there is option to specified number of eopch to be saved
if 'save_weight_every' in self.training_config:
save_weight_every = self.training_config['save_weight_every']
print('[INFO] Save weight every = ' + str(save_weight_every))
verbose_model_saver = RefModelCheckpoint(
verbose_weight_history_filepath,
single_gpu_model,
save_best_only=False,
save_weights_only=True,
period=save_weight_every)
else:
verbose_model_saver = RefModelCheckpoint(
verbose_weight_history_filepath,
single_gpu_model,
save_best_only=False,
save_weights_only=True)
model.summary()
# Initialize all variables, including local variables created by metrics calculations and optimizers.
sess = K.get_session()
init = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
sess.run(init)
#####
## DEBUG Print some training variable before loading checkpoint
#global_vars = tf.global_variables()
#print('[DEBUG]: First Weight Name = ' + str(global_vars[0].name))
#print('[DEBUG]: First Weight = ' + str(sess.run(global_vars[0])))
# Callback to model after finish variable initialization, init_from_checkpoint is loaded here.
self.trainable_model.on_after_init(single_gpu_model)
# If resume training, load latest checkpoint
# Checkpoint saving directory
checkpoint_dir = os.path.join(output_dir, 'checkpoint')
if not os.path.exists(checkpoint_dir):
os.makedirs(checkpoint_dir)
last_checkpoint_filepath = os.path.join(
checkpoint_dir, 'last_weight' + dir_suffix + '.h5')
if 'resume_if_possible' in self.training_config and self.training_config[
'resume_if_possible'] == True:
print('Init model ' + str(self) + ' from epoch: ' +
str(initial_epoch))
if os.path.exists(last_checkpoint_filepath):
print('Init model ' + str(self) + ' from checkpoint: ' +
last_checkpoint_filepath)
single_gpu_model.load_weights(last_checkpoint_filepath)
self.training_config['initial_epoch'] = initial_epoch
checkpoint_filepath = os.path.join(checkpoint_dir,
'best_weight' + dir_suffix + '.h5')
model_saver = RefModelCheckpoint(checkpoint_filepath,
single_gpu_model,
save_best_only=True,
save_weights_only=True)
# Also always save lastest model for continue training
last_model_saver = RefModelCheckpoint(last_checkpoint_filepath,
single_gpu_model,
save_best_only=False,
save_weights_only=True)
# Construct all training callbacks
training_callbacks = [model_saver, last_model_saver, tboard_log_saver]
if verbose_model_saver is not None:
training_callbacks.append(verbose_model_saver)
if self.callback_list is not None:
for callback in self.callback_list:
training_callbacks.append(callback.get_keras_callback())
# Save current epoch
training_callbacks.append(current_epoch_wrapper.get_keras_callback())
#####
## DEBUG Print some training variable before after checkpoint
#global_vars = tf.global_variables()
#print('[DEBUG]: First Weight Name = ' + str(global_vars[0].name))
#print('[DEBUG]: First Weight = ' + str(sess.run(global_vars[0])))
print('Start training.')
'''
with tf.Session(config = tf.ConfigProto(log_device_placement = False, allow_soft_placement=False)) as sess:
init = tf.global_variables_initializer()
sess.run(init)
model.fit(x=x_feed, y=y_feed,
batch_size=self.training_config['batch_size'],
epochs=self.training_config['epochs'],
validation_data=(x_valid_feed, y_valid_feed),
callbacks=training_callbacks,
initial_epoch=initial_epoch
)
'''
# print(model.trainable_weights)
model.fit(x=x_feed,
y=y_feed,
batch_size=self.training_config['batch_size'],
epochs=self.training_config['epochs'],
validation_data=(x_valid_feed, y_valid_feed),
callbacks=training_callbacks,
initial_epoch=initial_epoch)
print('Finished training.')
# Return trained model (single_gpu_model) and validation set as output.
# They are used for further benchmarking like in federated training.
return (single_gpu_model, x_valid_feed, y_valid_feed)
class seq2seq_train:
def __init__(self, cfg):
self.cfg = cfg
self.enc_inp = None
self.enc_outp = None
self.dec_inp = None
self.dec_outp = None
self.enc_model = None
self.model = None
self.__get_model__()
def __get_model__(self):
self.enc_inp = Input(shape=(self.cfg.input_seq_len(), ),
name="Encoder-Input")
embd = Embedding(self.cfg.num_input_tokens(),
self.cfg.latent_dim(),
name='Encoder-Embedding',
mask_zero=False)
embd_outp = embd(self.enc_inp)
x = BatchNormalization(name='Encoder-Batchnorm-1')(embd_outp)
_, state_h = GRU(self.cfg.latent_dim(),
return_state=True,
name='Encoder-Last-GRU')(x)
self.enc_model = Model(inputs=self.enc_inp,
outputs=state_h,
name='Encoder-Model')
self.enc_outp = self.enc_model(self.enc_inp)
self.cfg.logger.info("********** Encoder Model summary **************")
self.cfg.logger.info(self.enc_model.summary())
# get the decoder
self.dec_inp = Input(shape=(None, ), name='Decoder-Input')
dec_emb = Embedding(self.cfg.num_output_tokens(),
self.cfg.latent_dim(),
name='Decoder-Embedding',
mask_zero=False)(self.dec_inp)
dec_bn = BatchNormalization(name='Decoder-Batchnorm-1')(dec_emb)
decoder_gru = GRU(self.cfg.latent_dim(),
return_state=True,
return_sequences=True,
name='Decoder-GRU')
decoder_gru_output, _ = decoder_gru(dec_bn,
initial_state=self.enc_outp)
x = BatchNormalization(name='Decoder-Batchnorm-2')(decoder_gru_output)
dec_dense = Dense(self.cfg.num_output_tokens(),
activation='softmax',
name='Final-Output-Dense')
self.dec_outp = dec_dense(x)
model_inp = [self.enc_inp, self.dec_inp]
self.model = Model(model_inp, self.dec_outp)
self.cfg.logger.info("********** Full Model summary **************")
self.cfg.logger.info(str(self.model.summary()))
plot_model(self.model,
to_file=self.cfg.scratch_dir() + os.sep + "seq2seq.png")
def fit_model(self, input_vecs, output_vecs):
input_data = [input_vecs, output_vecs[:, :-1]]
output_data = output_vecs[:, 1:]
self.model.compile(optimizer=optimizers.Nadam(lr=0.001),
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
model_checkpoint = ModelCheckpoint(self.cfg.output_dir() + os.sep +
'model.hdf5',
monitor='val_loss',
save_best_only=True,
period=1)
csv_logger = CSVLogger(self.cfg.log_dir() + os.sep + 'history.csv')
tb_dir = self.cfg.log_dir() + os.sep + "tensorboard"
if os.path.isfile(tb_dir):
rmtree(tb_dir)
tensorboard = TensorBoard(log_dir=tb_dir,
histogram_freq=10,
batch_size=self.cfg.batch_size(),
write_graph=True,
write_grads=False,
write_images=False,
embeddings_freq=0,
embeddings_layer_names=None,
embeddings_metadata=None,
embeddings_data=None)
history = self.model.fit(
input_data,
np.expand_dims(output_data, -1),
batch_size=self.cfg.batch_size(),
epochs=self.cfg.nepochs(),
validation_split=self.cfg.validation_split(),
callbacks=[csv_logger, model_checkpoint, tensorboard])
return (history)
def create_serving_model(self):
# Create Keras serving model and return the model object
# If using multi-gpu, then we save model/log files in other directory than normal one
dir_suffix = ''
gpu_count = len(self.get_available_gpus())
if self.multi_gpu:
gpu_count = len(self.get_available_gpus())
# Changed to save multi-gpu model at the same path as single gpu model
#if gpu_count > 1:
# dir_suffix = '_' + str(gpu_count) + 'gpus'
print('Running on ' + str(gpu_count) + ' GPU(s)')
# If multi-model, wrap it as Data Parallel trainable model
if gpu_count > 1:
with tf.device('/cpu'):
[input_tensors,
output_tensors] = self.trainable_model.get_forward_tensors()
print("=== INPUT_TENSOR ===")
print(input_tensors)
print("=== OUTPUT_TENSOR ===")
print(output_tensors)
model = Model(input_tensors, output_tensors)
print("=== CPU TEMPLATE MODEL ===")
model.summary()
single_gpu_model = model # For saving weight
model = multi_gpu_model(model, gpus=gpu_count)
print("=== MULTI-GPU MODEL ===")
model.summary()
elif gpu_count == 1:
with tf.device('/gpu'):
[input_tensors,
output_tensors] = self.trainable_model.get_forward_tensors()
model = Model(input_tensors, output_tensors)
single_gpu_model = model
elif gpu_count == 0:
with tf.device('/cpu'):
[input_tensors,
output_tensors] = self.trainable_model.get_forward_tensors()
model = Model(input_tensors, output_tensors)
single_gpu_model = model
# Home of output directory (support multi-OS)
output_dir = os.path.join(
*re.split('/|\\\\', self.training_config['output_dir']))
if not os.path.exists(output_dir):
os.makedirs(output_dir)
# Construct all training callbacks
training_callbacks = []
if self.callback_list is not None:
for callback in self.callback_list:
training_callbacks.append(callback.get_keras_callback())
# Callback to model after finish variable initialization, init_from_checkpoint is loaded here.
self.trainable_model.on_after_init(single_gpu_model)
# If load best weight from the training folder.
checkpoint_dir = os.path.join(output_dir, 'checkpoint')
if not os.path.exists(checkpoint_dir):
os.makedirs(checkpoint_dir)
best_checkpoint_filepath = os.path.join(
checkpoint_dir, 'best_weight' + dir_suffix + '.h5')
if os.path.exists(best_checkpoint_filepath):
print('Init model ' + str(self) + ' from checkpoint: ' +
best_checkpoint_filepath)
single_gpu_model.load_weights(best_checkpoint_filepath)
model.summary()
return model
