def compute_model(model_size, opt_m, opt, learning_rate, encoder_max_len,
decoder_max_len, epoch_num, tokenizer):
"""
Modularize Compute call in case of failures
"""
global metrics
global tf_train_ds
global tf_valid_ds
global steps
global valid_steps
global model_count
global fall_back_epochs
global model
print(f"Computing Model ===> {model_count}")
if 'model' not in globals():
print("TRAINING A MODEL")
#Compile Model
model = T5Wrapper.from_pretrained(model_size)
model.compile(optimizer=opt_m, metrics=metrics)
#Handle epoch_num
ep = fall_back_epochs if epoch_num == "NONE" else epoch_num
#Model Fit
epochs_done = 0
time_callback = TimeHistory()
history_callback = model.fit(tf_train_ds,
epochs=ep,
steps_per_epoch=steps,
callbacks=[time_callback],
validation_data=tf_valid_ds,
validation_steps=valid_steps,
initial_epoch=epochs_done)
#Call Backs Data
times = time_callback.times
#Data points
total_time = sum(times)
print(f"Model Training Time: {total_time}")
else:
print("GRABBING THE MODEL")
#This is computed both on the validation and test data
compute_generation(model, tokenizer)
save_weights_only=False,
monitor='val_loss',
mode='min',
save_best_only=True)
callbacks = [tensorboard_callback, model_checkpoint_callback]
metrics = [tf.keras.metrics.SparseTopKCategoricalAccuracy(name='accuracy')]
# -
learning_rate = CustomSchedule(
) # learning_rate = 0.001 # Instead set a static learning rate
optimizer = tf.keras.optimizers.Adam(learning_rate)
# ### Init Model
model = T5Wrapper.from_pretrained('t5-small')
model.compile(optimizer=optimizer, metrics=metrics)
model.summary()
# +
#Print out structure of the model
# keras.utils.plot_model(model, show_shapes=True, dpi=90)
# -
# ### Start Tensorboard
#
# %load_ext tensorboard
# %tensorboard --logdir f"{base_dir}/tf_data/experiments/t5/logs"
save_best_only=True)
callbacks = [tensorboard_callback, model_checkpoint_callback]
metrics = [tf.keras.metrics.SparseTopKCategoricalAccuracy(name='accuracy') ]
# -
metrics = [tf.keras.metrics.SparseTopKCategoricalAccuracy(name='accuracy') ]
learning_rate = CustomSchedule() # learning_rate = 0.001 # Instead set a static learning rate
optimizer = tf.keras.optimizers.Adam(learning_rate)
# ### Init Model
model = T5Wrapper.from_pretrained('t5-small')
model.compile(optimizer=optimizer, metrics=metrics)
model.summary()
# +
#Print out structure of the model
# keras.utils.plot_model(model, show_shapes=True, dpi=90)
# -
# ### Start Tensorboard
#
# +
# # %load_ext tensorboard
mode='min',
save_best_only=True)
callbacks = [tensorboard_callback, model_checkpoint_callback]
metrics = [tf.keras.metrics.SparseTopKCategoricalAccuracy(name='accuracy')]
# -
metrics = [tf.keras.metrics.SparseTopKCategoricalAccuracy(name='accuracy')]
learning_rate = CustomSchedule(
) # learning_rate = 0.001 # Instead set a static learning rate
optimizer = tf.keras.optimizers.Adam(learning_rate)
# ### Init Model
model = T5Wrapper.from_pretrained('t5-base')
model.compile(optimizer=optimizer, metrics=metrics)
model.summary()
# +
#Print out structure of the model
# keras.utils.plot_model(model, show_shapes=True, dpi=90)
# -
# ### Start Tensorboard
#
# %load_ext tensorboard
# %tensorboard --logdir f"{base_dir}/tf_data/experiments/t5/logs"
def compute(model_size, opt_m, opt, learning_rate, encoder_max_len,
decoder_max_len, is_special_tokens):
"""
Modularize Compute call in case of failures
"""
global metrics
global tf_train_ds
global tf_valid_ds
global steps
global valid_steps
global epochs
global model_variants
global model_count
print(
f"opt: {opt} learning_rate: {learning_rate} encoder_max_len: {encoder_max_len} decoder_max_len: {decoder_max_len} is_special_tokens:{is_special_tokens}"
)
if 'model' in globals(): del model
#Compile Model
model = T5Wrapper.from_pretrained(model_size)
model.compile(optimizer=opt_m, metrics=metrics)
#Model Fit
epochs_done = 0
time_callback = TimeHistory()
history_callback = model.fit(tf_train_ds,
epochs=epochs,
steps_per_epoch=steps,
callbacks=[time_callback],
validation_data=tf_valid_ds,
validation_steps=valid_steps,
initial_epoch=epochs_done)
#Call Backs Data
times = time_callback.times
fit_data = history_callback.history
#Data points
total_time = sum(times)
avg_epoch_time = np.mean(times)
epoch_num = int(np.argmin(fit_data['val_loss']))
train_accuracy = np.max(fit_data['accuracy'])
train_loss = np.min(fit_data['loss'])
val_accuracy = np.max(fit_data['val_accuracy'])
val_loss = np.min(fit_data['val_loss'])
lr_mod = np.min(fit_data['lr'])
#Gather Details
model_details = {
'id': model_count,
'model_size': model_size,
'optimizer': opt,
'learning_rate':
learning_rate if type(learning_rate) == float else 'custom',
'lr_mod': lr_mod,
'train_accuracy': train_accuracy,
'train_loss': train_loss,
'val_accuracy': val_accuracy,
'val_loss': val_loss,
'total_time': round(total_time / 60, 2),
'avg_epoch_time': avg_epoch_time,
'epoch_num': epoch_num,
'encoder_max_len': encoder_max_len,
'decoder_max_len': decoder_max_len,
'is_special_token': is_special_tokens,
}
model_variants.append(model_details)
#This is computed both on the validation and test data
compute_generation(model, tokenizer)
# ## Kick it Off!
train_params = model_variants[0]
train_params
# +
metrics = [tf.keras.metrics.SparseTopKCategoricalAccuracy(name='accuracy')]
model_count = 1
####
# Using already trained model
model = T5Wrapper.from_pretrained(f'{exp_dir}/model')
###
#Grab all Model details from config
opt = train_params['optimizer']
lr = train_params['lr_mod']
epoch_num = train_params.get('epoch_num', "NONE")
is_special_token = train_params['is_special_token']
encoder_max_len = train_params['encoder_max_len']
decoder_max_len = train_params['decoder_max_len']
model_size = train_params['model_size']
print(
f"Model {model_count} opt: {opt} lr: {lr} epoch_num: {epoch_num} encoder_max_len: {encoder_max_len} decoder_max_len: {decoder_max_len} is_special_token:{is_special_token}"
)