def train(model_final, config, model_section):
img_width, img_height,\
batch_size, epochs, \
training_steps_per_epoch, validation_steps_per_epoch,\
positive_weight, model_name = get_metadata_model(config, model_section)
#Read Basic Metadata from config file
train_data_dir, validation_data_dir, test_data_dir,\
results_dir, models_dir, log_dir = map(lambda x : x[1],
config.items("base"))
X_train, Y_train = load_set(train_data_dir,
target_size=(img_height, img_width),
data_aug_range=[1, 3, 5, 7])
X_val, Y_val = load_set(validation_data_dir,
target_size=(img_height, img_width))
# prepare the tensorboard
timestamp = time.time()
tbCallBack = TensorBoard(log_dir=log_dir + '/' + model_name + '/' +
str(int(timestamp)),
histogram_freq=0,
write_graph=True,
write_images=True)
# Save the model according to the conditions
file_name = models_dir + '/' + model_section + "_-{epoch:02d}-{val_loss:.2f}.h5"
checkpoint = ModelCheckpoint(file_name,
monitor='val_acc',
verbose=1,
save_best_only=False,
save_weights_only=False,
mode='auto',
period=4)
# early = EarlyStopping(monitor='val_acc', min_delta=0, patience=10, verbose=1, mode='auto')
metrics = custom_metrics.Metrics()
# Train the model
print('Training model %s and saving snapshot at %s' %
(model_section, file_name))
model_final.fit(
X_train,
Y_train,
batch_size=batch_size,
epochs=epochs,
validation_data=(X_val, Y_val),
verbose=1,
class_weight={
0: 1.,
1: positive_weight
},
callbacks=[
metrics,
tbCallBack,
checkpoint,
#early
])
def run_keras_compile_fit(model_dir,
strategy,
model_fn,
train_input_fn,
eval_input_fn,
loss_fn,
metric_fn,
init_checkpoint,
epochs,
steps_per_epoch,
eval_steps,
labels_list,
custom_callbacks=None):
"""Runs BERT classifier model using Keras compile/fit API."""
with strategy.scope():
training_dataset = train_input_fn()
evaluation_dataset = eval_input_fn()
bert_model, sub_model = model_fn()
optimizer = bert_model.optimizer
bert_model.summary()
if init_checkpoint:
checkpoint = tf.train.Checkpoint(model=sub_model)
checkpoint.restore(
init_checkpoint).assert_existing_objects_matched()
bert_model.compile(optimizer=optimizer,
loss=loss_fn,
metrics=[metric_fn()])
summary_dir = os.path.join(model_dir, 'summaries')
summary_callback = tf.keras.callbacks.TensorBoard(summary_dir)
checkpoint_path = os.path.join(model_dir, 'checkpoint')
checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(
checkpoint_path, save_weights_only=False)
custom_metric = custom_metrics.Metrics(labels_list,
valid_data=evaluation_dataset)
if custom_callbacks is not None:
custom_callbacks += [
custom_metric, summary_callback, checkpoint_callback
]
else:
custom_callbacks = [
custom_metric, summary_callback, checkpoint_callback
]
history = bert_model.fit(x=training_dataset,
validation_data=evaluation_dataset,
steps_per_epoch=steps_per_epoch,
epochs=epochs,
validation_steps=eval_steps,
callbacks=custom_callbacks)
return bert_model, history, custom_metric
def predict(models, config, model_section, bst):
models_para = []
for ms in model_section:
mp = ModelParams(ms)
mp.get_metadata(config)
models_para.append(mp)
#Read Basic Metadata from config file
train_data_dir, validation_data_dir, test_data_dir,\
results_dir, models_dir, log_dir = map(lambda x : x[1],
config.items("base"))
img_height = 256
img_width = 256
X_test, _, return_img_names = load_set(test_data_dir,
target_size=(img_height, img_width),
shuffle=False,
return_img_name=True)
metrics = custom_metrics.Metrics()
cnn_outputs_test = []
for model, mp in zip(models, models_para):
cnn_outputs_test.append(
model.predict(X_test, batch_size=mp.batch_size, verbose=1))
print(len(cnn_outputs_test))
np.save("/sharedfiles/outputs/features/cnn_features_test",
cnn_outputs_test)
forest_input_test = ft.reduce(lambda x, y: x + y,
map(lambda x: 0.5 * x, cnn_outputs_test))
#xgb_test = xgb.DMatrix(forest_input_test)
predictions = bst.predict_proba(forest_input_test)
print('Prediction done.')
print(predictions[:10])
preds = pd.DataFrame({'name': return_img_names, 'risk': predictions[:, 1]})
preds.to_csv('xgboost.csv')
def train(models, config, model_section):
models_param = []
for ms in model_section:
mp = ModelParams(ms)
mp.get_metadata(config)
models_param.append(mp)
#Read Basic Metadata from config file
train_data_dir, validation_data_dir, test_data_dir,\
results_dir, models_dir, log_dir = map(lambda x : x[1],
config.items("base"))
img_height = 256
img_width = 256
X_train, Y_train = load_set(train_data_dir,
target_size=(img_height, img_width),
data_aug_range=[1, 3, 5, 7])
X_val, Y_val = load_set(validation_data_dir,
target_size=(img_height, img_width))
metrics = custom_metrics.Metrics()
cnn_outputs_train = []
cnn_outputs_val = []
for model, mp in zip(models, models_param):
cnn_outputs_train.append(
model.predict(X_train, batch_size=mp.batch_size, verbose=1))
cnn_outputs_val.append(
model.predict(X_val, batch_size=mp.batch_size, verbose=1))
np.save("/sharedfiles/outputs/features/cnn_features_train",
cnn_outputs_train)
np.save("/sharedfiles/outputs/features/cnn_features_val", cnn_outputs_val)
forest_input_train = ft.reduce(lambda x, y: x + y,
map(lambda x: 0.5 * x, cnn_outputs_train))
forest_input_val = ft.reduce(lambda x, y: x + y,
map(lambda x: 0.5 * x, cnn_outputs_val))
#xgb_train = xgb.DMatrix(forest_input_train, Y_train)
#xgb_val = xgb.DMatrix(forest_input_val, Y_val)
param = {
'max_depth': 10,
'learning_rate': 0.1,
'n_estimators': 100,
'objective': 'binary:logistic',
'random_state': 42
}
xgb = XGBClassifier(**param)
num_round = 50
print('Fitting XGBi with params %s' % param)
xgb.fit(forest_input_train, Y_train[:, 1])
y_pp = xgb.predict_proba(forest_input_val)[:, 1]
y_p = xgb.predict(forest_input_val)
for func in [precision_score, recall_score, accuracy_score]:
print(func, func(Y_val[:, 1], y_p))
for func in [average_precision_score, roc_auc_score]:
print(func, func(Y_val[:, 1], y_pp))
return xgb
def run_keras_compile_fit(model_dir,
strategy,
model_fn,
train_input_fn,
eval_input_fn,
loss_fn,
metric_fn,
init_checkpoint,
epochs,
steps_per_epoch,
eval_steps,
labels_list,
custom_callbacks=None):
"""Runs BERT classifier model using Keras compile/fit API."""
"""
if FLAGS.mode == 'predict':
testing_dataset = test_input_fn()
bert_model, sub_model = model_fn()
optimizer = bert_model.optimizer
new_model = tf.keras.models.load_model(FLAGS.model_export_path,
custom_objects={"KerasLayer": sub_model,
"AdamWeightDecay": bert_model.optimizer,
"classification_loss_fn": loss_fn})
pre_ret = new_model.predict(testing_dataset)
logging.info('liran05:{}'.format(pre_ret))
return bert_model, None, None
"""
with strategy.scope():
training_dataset = train_input_fn()
evaluation_dataset = eval_input_fn()
bert_model, sub_model = model_fn()
optimizer = bert_model.optimizer
bert_model.summary()
if init_checkpoint:
checkpoint = tf.train.Checkpoint(model=sub_model)
checkpoint.restore(
init_checkpoint).assert_existing_objects_matched()
bert_model.compile(optimizer=optimizer,
loss=loss_fn,
metrics=[metric_fn()])
summary_dir = os.path.join(model_dir, 'summaries')
summary_callback = tf.keras.callbacks.TensorBoard(summary_dir)
checkpoint_path = os.path.join(model_dir, 'checkpoint')
checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(
checkpoint_path, save_weights_only=True)
#eval_data_list = list(evaluation_dataset.as_numpy_iterator())
custom_metric = custom_metrics.Metrics(labels_list,
valid_data=evaluation_dataset)
if custom_callbacks is not None:
custom_callbacks += [
custom_metric, summary_callback, checkpoint_callback
]
else:
custom_callbacks = [
custom_metric, summary_callback, checkpoint_callback
]
history = bert_model.fit(x=training_dataset,
validation_data=evaluation_dataset,
steps_per_epoch=steps_per_epoch,
epochs=epochs,
validation_steps=eval_steps,
callbacks=custom_callbacks)
return bert_model, history, custom_metric