def train_kfold(log_dir,
hparams,
model_name,
k,
state,
X,
Y,
X_test=None,
Y_test=None,
X_train_add=None,
Y_train_add=None,
batch_size=20,
epochs=200):
'''
X: The training set
X_test: The testing set
X_train_add: The additional set for model training, used for the 2nd experiment in paper.
'''
''' Training '''
# Log
timestamp = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S-%f')
model_dir = os.path.join(log_dir, timestamp, 'models')
os.makedirs(model_dir)
hist_dir = os.path.join(log_dir, timestamp, 'history')
os.makedirs(hist_dir)
eval_dir = os.path.join(log_dir, timestamp, 'evaluate')
os.makedirs(eval_dir)
params_savename = os.path.join(log_dir, timestamp, 'params.json')
summary_savename = os.path.join(log_dir, timestamp, 'summary.json')
test_hist_savename = os.path.join(log_dir, timestamp, 'test.json')
# Start training
kfold = StratifiedKFold(n_splits=k, shuffle=True, random_state=state)
fold = 1
accs = []
for train_index, val_index in kfold.split(X, Y):
model, params = compile_model(model_name, hparams)
if fold == 1:
print(model.summary())
print(params)
print('\n' + '=' * 60 + ' Fold: ' + str(fold) + ' ' + '=' * 60 + '\n')
# Callback functions
model_savename = os.path.join(model_dir,
'model{0}.h5'.format(str(fold)))
hist_savename = os.path.join(hist_dir,
'history{0}.json'.format(str(fold)))
val_savename = os.path.join(eval_dir,
'evaluate{0}.json'.format(str(fold)))
cb_list = [
callbacks.ModelCheckpoint(filepath=model_savename,
monitor='val_acc',
save_best_only=True),
callbacks.EarlyStopping(
monitor='acc',
patience=6,
)
]
# Add new training sets
try:
if X_train_add.any() and Y_train_add.any():
x_train = np.concatenate([X[train_index], X_train_add], axis=0)
y_train = np.concatenate([Y[train_index], Y_train_add], axis=0)
index = list(range(len(y_train)))
random.seed(state + 1)
random.shuffle(index)
x_train = x_train[index]
y_train = y_train[index]
except AttributeError:
x_train = X[train_index]
y_train = Y[train_index]
history = model.fit(x_train,
y_train,
validation_data=(X[val_index], Y[val_index]),
batch_size=batch_size,
epochs=epochs,
callbacks=cb_list,
verbose=2)
# Log
hist_dict = history.history
m = models.load_model(model_savename, custom_objects={'tf': tf})
val_dict = evaluate_model(m, X[val_index], Y[val_index])
accs.append(val_dict['accuracy'])
log_to_json(hist_dict, hist_savename)
log_to_json(val_dict, val_savename)
fold += 1
K.clear_session()
print('Session cleared.')
# Summary
try:
if X_test.any() and Y_test.any():
model_path = os.path.join(
model_dir, 'model{0}.h5'.format(accs.index(max(accs)) + 1))
m = models.load_model(model_path, custom_objects={'tf': tf})
test_dict = evaluate_model(m, X_test, Y_test)
log_to_json(test_dict, test_hist_savename)
except AttributeError:
pass
log_to_json(hparams, params_savename)
summary = summary_kfold(eval_dir)
print(summary)
log_to_json(summary, summary_savename)
def test(params):
# 加载预训练模型的tokenizer
tokenizer = BertTokenizer.from_pretrained(params["bert_pretrain_path"] +
'bert-base-chinese-vocab.txt')
test_batch = batcher(tokenizer, params, params["test_dataset_path"])
df_test = pd.read_csv(params["test_dataset_path"],
engine='python',
encoding="utf-8")
df_sub = df_test[['微博id']]
if params["use_cross_valid"]:
test_preds = []
for i in range(params["n_splits"]):
cross_pred = []
K.clear_session()
print("Building the model ...")
model = bert_classify(params)
print("Creating the checkpoint manager")
checkpoint_dir = "{}/cross_valid_checkpoint/ckpt_{}/checkpoint".format(
params["model_checkpoint_dir"], i)
ckpt = tf.train.Checkpoint(BertModel=model)
ckpt_manager = tf.train.CheckpointManager(ckpt,
checkpoint_dir,
max_to_keep=5)
ckpt.restore(ckpt_manager.latest_checkpoint)
if ckpt_manager.latest_checkpoint:
print("Restored from {}".format(
ckpt_manager.latest_checkpoint))
else:
print(checkpoint_dir)
raise Exception("No checkpoint files.")
for test_batcher in tqdm(test_batch):
pred = model(test_batcher[0])
cross_pred.append(pred)
# print(pred.shape)
cross_pred = tf.concat(cross_pred, axis=0)
test_preds.append(cross_pred)
test_preds = tf.stack(test_preds, axis=0)
sub = np.average(test_preds, axis=0)
sub = np.argmax(sub, axis=1)
df_sub['y'] = sub - 1
df_sub.columns = ['id', 'y']
df_sub.to_csv('test_sub.csv', index=False, encoding='utf-8')
else:
cross_pred = []
K.clear_session()
print("Building the model ...")
model = bert_classify(params)
print("Creating the checkpoint manager")
checkpoint_dir = "{}/split_valid_checkpoint/checkpoint".format(
params["model_checkpoint_dir"])
ckpt = tf.train.Checkpoint(BertModel=model)
ckpt_manager = tf.train.CheckpointManager(ckpt,
checkpoint_dir,
max_to_keep=5)
ckpt.restore(ckpt_manager.latest_checkpoint)
if ckpt_manager.latest_checkpoint:
print("Restored from {}".format(ckpt_manager.latest_checkpoint))
else:
print(checkpoint_dir)
raise Exception("No checkpoint files.")
for test_batcher in tqdm(test_batch):
pred = model(test_batcher[0])
cross_pred.append(pred)
cross_pred = tf.concat(cross_pred, axis=0)
sub = np.argmax(cross_pred, axis=1)
df_sub['y'] = sub - 1
df_sub.columns = ['id', 'y']
df_sub.to_csv('test_sub.csv', index=False, encoding='utf-8')
def main(dataset_name):
dataset = load_dataset()
raw_data = np.asarray(dataset['raw']['data'])
raw_label = np.asarray(dataset['raw']['label'])
num_classes = len(np.unique(raw_label))
rskf = RepeatedStratifiedKFold(n_splits=k_folds,
n_repeats=k_fold_reps,
random_state=42)
for fs_method, fs_range in fs_methods:
print('FS-Method : ', fs_method.__name__)
nfeats = []
accuracies = []
svc_accuracies = []
BAs = []
svc_BAs = []
mAPs = []
svc_mAPs = []
mus = []
name = dataset_name + '_mu_' + str(mu)
print(name)
for j, (train_index,
test_index) in enumerate(rskf.split(raw_data, raw_label)):
print('k_fold', j, 'of', k_folds * k_fold_reps)
train_data, train_labels = raw_data[train_index].copy(
), raw_label[train_index].copy()
test_data, test_labels = raw_data[test_index].copy(
), raw_label[test_index].copy()
train_labels = to_categorical(train_labels,
num_classes=num_classes)
test_labels = to_categorical(test_labels, num_classes=num_classes)
valid_features = np.where(np.abs(train_data).sum(axis=0) > 0)[0]
if len(valid_features) < train_data.shape[1]:
print('Removing', train_data.shape[1] - len(valid_features),
'zero features')
train_data = train_data[:, valid_features]
test_data = test_data[:, valid_features]
model_kwargs = {
# 'nclasses': num_classes,
'mu': mu / len(train_data),
'degree': 3
}
print('mu :', model_kwargs['mu'], ', batch_size :', batch_size)
svc_kwargs = {'C': 1.0, 'solver': 0.}
print('Starting feature selection')
best_fs = 0
best_value = None
for fs_value in fs_range:
fs_class = fs_method(10, fs_value, matlab_engine=matlab_engine)
fs_class.fit(train_data, 2. * train_labels[:, -1] - 1.)
svc_train_data = fs_class.transform(train_data)
norm = normalization_func()
svc_train_data_norm = norm.fit_transform(svc_train_data)
for s in [0, 1, 2, 3]:
for my_c in [
0.001, 0.01, 0.1, 0.5, 1.0, 1.4, 1.5, 1.6, 2.0,
2.5, 5.0, 25.0, 50.0, 100.0
]:
cmd = '-v 5 -s ' + str(s) + ' -c ' + str(my_c) + ' -q'
cv = liblinearutil.train(
(2 * train_labels[:, -1] - 1).tolist(),
svc_train_data_norm.tolist(), cmd)
if cv > best_fs:
best_fs = cv
best_value = fs_value
print('best fs_value: ', best_value)
fs_class = fs_method(200, best_value, matlab_engine=matlab_engine)
fs_class.fit(train_data, 2. * train_labels[:, -1] - 1.)
print('Finishing feature selection')
for i, n_features in enumerate([10, 50, 100, 150, 200]):
n_accuracies = []
n_svc_accuracies = []
n_BAs = []
n_svc_BAs = []
n_mAPs = []
n_svc_mAPs = []
n_train_accuracies = []
print('n_features : ', n_features)
fs_class.n_features_to_select = n_features
svc_train_data = fs_class.transform(train_data)
svc_test_data = fs_class.transform(test_data)
norm = normalization_func()
svc_train_data_norm = norm.fit_transform(svc_train_data)
svc_test_data_norm = norm.transform(svc_test_data)
bestcv = -1
bestc = None
bestSolver = None
for s in [0, 1, 2, 3]:
for my_c in [
0.001, 0.01, 0.1, 0.5, 1.0, 1.4, 1.5, 1.6, 2.0,
2.5, 5.0, 25.0, 50.0, 100.0
]:
cmd = '-v 5 -s ' + str(s) + ' -c ' + str(my_c) + ' -q'
cv = liblinearutil.train(
(2 * train_labels[:, -1] - 1).tolist(),
svc_train_data_norm.tolist(), cmd)
if cv > bestcv:
bestcv = cv
bestc = my_c
bestSolver = s
svc_kwargs['C'] = bestc
svc_kwargs['solver'] = bestSolver
print('Best -> C:', bestc, ', s:', bestSolver, ', acc:',
bestcv)
for r in range(reps):
model = train_SVC(svc_train_data_norm, train_labels,
svc_kwargs)
_, accuracy, test_pred = liblinearutil.predict(
(2 * test_labels[:, -1] - 1).tolist(),
svc_test_data_norm.tolist(), model, '-q')
test_pred = np.asarray(test_pred)
n_svc_accuracies.append(accuracy[0])
n_svc_BAs.append(balance_accuracy(test_labels, test_pred))
n_svc_mAPs.append(
average_precision_score(test_labels[:, -1], test_pred))
del model
model = train_Keras(svc_train_data, train_labels,
svc_test_data, test_labels,
model_kwargs)
train_data_norm = model.normalization.transform(
svc_train_data)
test_data_norm = model.normalization.transform(
svc_test_data)
test_pred = model.predict(test_data_norm)
n_BAs.append(balance_accuracy(test_labels, test_pred))
n_mAPs.append(
average_precision_score(test_labels[:, -1], test_pred))
n_accuracies.append(
model.evaluate(test_data_norm, test_labels,
verbose=0)[-1])
n_train_accuracies.append(
model.evaluate(train_data_norm,
train_labels,
verbose=0)[-1])
del model
K.clear_session()
print(
'n_features : ',
n_features,
', acc : ',
n_accuracies[-1],
', BA : ',
n_BAs[-1],
', mAP : ',
n_mAPs[-1],
', train_acc : ',
n_train_accuracies[-1],
', svc_acc : ',
n_svc_accuracies[-1],
', svc_BA : ',
n_svc_BAs[-1],
', svc_mAP : ',
n_svc_mAPs[-1],
)
if i >= len(accuracies):
accuracies.append(n_accuracies)
svc_accuracies.append(n_svc_accuracies)
BAs.append(n_BAs)
mAPs.append(n_mAPs)
svc_BAs.append(n_svc_BAs)
svc_mAPs.append(n_svc_mAPs)
nfeats.append(n_features)
mus.append(model_kwargs['mu'])
else:
accuracies[i] += n_accuracies
svc_accuracies[i] += n_svc_accuracies
BAs[i] += n_BAs
mAPs[i] += n_mAPs
svc_BAs[i] += n_svc_BAs
svc_mAPs[i] += n_svc_mAPs
output_filename = directory + 'LinearSVC_' + fs_method.__name__ + '.json'
if not os.path.isdir(directory):
os.makedirs(directory)
info_data = {
'reps': reps,
'classification': {
'mus': mus,
'n_features': nfeats,
'accuracy': accuracies,
'mean_accuracy': np.array(accuracies).mean(axis=1).tolist(),
'svc_accuracy': svc_accuracies,
'mean_svc_accuracy':
np.array(svc_accuracies).mean(axis=1).tolist(),
'BA': BAs,
'mean_BA': np.array(BAs).mean(axis=1).tolist(),
'mAP': mAPs,
'mean_mAP': np.array(mAPs).mean(axis=1).tolist(),
'svc_BA': svc_BAs,
'svc_mean_BA': np.array(svc_BAs).mean(axis=1).tolist(),
'svc_mAP': svc_mAPs,
'svc_mean_mAP': np.array(svc_mAPs).mean(axis=1).tolist(),
}
}
for k, v in info_data['classification'].items():
if 'mean' in k:
print(k, v)
with open(output_filename, 'w') as outfile:
json.dump(info_data, outfile)
def prediction(constants, model_name, dataset_folder, dataset_name, frame,
repeat_index, img_folder, save_path):
img_path = dataset_folder + dataset_name + img_folder
if constants.self_training_type is None:
save_path = save_path + '{}/frame{}_{}_repeat{}/'.format(
dataset_name, str(frame), model_name, str(repeat_index))
else:
save_path = save_path + '{}_{}/frame{}_repeat{}/'.format(
model_name, dataset_name, str(frame), str(repeat_index))
print('save_path:', save_path)
if os.path.isdir(save_path) == 0:
os.makedirs(save_path)
# ------------------- Data loading -------------------
a_strategy = constants.strategy_type
if 'TIRF' in dataset_name and 'specialist' in constants.strategy_type:
a_strategy = constants.strategy_type + '_normalize'
prediction_data_generator = DataGenerator(img_path,
frame,
128,
68,
a_strategy,
img_format=constants.img_format)
imgs_val, namelist, image_cols, image_rows, orig_cols, orig_rows = prediction_data_generator.get_expanded_whole_frames(
)
print('img size:', image_rows, image_cols)
print('orig img size:', orig_rows, orig_cols)
print('imgs_val: ', imgs_val.dtype, imgs_val.shape)
# ------------------- Load trained Model -------------------
weights_path = constants.get_trained_weights_path(str(frame), model_name,
str(repeat_index))
# print(debugger.check_loaded_weights(weights_path))
if "Res50V2" == str(constants.strategy_type):
model = ResNet50V2Keras(image_rows,
image_cols,
0,
image_cols - orig_cols,
image_rows - orig_rows,
weights_path=weights_path)
elif "Dense201" == str(constants.strategy_type):
model = DenseNet201Keras(image_rows,
image_cols,
0,
image_cols - orig_cols,
image_rows - orig_rows,
weights_path=weights_path)
elif "InceptionResV2" == str(constants.strategy_type):
model = InceptionResV2(image_rows,
image_cols,
0,
image_cols - orig_cols,
image_rows - orig_rows,
weights_path=weights_path)
elif "deeplabv3" == str(constants.strategy_type):
K.set_image_data_format('channels_last')
imgs_val = np.moveaxis(imgs_val, 1,
-1) # first channel to last channel
print(imgs_val.dtype, imgs_val.shape)
model = Deeplabv3(input_shape=(image_rows, image_cols, 3),
output_shape=(orig_rows, orig_cols))
model.load_weights(weights_path, by_name=True)
elif "VGG16_dropout" == str(constants.strategy_type):
model = VGG16_dropout(image_rows,
image_cols,
0,
image_cols - orig_cols,
image_rows - orig_rows,
weights_path=weights_path)
elif "VGG16_batchnorm" == str(constants.strategy_type):
model = VGG16_batchnorm(image_rows,
image_cols,
0,
image_cols - orig_cols,
image_rows - orig_rows,
weights_path=weights_path)
elif "VGG16_instancenorm" == str(constants.strategy_type):
model = VGG16_instancenorm(image_rows,
image_cols,
0,
image_cols - orig_cols,
image_rows - orig_rows,
weights_path=weights_path)
elif "movie3" in str(constants.strategy_type):
model = VGG16_movie(image_rows,
image_cols,
0,
image_cols - orig_cols,
image_rows - orig_rows,
weights_path=weights_path)
elif "VGG16_dac_input256" == constants.strategy_type:
model = VGG16_dac(image_rows,
image_cols,
0,
image_cols - orig_cols,
image_rows - orig_rows,
weights_path=weights_path)
elif "VGG16_spp_input256" == constants.strategy_type:
model = VGG16_spp(image_rows,
image_cols,
0,
image_cols - orig_cols,
image_rows - orig_rows,
weights_path=weights_path)
elif "VGG16" in str(constants.strategy_type):
model = VGG16(image_rows,
image_cols,
0,
image_cols - orig_cols,
image_rows - orig_rows,
weights_path=weights_path)
elif "VGG19_dropout_dac" in str(constants.strategy_type):
model = VGG19_dropout_dac(image_rows,
image_cols,
0,
image_cols - orig_cols,
image_rows - orig_rows,
weights_path=weights_path)
elif "VGG19_dropout_feature_extractor" in str(constants.strategy_type):
model = VGG19_dropout_feature_extractor(image_rows,
image_cols,
0,
image_cols - orig_cols,
image_rows - orig_rows,
weights_path=weights_path)
elif "VGG19_batchnorm_dropout" == str(constants.strategy_type):
model = VGG19_batchnorm_dropout(image_rows,
image_cols,
0,
image_cols - orig_cols,
image_rows - orig_rows,
weights_path=weights_path)
elif "VGG19_batchnorm" == str(constants.strategy_type):
model = VGG19_batchnorm(image_rows,
image_cols,
0,
image_cols - orig_cols,
image_rows - orig_rows,
weights_path=weights_path)
elif "VGG19_dropout" in str(constants.strategy_type):
model = VGG19_dropout(image_rows,
image_cols,
0,
image_cols - orig_cols,
image_rows - orig_rows,
weights_path=weights_path)
elif "VGG19" in str(constants.strategy_type):
model = VGG19(image_rows,
image_cols,
0,
image_cols - orig_cols,
image_rows - orig_rows,
weights_path=weights_path,
encoder_weights=None)
elif "EFF_B7" == str(
constants.strategy_type) or "EFF_B7_no_preprocessing" == str(
constants.strategy_type):
K.set_image_data_format('channels_last')
imgs_val = np.moveaxis(imgs_val, 1,
-1) # first channel to last channel
print(imgs_val.dtype, imgs_val.shape)
model = EFF_B7(image_rows,
image_cols,
0,
image_cols - orig_cols,
image_rows - orig_rows,
weights_path=weights_path)
elif "unet_feature_extractor" in str(constants.strategy_type):
model = UNet_feature_extractor(image_rows,
image_cols,
0,
image_cols - orig_cols,
image_rows - orig_rows,
weights_path=weights_path)
elif "unet" in str(constants.strategy_type):
model = UNet(image_rows,
image_cols,
0,
image_cols - orig_cols,
image_rows - orig_rows,
weights_path=weights_path)
print('model layers: ', len(model.layers))
plot_model(model,
to_file='model_plots/model_round{}_{}_predict.png'.format(
constants.round_num, constants.strategy_type),
show_shapes=True,
show_layer_names=True,
dpi=144)
# ------------------- predict segmented images and save them -------------------
if "feature_extractor" in str(constants.strategy_type):
segmented_output, style_output = model.predict(imgs_val,
batch_size=1,
verbose=1)
np.save(save_path + 'style_feature_vector.npy', style_output)
else:
segmented_output = model.predict(imgs_val, batch_size=1, verbose=1)
segmented_output = 255 * segmented_output # 0=black color and 255=white color
if "deeplabv3" == str(constants.strategy_type) or "EFF_B7" == str(
constants.strategy_type) or "EFF_B7_no_preprocessing" == str(
constants.strategy_type):
# move last channel to first channel
segmented_output = np.moveaxis(segmented_output, -1, 1)
print(segmented_output.shape)
for f in range(len(namelist)):
if constants.strategy_type == 'movie3' or constants.strategy_type == 'movie3_loss':
out = segmented_output[f, 1, :, :]
else:
out = segmented_output[f, 0, :, :]
cv2.imwrite(save_path + namelist[f], out)
K.clear_session()
sample_weight=None,
multioutput='uniform_average')
with open(SAVE_DIR + '/results_model_baseline_' + MODE + '.csv',
'a') as out_stream:
out_stream.write(
str(SEED) + ', ' + str(DATA_FILE[0:-4]) + ', ' + str(i) +
', ' + str(early_stopping_epoch) + ', ' +
str(all_trainable_count) + ', ' + str(acc_fold) + ', ' +
str(MAE) + ', ' + str(recall_fold) + ', ' + str(f1_fold) +
'\n')
print('Accuracy[{:.4f}] Recall[{:.4f}] F1[{:.4f}] at fold[{}]'.
format(acc_fold, recall_fold, f1_fold, i))
print('______________________________________________________')
K.clear_session()
ic_acc = st.t.interval(0.9,
len(avg_acc) - 1,
loc=np.mean(avg_acc),
scale=st.sem(avg_acc))
ic_recall = st.t.interval(0.9,
len(avg_recall) - 1,
loc=np.mean(avg_recall),
scale=st.sem(avg_recall))
ic_f1 = st.t.interval(0.9,
len(avg_f1) - 1,
loc=np.mean(avg_f1),
scale=st.sem(avg_f1))
print('Mean Accuracy[{:.4f}] IC [{:.4f}, {:.4f}]'.format(
def setUp(self):
tf.reset_default_graph()
K.clear_session()
self.previous_logging_level = logging.getLogger().level
logging.getLogger().setLevel(logging.ERROR)
def lav_test_case(
test: unittest.TestCase,
scenario: Type[ScenarioBase],
debug=False,
delta=1e-5,
batch_size_test=True,
ignore_binary_metric=False,
ignore_array_metric=False,
instantiate_graph=False,
args: AdditionalTrainerArgs = AdditionalTrainerArgs(),
):
with tempfile.TemporaryDirectory() as tmp_dir:
trainer_params = scenario.default_trainer_params()
trainer_params.output_dir = tmp_dir
trainer_params.epochs = 1
trainer_params.samples_per_epoch = 3
trainer_params.skip_model_load_test = True # not required in this test
trainer_params.force_eager = debug
trainer_params.export_best = True
trainer_params.export_final = True
trainer_params.write_checkpoints = False
trainer_params.random_seed = 324
trainer_params.scenario.data.pre_proc.run_parallel = False # Deterministic results!
args.apply(trainer_params)
batch_and_limit = 5
trainer = scenario.create_trainer(trainer_params)
trainer.train()
json_path = os.path.join(tmp_dir, "trainer_params.json")
with open(json_path) as f:
trainer_params_dict = json.load(f)
trainer_params_dict["epochs"] = 2
lav_params = scenario.lav_cls().params_cls()()
lav_params.print_samples = True
lav_params.model_path = os.path.join(trainer_params.output_dir, "export")
lav_params.pipeline = trainer_params.gen.setup.val
clear_session()
scenario_params = scenario.params_from_path(lav_params.model_path)
lav = scenario.create_lav(lav_params, scenario_params)
lav.run([trainer_params.gen.val_gen()], instantiate_graph=instantiate_graph)
clear_session()
set_global_random_seed(trainer_params.random_seed)
lav_params.model_path = os.path.join(trainer_params.output_dir, "best")
scenario_params = scenario.params_from_path(lav_params.model_path)
lav_params.pipeline.batch_size = 1
lav_params.pipeline.limit = batch_and_limit
lav = scenario.create_lav(lav_params, scenario_params)
bs1_results = next(
iter(lav.run([trainer_params.gen.val_gen()], run_eagerly=debug, instantiate_graph=instantiate_graph))
)
lav.benchmark_results.pretty_print()
if batch_size_test:
clear_session()
set_global_random_seed(trainer_params.random_seed)
lav_params.model_path = os.path.join(trainer_params.output_dir, "best")
scenario_params = scenario.params_from_path(lav_params.model_path)
lav_params.pipeline.batch_size = batch_and_limit
lav_params.pipeline.limit = batch_and_limit
lav = scenario.create_lav(lav_params, scenario_params)
bs5_results = next(
iter(lav.run([trainer_params.gen.val_gen()], run_eagerly=debug, instantiate_graph=instantiate_graph))
)
lav.benchmark_results.pretty_print()
assert_equality_dict(test, bs1_results, bs5_results, delta, ignore_binary_metric, ignore_array_metric)
def run(params):
args = Struct(**params)
set_seed(args.rng_seed)
ext = extension_from_parameters(args)
verify_path(args.save)
prefix = args.save + ext
logfile = args.logfile if args.logfile else prefix + '.log'
set_up_logger(logfile, args.verbose)
logger.info('Params: {}'.format(params))
loader = ComboDataLoader(seed=args.rng_seed,
val_split=args.validation_split,
cell_features=args.cell_features,
drug_features=args.drug_features,
response_url=args.response_url,
use_landmark_genes=args.use_landmark_genes,
preprocess_rnaseq=args.preprocess_rnaseq,
exclude_cells=args.exclude_cells,
exclude_drugs=args.exclude_drugs,
use_combo_score=args.use_combo_score,
cv_partition=args.cv_partition,
cv=args.cv)
# test_loader(loader)
# test_generator(loader)
train_gen = ComboDataGenerator(loader, batch_size=args.batch_size).flow()
val_gen = ComboDataGenerator(loader,
partition='val',
batch_size=args.batch_size).flow()
train_steps = int(loader.n_train / args.batch_size)
val_steps = int(loader.n_val / args.batch_size)
model = build_model(loader, args, verbose=True)
print('Creating model PNG')
from keras.utils import plot_model
plot_model(model, 'model_global_combo.png', show_shapes=True)
print('Model PNG has been created successfuly!')
model.summary()
# plot_model(model, to_file=prefix+'.model.png', show_shapes=True)
if args.cp:
model_json = model.to_json()
with open(prefix + '.model.json', 'w') as f:
print(model_json, file=f)
def warmup_scheduler(epoch):
lr = args.learning_rate or base_lr * args.batch_size / 100
if epoch <= 5:
K.set_value(model.optimizer.lr,
(base_lr * (5 - epoch) + lr * epoch) / 5)
logger.debug('Epoch {}: lr={}'.format(epoch,
K.get_value(model.optimizer.lr)))
return K.get_value(model.optimizer.lr)
df_pred_list = []
cv_ext = ''
cv = args.cv if args.cv > 1 else 1
fold = 0
while fold < cv:
if args.cv > 1:
logger.info('Cross validation fold {}/{}:'.format(fold + 1, cv))
cv_ext = '.cv{}'.format(fold + 1)
model = build_model(loader, args)
optimizer = optimizers.deserialize({
'class_name': args.optimizer,
'config': {}
})
base_lr = args.base_lr or K.get_value(optimizer.lr)
if args.learning_rate:
K.set_value(optimizer.lr, args.learning_rate)
model.compile(loss=args.loss, optimizer=optimizer, metrics=[mae, r2])
# calculate trainable and non-trainable params
params.update(candle.compute_trainable_params(model))
candle_monitor = candle.CandleRemoteMonitor(params=params)
timeout_monitor = candle.TerminateOnTimeOut(params['timeout'])
reduce_lr = ReduceLROnPlateau(monitor='val_loss',
factor=0.5,
patience=5,
min_lr=0.00001)
warmup_lr = LearningRateScheduler(warmup_scheduler)
checkpointer = ModelCheckpoint(prefix + cv_ext + '.weights.h5',
save_best_only=True,
save_weights_only=True)
tensorboard = TensorBoard(log_dir="tb/tb{}{}".format(ext, cv_ext))
history_logger = LoggingCallback(logger.debug)
model_recorder = ModelRecorder()
# callbacks = [history_logger, model_recorder]
callbacks = [
candle_monitor, timeout_monitor, history_logger, model_recorder
]
if args.reduce_lr:
callbacks.append(reduce_lr)
if args.warmup_lr:
callbacks.append(warmup_lr)
if args.cp:
callbacks.append(checkpointer)
if args.tb:
callbacks.append(tensorboard)
if args.gen:
history = model.fit_generator(train_gen,
train_steps,
epochs=args.epochs,
callbacks=callbacks,
validation_data=val_gen,
validation_steps=val_steps)
fold += 1
else:
if args.cv > 1:
x_train_list, y_train, x_val_list, y_val, df_train, df_val = loader.load_data_cv(
fold)
else:
x_train_list, y_train, x_val_list, y_val, df_train, df_val = loader.load_data(
)
y_shuf = np.random.permutation(y_val)
log_evaluation(evaluate_prediction(y_val, y_shuf),
description='Between random pairs in y_val:')
history = model.fit(x_train_list,
y_train,
batch_size=args.batch_size,
shuffle=args.shuffle,
epochs=args.epochs,
callbacks=callbacks,
validation_data=(x_val_list, y_val))
if args.cp:
model.load_weights(prefix + cv_ext + '.weights.h5')
if not args.gen:
y_val_pred = model.predict(x_val_list,
batch_size=args.batch_size).flatten()
scores = evaluate_prediction(y_val, y_val_pred)
if args.cv > 1 and scores[args.loss] > args.max_val_loss:
logger.warn(
'Best val_loss {} is greater than {}; retrain the model...'
.format(scores[args.loss], args.max_val_loss))
continue
else:
fold += 1
log_evaluation(scores)
df_val.is_copy = False
df_val['GROWTH_PRED'] = y_val_pred
df_val['GROWTH_ERROR'] = y_val_pred - y_val
df_pred_list.append(df_val)
if args.cp:
# model.save(prefix+'.model.h5')
model_recorder.best_model.save(prefix + '.model.h5')
# test reloadded model prediction
# new_model = keras.models.load_model(prefix+'.model.h5')
# new_model.load_weights(prefix+cv_ext+'.weights.h5')
# new_pred = new_model.predict(x_val_list, batch_size=args.batch_size).flatten()
# print('y_val:', y_val[:10])
# print('old_pred:', y_val_pred[:10])
# print('new_pred:', new_pred[:10])
plot_history(prefix, history, 'loss')
plot_history(prefix, history, 'r2')
if K.backend() == 'tensorflow':
K.clear_session()
if not args.gen:
pred_fname = prefix + '.predicted.growth.tsv'
if args.use_combo_score:
pred_fname = prefix + '.predicted.score.tsv'
df_pred = pd.concat(df_pred_list)
df_pred.to_csv(pred_fname, sep='\t', index=False, float_format='%.4g')
logger.handlers = []
return history
def main(_):
train_dir = os.path.join(FLAGS.image_dir, "train")
val_dir = os.path.join(FLAGS.image_dir, "val")
test_dir = os.path.join(FLAGS.image_dir, "test")
checkpoint_dir = os.path.join(os.path.dirname(FLAGS.model_dir),
"checkpoints")
checkpoint_path = os.path.join(checkpoint_dir, "checkpoint.pb")
os.makedirs(checkpoint_dir, exist_ok=True)
timestamp = time.strftime("%Y%m%d-%H%M%S")
tb_dir = os.path.join(FLAGS.summaries_dir, timestamp)
logger.info("\n===\tSetting up data loaders for train, val and test data.")
train_gen, val_gen, test_gen = create_data_feeders(
train_dir,
val_dir,
test_dir,
batch_size=FLAGS.train_batch_size,
image_size=FLAGS.image_size)
logger.info("\n===\tSaving key file with label names <> index conversion.")
save_labels_to_file(train_gen, FLAGS.output_labels)
logger.info(
"\n===\tCreating a classification model based on pre-trained weights.")
num_train_images = 2055
num_labels = 5
model = create_model(input_shape=(FLAGS.image_size, FLAGS.image_size, 3),
num_labels=num_labels,
learning_rate=FLAGS.learning_rate,
optimizer=FLAGS.optimizer_name)
logger.info("\n===\tInitial accuracy (before retraining):")
untrained_path = tf.contrib.saved_model.save_keras_model(
model, checkpoint_dir).decode("utf-8")
evaluate_model(untrained_path, test_gen)
logger.info("\n===\tPreparing Tensorboard callback, to monitor training.")
callbacks = create_callbacks(output_model_path=checkpoint_path,
summary_dir=tb_dir)
logger.info("\n===\tRetraining downloaded model.")
steps_per_epoch = num_train_images // FLAGS.train_batch_size
model.fit_generator(train_gen,
steps_per_epoch=steps_per_epoch,
epochs=FLAGS.how_many_training_steps //
steps_per_epoch,
validation_data=val_gen,
validation_steps=5,
callbacks=callbacks)
logger.info(
"\n===\tExporting the model so it can be served (TF Serving, TF Lite, etc.) to: {}"
.format(FLAGS.model_dir))
output_path = tf.contrib.saved_model.save_keras_model(
model, checkpoint_dir).decode("utf-8")
copy_tree(output_path, FLAGS.model_dir)
logger.info("\n===\tReporting final model accuracy:")
evaluate_model(FLAGS.model_dir, test_gen)
K.clear_session()
def _test_model(self, model, input_shape):
batch_size = 1000
epochs = 1
steps_per_epoch = 125
k_lip_data = 2.0
# clear session to avoid side effects from previous train
K.clear_session()
# create the keras model, defin opt, and compile it
optimizer = Adam(lr=0.001)
model.compile(
optimizer=optimizer, loss="mean_squared_error", metrics=[metrics.mse]
)
# model.summary()
# create the synthetic data generator
output_shape = model.compute_output_shape((batch_size,) + input_shape)[1:]
kernel = build_kernel(input_shape, output_shape, k_lip_data)
# define logging features
logdir = os.path.join(
"logs", "lip_layers", "condense_test"
) # , datetime.now().strftime("%Y_%m_%d-%H_%M_%S")))
callback_list = [callbacks.TensorBoard(logdir)]
# train model
model.__getattribute__(FIT)(
linear_generator(batch_size, input_shape, kernel),
steps_per_epoch=steps_per_epoch,
epochs=epochs,
verbose=0,
callbacks=callback_list,
)
# the seed is set to compare all models with the same data
np.random.seed(42)
# get original results
loss, mse = model.__getattribute__(EVALUATE)(
linear_generator(batch_size, input_shape, kernel),
steps=10,
verbose=0,
)
# generate vanilla
vanilla_model = model.vanilla_export()
vanilla_model.compile(
optimizer=optimizer, loss="mean_squared_error", metrics=[metrics.mse]
)
np.random.seed(42)
# evaluate vanilla
loss2, mse2 = model.__getattribute__(EVALUATE)(
linear_generator(batch_size, input_shape, kernel),
steps=10,
verbose=0,
)
np.random.seed(42)
# check if original has changed
vanilla_loss, vanilla_mse = vanilla_model.__getattribute__(EVALUATE)(
linear_generator(batch_size, input_shape, kernel),
steps=10,
verbose=0,
)
model.summary()
vanilla_model.summary()
self.assertAlmostEqual(
mse,
vanilla_mse,
4,
"the exported vanilla model must have same behaviour as original",
)
self.assertAlmostEqual(
mse, mse2, 4, "exporting a model must not change original model"
)
# add one epoch to orginal
model.__getattribute__(FIT)(
linear_generator(batch_size, input_shape, kernel),
steps_per_epoch=steps_per_epoch,
epochs=1,
verbose=0,
callbacks=callback_list,
)
np.random.seed(42)
loss3, mse3 = model.__getattribute__(EVALUATE)(
linear_generator(batch_size, input_shape, kernel),
steps=10,
verbose=0,
)
# check if vanilla has changed
np.random.seed(42)
vanilla_loss2, vanilla_mse2 = vanilla_model.__getattribute__(EVALUATE)(
linear_generator(batch_size, input_shape, kernel),
steps=10,
verbose=0,
)
self.assertAlmostEqual(
vanilla_mse,
vanilla_mse2,
4,
"exported model must be completely independent from original",
)
self.assertNotAlmostEqual(
mse,
mse3,
4,
"all tests passe but integrity check failed: the test cannot conclude that "
+ "vanilla_export create a distinct model",
)
def main():
parser = argparse.ArgumentParser(description='Evaluate genesis and cam')
parser.add_argument('--json', type=str, default="../data/cross_val.json")
parser.add_argument('--genesis_weights',
type=str,
default='video_genesis_lr1e4')
parser.add_argument('--frame_model_weights',
type=str,
default='trained_models_cam')
parser.add_argument('--frame_model_id', type=str, default='vgg_cam')
parser.add_argument('--videos',
type=str,
default='../data/pocus_videos/convex')
parser.add_argument('--output_dir',
type=str,
default='evaluation_outputs.dat')
args = parser.parse_args()
K.clear_session()
print("---------------------------------")
print("WARNING: THIS SCRIPT MUST BE RUN ON A GPU")
print("---------------------------------")
with open(args.json, "r") as infile:
cross_val_split = json.load(infile)
VIDEO_DIR = args.videos
all_genesis_preds = []
all_frame_preds = []
for i in range(5):
gen_eval = GenesisEvaluator(weights_dir=args.genesis_weights,
ensemble=False,
split=i)
K.set_image_data_format("channels_last")
normal_eval = VideoEvaluator(weights_dir=args.frame_model_weights,
ensemble=False,
split=i,
model_id=args.frame_model_id,
num_classes=4)
files = cross_val_split[str(i)]["test"][0]
# print(files)
for f in files:
print("evaluate", f)
# TEST if the video is working
vid3d = Videoto3D("", 64, 64, 5, 5)
vid3d.max_vid = {"cov": 20, "pne": 20, "reg": 20}
X_test, _, fn = vid3d.video3d([os.path.join(VIDEO_DIR, f)],
["cov"])
if len(np.unique(fn)) != 1:
print("ERROR: WRONG FILE!")
print(fn)
print(X_test.shape)
continue
# run genesis model
K.set_image_data_format("channels_first")
preds = gen_eval(os.path.join(VIDEO_DIR, f))
vid_pred_genesis = np.argmax(np.mean(preds, axis=(0, 1)))
all_genesis_preds.append(preds)
# run cam model
K.set_image_data_format("channels_last")
preds_framebased = normal_eval(os.path.join(VIDEO_DIR, f))
frame_pred = np.argmax(np.mean(preds_framebased, axis=0))
all_frame_preds.append(preds_framebased)
print(preds.shape, preds_framebased.shape)
print("genesis pred", vid_pred_genesis, "frame based pred",
frame_pred)
print("-------------")
with open(args.output_dir, "wb") as outfile:
pickle.dump((all_genesis_preds, all_frame_preds), outfile)
def save(self, path):
self.model.save(path)
K.clear_session()
def modify_search_args(args: Namespace):
"""
Modifies and validates searching arguments in place.
:param args: Arguments.
"""
# TODO : Take glob_mod_name as input argument
glob_mod_name = 'g1'
local_mod_no = glob_mod_name[-1]
#Load Global Model
print("--------< LOADING GLOBAL MODEL >----------")
global_model = load_model(os.path.join(args.global_model_path + '/'))
print('Global model loaded successfully!')
print("-" * 62)
# initial list to collect local model weights after scalling
scaled_local_weight_list = list()
print("--------< LOADING CLIENT MODELS >----------")
for subdir, dirs, files in os.walk(args.client_dir):
if str(subdir)[-2:] == 'l{}'.format(local_mod_no):
# TODO - sampleNo/Total sample - write code below, using metadataof each client
# for file in files:
# if 'metadata' in str(file):
# with open(os.path.join(subdir, file)) as f:
# m_data = json.load(f)
# print(os.path.join(subdir, file))
# scale the model weights and add to list
#TODO scaling factor is hard coded at the moment
# scaling_factor = weight_scalling_factor(clients_batched, client)
#Loading local model
local_model = load_model(os.path.join(subdir + '/'))
# scale the model weights and add to list
scaling_factor = 0.2
scaled_weights = scale_model_weights(local_model.get_weights(),
scaling_factor)
scaled_local_weight_list.append(scaled_weights)
# clear session to free memory after each communication round
K.clear_session()
print("--------< RUNNING FEDERATED AVERAGING ALGORITHM... >----------")
# to get the average over all the local model, we simply take the sum of the scaled weights
average_weights = sum_scaled_weights(scaled_local_weight_list)
# update global model
global_model.set_weights(average_weights)
#Load test data
test_batched = load_data(args.test_dataset_path)
#NOTE: no. of keys in global model metadata file = rounds done till now
try:
with open(args.metadata_path + '/metrics.json') as f:
glob_mod_metadata = json.load(f)
comm_round = len(glob_mod_metadata.keys()) + 1
except:
glob_mod_metadata = dict()
comm_round = 1
#test global model and print out metrics after each communications round
for (X_test, Y_test) in test_batched:
global_acc, global_loss = test_model(X_test, Y_test, global_model,
comm_round)
#Save metrics in dict
glob_mod_metadata['round_{}_results'.format(comm_round)] = {
'global_acc': global_acc,
'global_loss': global_loss
}
#Save metrics in metadata
with open(args.metadata_path + '/metrics.json', 'w') as f:
json.dump(glob_mod_metadata, f, indent=4)
#Save hyperparameter / config in dict
glob_mod_config = dict()
glob_mod_config['round_{}_config'.format(comm_round)] = {
'global_acc': global_model.to_json()
}
with open(args.global_model_path + '/global_mod_config.json', 'w') as f:
json.dump(glob_mod_config, f, indent=4)
# log setup
current_script_name = os.path.basename(__file__).split('.')[0]
log_path_filename = ''.join([local_submodule_settings['log_path'], current_script_name, '.log'])
logging.basicConfig(filename=log_path_filename, level=logging.DEBUG,
format='%(asctime)s %(levelname)s %(name)s %(message)s')
logger = logging.getLogger(__name__)
logHandler = handlers.RotatingFileHandler(log_path_filename, maxBytes=10485760, backupCount=5)
logger.addHandler(logHandler)
# set random seed for reproducibility --> done in _2_train.py module
np.random.seed(42)
tf.random.set_seed(42)
# clear session
kb.clear_session()
# functions definitions
def general_mean_scaler(local_array):
if len(local_array) == 0:
return "argument length 0"
mean_local_array = np.mean(local_array, axis=1)
mean_scaling = np.divide(local_array, 1 + mean_local_array)
return mean_scaling, mean_local_array
def window_based_normalizer(local_window_array):
if len(local_window_array) == 0:
return "argument length 0"
def main(args):
myseed = None if args.seed == 0 else args.seed
p1 = args.p1
lr = args.lr
W = 1
batchsize = args.batchsize
maxepochs = args.maxepochs
initializer = args.initializer
activation = args.activation
masktype = args.masktype
train_weights = str2bool(args.trainweights)
train_masks = str2bool(args.trainmasks)
alpha = args.alpha
trainingtype = "" + "TrainWeights" * train_weights + "TrainMasks" * train_masks
outputpath = args.outputpath + "/" + trainingtype
data = None
network = None
if "Conv" in args.nettype:
csize = int(args.nettype[-1])
# Pre-calculated W scaling factor (depends on the architecture)
# See https://arxiv.org/abs/2006.16627 for how to calculate W
if initializer == "binary":
if csize == 6:
W = 8.344820201940066e-12
if csize == 4:
W = 4.806616356300754e-09
if csize == 2:
W = 1.384305440187043e-06
data = utils.SetMyData("CIFAR", W)
outputpath += "/Conv" + str(csize)
network = PrepareConvolutional(csize, data, myseed, initializer,
activation, masktype, train_weights,
train_masks, p1, alpha)
if args.nettype == "LeNet":
if initializer == "binary":
W = 0.0005942073791483592
data = utils.SetMyData("MNIST", W)
outputpath += "/LeNet"
network = PrepareMaskedMLP(data, myseed, initializer, activation,
masktype, train_weights, train_masks, p1,
alpha)
outputpath += "/P1_" + str(p1) + "/alpha_" + str(alpha)
outputpath += "/" + masktype + "_" + activation + "_" + initializer + "_LR" + str(
lr)
outputpath += "/" + "bs_" + str(batchsize) + "/"
network.compile(loss='categorical_crossentropy',
optimizer=tf.keras.optimizers.Adam(lr=lr),
metrics=['accuracy'])
network.summary()
NetworkTrainer(network, data, outputpath, batchsize, maxepochs)
kb.clear_session()
return
def test_tuners(tuner, data, y, args):
import os
from hypermodel_utils import load_partition, get_callbacks, get_results_table, allocate_stats, mlflow_logs
import tensorflow.keras.callbacks as C
MODEL_TYPE = args.model_type
TIMESTAMP = args.timestamp
samples = args.samples
X = [x for x in data.X]
FIT_MAX_EPOCHS = args.epochs
# Get the best hyperparameters from the search
best_tuner_mcc = -1000.0
best_tuner_idx = 0
tuners = tuner.oracle.get_best_trials(num_trials=1)
results = list()
for idx, t in enumerate(tuners):
# loss = t.metrics.get_best_value('loss')
# acc = t.metrics.get_best_value('accuracy')
# print('- ' * 30)
# print('{}\t{}'.format(loss, acc))
# print(t.hyperparameters.values)
# Build the model using the best hyperparameters
# params = tuner.get_best_hyperparameters()[idx]
params = t.hyperparameters
print('PARAMS:', params)
model = tuner.hypermodel.build(params)
model.summary()
kf = StratifiedShuffleSplit(n_splits=args.cv,
random_state=args.samples[0])
kf.get_n_splits(X[0], y)
cv_partition = 0
results.append(get_results_table())
best_weights = [None for _ in range(args.cv)]
best_stats = [None for _ in range(args.cv)]
# Cross validation loop
for cv_train_index, cv_test_index in kf.split(X[0], y):
cv_partition += 1
(x_train, y_train), (x_test, y_test) = data.load_partition(
cv_train_index, cv_test_index)
callbacks = get_callbacks(args, cv_partition)
best_cv_mcc = -10000.0
best_cv_stats = None
# Validation samples loop
for s, seed in enumerate(samples):
print('TUNER {} - CV {} - {} Train on SEED {}'.format(
idx, cv_partition, s, seed))
weight_file_name = '{}-{}-partition_{}-seed_{}'.format(
MODEL_TYPE, TIMESTAMP, cv_partition, s)
weight_file_name += '-epoch_{epoch:02d}.hdf5'
weight_path = os.path.join(args.weights_dir, weight_file_name)
callbacks[0] = C.ModelCheckpoint(weight_path,
save_best_only=True,
save_weights_only=True,
verbose=1)
kf = StratifiedShuffleSplit(n_splits=1,
random_state=seed,
test_size=0.05)
kf.get_n_splits(x_train, y_train)
for t_index, v_index in kf.split(x_train[0], y_train):
(xx_train, yy_train), val_data = data.load_partition(
t_index, v_index)
model.fit(x=xx_train,
y=yy_train,
batch_size=args.batch_size,
epochs=FIT_MAX_EPOCHS,
validation_data=val_data,
callbacks=callbacks,
class_weight={
0: .2,
1: .8
},
verbose=0)
# # Data Augmentation =================================================
# train_datagen = AugmentedGeneratorMultipleInputs(xx_train, yy_train, args.batch_size)
# _steps_per_epoch = int(len(yy_train) / args.batch_size)
# print("_steps_per_epoch", _steps_per_epoch)
# # model.fit_generator(train_datagen.flow(xx_train, yy_train, batch_size=args.batch_size),
# model.fit_generator(train_datagen,
# validation_data=val_data,
# steps_per_epoch=_steps_per_epoch,
# epochs=FIT_MAX_EPOCHS,
# callbacks=callbacks,
# verbose=1)
# # ==================================================================
K.clear_session()
# # Test model fitted using seed validation set
# stats, y_pred = get_test_stats(model, x_test, y_test)
# print(stats.Mcc)
# print(stats.F1)
cv_idx = cv_partition - 1
best_stats[cv_idx], best_weights[cv_idx] = do_partition_test(
model, cv_partition, args, test_data=(x_test, y_test))
results[idx] = allocate_stats(results[idx], best_stats[cv_idx],
cv_partition)
# Persist weights files
cv_mean_mcc = np.mean(results[idx]['mcc'])
tuner_folder_name = '{}-{}-tuner_{}-mcc_{:.4f}'.format(
MODEL_TYPE, TIMESTAMP, idx, cv_mean_mcc)
folder_name = os.path.join(args.best_weights_dir, tuner_folder_name)
if not os.path.exists(folder_name):
os.makedirs(folder_name)
for cv in range(args.cv):
weight_name = '{}-{}-partition_{}-mcc_{:.4f}.hdf5'.format(
MODEL_TYPE, TIMESTAMP, cv, results[idx]['mcc'][cv])
weight_path = os.path.join(folder_name, weight_name)
model.set_weights(best_weights[cv])
model.save_weights(weight_path)
for idx, t in enumerate(tuners):
print('EVAL TUNER {} - {}'.format(idx, np.mean(results[idx]['mcc'])))
model = tuner.hypermodel.build(t.hyperparameters)
mlflow_logs(args, t.hyperparameters.values, results[idx], model, idx)
if __name__ == '__main__':
from itertools import product
import numpy as np
import random
from time import time
t0 = time()
expr_out_dir = "/media/wheatley/38882E5E882E1AC0/Deep_Face_Verifier/experiments"
logs_out_dir = "/media/wheatley/38882E5E882E1AC0/Deep_Face_Verifier/tensorboard_logs"
extracted_dir = "extracted"
models_types = ["eucl", "cos"]
extract_layers = np.arange(3)
combination_funcs = np.arange(4)
n_neurons = [16, 1024]
n_layers = np.arange(2)
combs = list(product(models_types, extract_layers, combination_funcs, n_neurons, n_layers))
random.shuffle(combs)
for m_type, ext_lay, comb_f, n_neur, n_lay in combs:
n_neur = 0 if (n_lay == 0) else n_neur
exp_name = "{}_outLay:{}_combFun:{}_nNeur:{}_nLay:{}".format(m_type, ext_lay, comb_f, n_neur, n_lay)
if not os.path.isdir(os.path.join(expr_out_dir, exp_name)):
clear_session()
input_size = 2622 if ext_lay == 2 else 4096
model = build_model.build_eucl_model(input_size, comb_f, n_neur, n_lay, True) if m_type == "eucl" else build_model.build_cos_model(input_size, comb_f, n_neur, n_lay, True)
run_experiment(model, os.path.join(extracted_dir, "layer_{}".format(ext_lay)), exp_name,
experiments_dir=expr_out_dir, tensorboard_logs_dir=logs_out_dir)
print("Total Time:", time() - t0)
def train_test(model, data, y, args):
import os
import tensorflow.keras.backend as bk
import tensorflow as tf
from hypermodel_utils import load_partition, get_callbacks, get_results_table, allocate_stats, mlflow_logs
MODEL_TYPE = args.model_type
TIMESTAMP = args.timestamp
samples = args.samples
X = [x for x in data.X]
FIT_MAX_EPOCHS = args.epochs
results = list()
model.summary()
kf = StratifiedShuffleSplit(n_splits=args.cv, random_state=args.samples[0])
kf.get_n_splits(X[0], y)
results.append(get_results_table())
best_weights = [None for _ in range(args.cv)]
best_stats = [None for _ in range(args.cv)]
metrics_stats = {
m: {'p{:02d}'.format(x): []
for x in range(args.cv)}
for m in ('Prec', 'Sn', 'Sp', 'Acc', 'F1', 'Mcc')
}
weights_sufix = '.hdf5'
# Cross validation loop
for idx, (cv_train_index, cv_test_index) in enumerate(kf.split(X[0], y)):
# TODO split nucleotides strings, not matrix
(x_train, y_train), (x_test, y_test) = data.load_partition(
cv_train_index, cv_test_index)
callbacks = get_callbacks(args, idx)
best_cv_mcc = -10000.0
# === Perform Training Phase ===
# Iterate over samples to create different samples on same partition based on different train-val splits
for s, seed in enumerate(samples):
print('CV PARTITION {} - CV SEED {} | Train on SEED {}'.format(
idx, s, seed))
weight_file_prefix = '{}-{}-partition_{:02d}'.format(
MODEL_TYPE, TIMESTAMP, idx)
weight_file_name = weight_file_prefix + '-sample_{:02d}'.format(s)
weight_file_name += '-epoch_{epoch:02d}.hdf5'
weight_path = os.path.join(args.weights_dir, weight_file_name)
callbacks[0] = tf.keras.callbacks.ModelCheckpoint(
weight_path,
save_best_only=True,
save_weights_only=True,
verbose=1)
kf = StratifiedShuffleSplit(n_splits=1,
random_state=seed,
test_size=0.05)
kf.get_n_splits(x_train, y_train)
histories = []
for t_index, v_index in kf.split(x_train[0], y_train):
(xx_train,
yy_train), val_data = data.load_partition(t_index, v_index)
history = model.fit(x=xx_train,
y=yy_train,
batch_size=args.batch_size,
epochs=FIT_MAX_EPOCHS,
validation_data=val_data,
callbacks=callbacks,
class_weight={
0: .2,
1: .8
},
verbose=0)
# PLOT HISTORY
_plot_history(args, 'accuracy', history, idx, s)
_plot_history(args, 'loss', history, idx, s)
# # Data Augmentation =================================================
# train_datagen = AugmentedGeneratorMultipleInputs(xx_train, yy_train, args.batch_size)
# _steps_per_epoch = int(len(yy_train) / args.batch_size)
# print("_steps_per_epoch", _steps_per_epoch)
# # model.fit_generator(train_datagen.flow(xx_train, yy_train, batch_size=args.batch_size),
# model.fit_generator(train_datagen,
# validation_data=val_data,
# steps_per_epoch=_steps_per_epoch,
# epochs=FIT_MAX_EPOCHS,
# callbacks=callbacks,
# verbose=1)
# # ==================================================================
# # Test model fitted using seed validation set
# stats, y_pred = get_test_stats(model, x_test, y_test)
# print(stats.Mcc)
# print(stats.F1)
# === Perform Test Phase ===
sample_weights = [
x for x in os.listdir(args.weights_dir + os.path.sep) if
x.startswith(weight_file_prefix) and x.endswith(weights_sufix)
]
# Iterate over all weights saved on checkpoints for this sample of this partition
for i, f in enumerate(sample_weights):
best_sample_mcc = -10000.0
best_sample_stats = None
bk.clear_session()
name = os.path.join(args.weights_dir, f)
# print(name)
model.load_weights(name)
stats, y_pred = get_test_stats(model=model, X=x_test, y=y_test)
# Select best sample weights
if best_sample_mcc < stats.Mcc:
best_sample_mcc = stats.Mcc
best_sample_stats = stats
# Select best weigths for this partition
if best_cv_mcc < stats.Mcc:
best_cv_mcc = stats.Mcc
selected_weight = f
best_stats[idx] = stats
# print(stats)
for metric in best_sample_stats.get_stats_types():
sample_stats = best_sample_stats.to_dict()
metrics_stats[metric]['p{:02}'.format(idx)].append(
sample_stats[metric])
# Delete temporary weights
for i, f in enumerate(sample_weights):
if f != selected_weight:
# print('Deleting weight: {}'.format(f))
path = args.weights_dir + '/' + f
os.remove(path)
bk.clear_session()
print(best_sample_stats)
# Persist best weights of this partition on logs
args.logs.set_metrics(**metrics_stats) # Log metrics
args.logs.set_artifacts() # Log artifacts
model.load_weights(os.path.join(args.weights_dir, selected_weight))
persist_model_path = os.path.join(
args.best_weights_dir,
'model_{}_{}_p{:02d}'.format(MODEL_TYPE, TIMESTAMP, idx))
# mlflow_keras.save_model(model, persist_model_path)
# mlflow_keras.log_model(model, args.logs.get_model_path())
mlflow_keras.save_model(model, args.logs.get_model_path(idx))
mlflow_keras.log_model(model, 'models')
# print('Deleting weight: {}'.format(selected_weight))
path = args.weights_dir + '/' + selected_weight
os.remove(path)
def reset():
K.clear_session()
def fit(self,
trainX,
trainy,
timey,
valX=None,
valy=None,
use_pretrained=False):
"""
Fit the model to training data
"""
start_time = time.time()
# clear memory
K.clear_session()
# allocate lists for the ensemble
self.ensemble = []
self.history = []
self.val_loss = []
self.segment_len = trainX.shape[0] // self.hyperparameters['n_segments']
if self.hyperparameters['n_segments'] == 1 and (valX is not None
or valy is not None):
warnings.warn(
"Validation and test data set are the same if n_segements is 1!"
)
i = 0
while i < self.hyperparameters['n_members_segment']:
j = 0
while j < self.hyperparameters['n_segments']:
print("build")
ensemble_member = self.build_model(trainX.shape[1])
n_ens_sel = len(self.ensemble)
small_print_header(
f"Train member Nr {n_ens_sel+1}/{self.hyperparameters['n_members']}"
)
if use_pretrained:
ensemble_member.load_weights(self.pretrained_weights)
print("compile")
ensemble_member.compile(loss=self.loss,
optimizer=self.optimizer,
metrics=[self.loss])
# validate on the spare segment
if self.hyperparameters['n_segments'] != 1:
if valX is not None or valy is not None:
warnings.warn(
"Validation data set will be one of the segments. The provided validation data set is not used!"
)
start_ind = j * self.segment_len
end_ind = (j + 1) * self.segment_len
trainXens = np.delete(trainX,
np.s_[start_ind:end_ind],
axis=0)
trainyens = np.delete(trainy, np.s_[start_ind:end_ind])
# upsample el nino like periods
if False:
timeyens = np.delete(timey, np.s_[start_ind:end_ind])
elninolike = (timeyens >= f'1982-01-01') & (
timeyens <= f'2001-12-01')
laninalike = np.invert(elninolike)
trainXens_nino, trainyens_nino = trainXens[
elninolike], trainyens[elninolike]
trainXens_nina, trainyens_nina = trainXens[
laninalike], trainyens[laninalike]
nino_choices = np.random.choice(np.arange(
len(trainyens_nino)),
size=1000)
nina_choices = np.random.choice(np.arange(
len(trainyens_nina)),
size=1000)
trainXens = np.concatenate(
(trainXens_nino[nino_choices],
trainXens_nina[nina_choices]))
trainyens = np.concatenate(
(trainyens_nino[nino_choices],
trainyens_nina[nina_choices]))
valXens = trainX[start_ind:end_ind]
valyens = trainy[start_ind:end_ind]
if False:
valXens, trainXens = trainXens, valXens
valyens, trainyens = trainyens, valyens
# validate on test data set
elif self.hyperparameters['n_segments'] == 1:
if valX is None or valy is None:
raise MissingArgumentError(
"When segments length is 1, a validation data set must be provided."
)
trainXens = trainX
trainyens = trainy
valXens = valX
valyens = valy
history = ensemble_member.fit(
trainXens,
trainyens,
epochs=self.hyperparameters['epochs'],
batch_size=self.hyperparameters['batch_size'],
verbose=self.hyperparameters['verbose'],
shuffle=True,
callbacks=[self.es],
validation_data=(valXens, valyens))
self.history.append(history)
self.val_loss.append(
ensemble_member.evaluate(valXens, valyens)[1])
self.ensemble.append(ensemble_member)
j += 1
i += 1
self.mean_val_loss = np.mean(self.val_loss)
print(f'Loss: {self.mean_val_loss}')
# print computation time
end_time = time.time()
passed_time = np.round(end_time - start_time, decimals=1)
print(f'Computation time: {passed_time}s')
def make_image_classifier(tfhub_module,
image_dir,
hparams,
requested_image_size=None,
saveModelDir=False):
"""Builds and trains a TensorFLow model for image classification.
Args:
tfhub_module: A Python string with the handle of the Hub module.
image_dir: A Python string naming a directory with subdirectories of images,
one per class.
hparams: A HParams object with hyperparameters controlling the training.
requested_image_size: A Python integer controlling the size of images to
feed into the Hub module. If the module has a fixed input size, this
must be omitted or set to that same value.
"""
print("Using hparams:")
for key, value in hparams._asdict().items():
print("\t{0} : {1}".format(key, value))
module_layer = hub.KerasLayer(tfhub_module,
trainable=hparams.do_fine_tuning)
image_size = _image_size_for_module(module_layer, requested_image_size)
print("Using module {} with image size {}".format(tfhub_module,
image_size))
train_data_and_size, valid_data_and_size, labels = _get_data_with_keras(
image_dir, image_size, hparams.batch_size, hparams.validation_split,
hparams.do_data_augmentation)
print("Found", len(labels), "classes:", ", ".join(labels))
model = build_model(module_layer, hparams, image_size, len(labels))
# If we are fine-tuning, check and see if weights
# already exists at the output directory. This way, a user
# can simply run two consecutive training sessions. One without
# fine-tuning, followed by another with.
if hparams.do_fine_tuning:
if saveModelDir is not None:
existingWeightsPath = os.path.join(saveModelDir,
"saved_model_weights.h5")
if os.path.exists(existingWeightsPath):
print("Loading existing weights for fine-tuning")
model.load_weights(existingWeightsPath)
train_result = train_model(model, hparams, train_data_and_size,
valid_data_and_size)
# Tear down model, set training to 0 and then re-create.
# 1 - Save model weights as Keras H5.
tempDir = tempfile.gettempdir()
tempModelWeightsFile = os.path.join(tempDir, "weights.h5")
model.save_weights(tempModelWeightsFile)
# 2 - Set training to 0
K.clear_session()
K.set_learning_phase(0)
# 3 - Create model again
model = build_model(module_layer, hparams, image_size, len(labels))
# 4 - Load model weights.
model.load_weights(tempModelWeightsFile)
# Clean up temp weights file
os.remove(tempModelWeightsFile)
# 5 - Pass model to lib.model_to_frozen_graph.
frozen_inference_graph = model_to_frozen_graph(model)
return model, labels, train_result, frozen_inference_graph
def __del__(self):
K.clear_session()
def predict_traffic_file():
#K.clear_session()
#BELOW docstring lines are required to support swagger documentation
""" Endpoint returning traffic sign image prediction
---
parameters:
- name: input_file
in: formData
type: file
required: true
"""
# Get the input file from the http request
#read image file string data
filestr = request.files['input_file'].read()
#convert string data to numpy array
npimg = np.fromstring(filestr, np.uint8)
# convert numpy array to image
img = cv2.imdecode(npimg, cv2.IMREAD_UNCHANGED)
in_image = cv2.resize(img, (32, 32), interpolation = cv2.INTER_AREA)
in_image = np.expand_dims(in_image, axis = 0)
# Load the saved traffic sign keras model
model_filename = "model.h5"
# Load model from file - read mode
traffic_model = load_model(model_filename, compile=False)
# Make prediction using the input image file
## check dimensions before and after of the numpy array to see 2D, 3D, 4D
result = traffic_model.predict(in_image)
def sign(i):
switcher={
0:'Maximum speed limit (10 km/h)',
1:'Maximum speed limit (30 km/h)',
2:'Maximum speed limit (50 km/h)',
3:'Maximum speed limit (60 km/h)',
4:'Maximum speed limit (70 km/h)',
5:'Maximum speed limit (80 km/h)',
6:'End 80 km/h speed limit',
7:'Maximum speed limit (100 km/h)',
8:'Maximum speed limit (120 km/h)',
9:'No Overtaking',
10:'No overtaking by trucks/heavy goods vehicles',
11:'Crossroads ahead with a minor road',
12:'Priority road',
13:'Give way',
14:'Stop',
15:'Road closed to all traffic',
16:'No trucks/heavy goods vehicles',
17:'No Entry',
18:'Attention: Other Dangers!',
19:'Curve to the left',
20:'Curve to the right',
21:'Series of curves, first to the left',
22:'Uneven road surface',
23:'Slippery road surface',
24:'Road narrows on the right',
25:'Roadworks',
26:'Traffic signals ahead',
27:'Pedestrian crossing ahead',
28:'Watch for children',
29:'Watch for cyclists',
30:'Risk of ice',
31:'Watch for wild animals',
32:'End all previously signed restrictions or prohibitions',
33:'Turn right only',
34:'Turn left only',
35:'Proceed straight ahead only',
36:'Proceed straight or turn right',
37:'Proceed straight or turn left',
38:'Keep right',
39:'Keep left',
40:'Roundabout',
41:'End overtaking prohibition',
42:'End overtaking prohibition for trucks/heavy goods vehicles',
}
return switcher.get(i,"Unknown")
print(sign(np.argmax(result)))
K.clear_session()
# Send the prediction as response
return str(sign(np.argmax(result)))
val_labels) = (train_data / 255,
train_labels / 255), (val_data / 255,
val_labels / 255)
epochs = [500]
csv_file = "../historique_tests/best_mlp_without_regularisation.csv"
structs = generateStructsFromCSV(csv_file)
struct = structs[5]
for j in range(1):
struct.use_dropout = True
struct.dropout_indexes = [0]
# dropout_indexes_number = randint(1, struct.nb_hidden_layers)
struct.dropout_value = 0.3
# for j in range(dropout_indexes_number):
# dropout_indexes.append(randint(1, struct.nb_hidden_layers))
model = [create_custom_mlp(struct)]
desc = [getMlpStructAsString(struct)]
print(desc[0])
test_models('mlp_best_models_with_dropout_1',
model,
desc,
train_data,
train_labels,
val_data,
val_labels,
epochs_p=epochs,
batch_size_p=8192)
clear_session() #clear tf GPU memory
def parameterSearch(self,
paramSets,
X,
Y,
numSplits=2,
valSplit=0.0,
epochs=1,
batchSize=None,
saveModel=False,
visualize=False,
saveLoc=''):
# create CV dat LOOV
#numSplits = 2
Kf = StratifiedKFold(n_splits=numSplits)
callBacks = [
EarlyStopping(monitor='val_loss',
patience=3,
restore_best_weights=True)
]
if (visualize):
callBacks.append(
TensorBoard(log_dir='./logs',
histogram_freq=3,
write_graph=False,
write_images=False,
update_freq='epoch',
profile_batch=2,
embeddings_freq=0,
embeddings_metadata=None))
#for each parameter set
# make a model
#
#X = [0,1,2,3,4,5,6,7,8,9]
modelFile = open(self.outputPath + "fileModel.csv", 'w')
resultFile = open(self.outputPath + "fileResult.csv", 'w')
resultFile.write(
"modelNum|True REM|False REM|False NonREM|True NonREM|Acc|Sens|Spec|Recall|Precision|f1score|finalLoss\n"
)
modelNum = 0
for paramSet in paramSets:
modelFile.write(str(modelNum) + "|")
json.dump(paramSet, modelFile)
modelFile.write("\n")
print("\n\n=================\nTesting Model " + str(modelNum) +
"\n=================\n")
#print(paramSet, flush=True)
try:
model = self.convModel(paramSet)
print(model.summary())
#model.save_weights('temp_weights.h5')
j = 0
for trainInd, testInd in Kf.split(X, np.argmax(Y, axis=1)):
fitHistory = model.fit(X[trainInd],
Y[trainInd],
batch_size=batchSize,
verbose=0,
validation_split=valSplit,
epochs=epochs,
callbacks=callBacks)
if (saveModel):
modelWeightFile = saveLoc + f'{modelNum}.{j}.weights.h5'
model.save_weights(modelWeightFile)
#model.save(modelWeightFile)
Ypred = np.zeros((testInd.shape[0], Y.shape[1]))
Yi = 0
for pred in np.argmax(model.predict(X[testInd],
batch_size=None),
axis=1):
Ypred[Yi][pred] = 1
Yi += 1
#NOTE:
#confusionMatrix = multilabel_confusion_matrix(Y[testInd], Ypred)[0]
##print(confusionMatrix)
##confusionMatrix = confusion_matrix(np.argmax(Y[testInd], axis=1), np.argmax(Ypred, axis=1))
##print(confusionMatrix)
##print('f1_score:',f1_score(Y[testInd], Ypred, average='macro'))
#resultFile.write(str(modelNum) + "|")
##for row in confusionMatrix:
## for el in row:
## resultFile.write(str(el) + "|")
##"modelNum|True REM|False NonREM|False REM|True NonREM|Acc|Sens|Spec|Recall|Precision|f1score\n"
#
#tn = confusionMatrix[0][0]
#fn = confusionMatrix[1][0]
#tp = confusionMatrix[1][1]
#fp = confusionMatrix[0][1]
tp = tn = fn = fp = 0
Yi = 0
for y in Y[testInd]:
tp += Ypred[Yi][0] * y[0]
fp += max(Ypred[Yi][0] - y[0], 0)
tn += Ypred[Yi][1] * y[1]
fn += max(Ypred[Yi][1] - y[1], 0)
Yi += 1
acc = sens = spec = prec = rec = f1 = 0
acc = (tp + tn) / (tp + tn + fp + fn)
if (tp + fn > 0):
sens = tp / (tp + fn)
if (tn + fp > 0):
spec = tn / (tn + fp)
if (tp + fp > 0):
prec = tp / (tp + fp)
if (tp + fn > 0):
rec = tp / (tp + fn)
if (prec + rec > 0):
f1 = 2 * ((prec * rec) / (prec + rec))
resultFile.write(
f"{modelNum}|{tp:.3f}|{fp:.3f}|{fn:.3f}|{tn:.3f}|{acc:.3f}|{sens:.3f}|{spec:.3f}|{rec:.3f}|{prec:.3f}|{f1:.3f}|{fitHistory.history['loss'][-1]:10.3f}\n"
)
print(
f"{'Validate':10s}|{'modelNum':10s}|{'tp':10s}|{'fp':10s}|{'fn':10s}|{'tn':10s}|{'acc':10s}|{'sens':10s}|{'spec':10s}|{'rec':10s}|{'prec':10s}|{'f1':10s}|{'loss':10s}\n"
)
print(
f"{j:10d}|{modelNum:10d}|{tp:10.3f}|{fp:10.3f}|{fn:10.3f}|{tn:10.3f}|{acc:10.3f}|{sens:10.3f}|{spec:10.3f}|{rec:10.3f}|{prec:10.3f}|{f1:10.3f}|{fitHistory.history['loss'][-1]:10.3f}\n",
flush=True)
#resultFile.write(str(f1_score(Y[testInd], Ypred, average='macro')) + "|\n")
#model.load_weights('temp_weights.h5')
self.reset_weights(model)
j += 1
except Exception as e:
resultFile.write("error\n")
print(str(e))
K.clear_session()
modelNum += 1
if self.killer.kill_now:
resultFile.write("killed\n")
print("killed")
break
modelFile.close()
resultFile.close()
def run_whole_procedure(fig_prefix, model_type, cp_family, att_type):
"""
execute the whole procedure
:param fig_prefix:
:return:
"""
global ins2vec_model, word_index, x_test, y_test
# clear existing tf graph at the start of each iteration
kb.clear_session()
# re-setup
sess_setup()
# step 1: load pre-trained ins2vec model
print('loading pre-trained ins2vec model...')
ins2vec_model = Word2Vec.load(dic_file_path)
# step 2: construct ins2vec embeddings
print('constructing pre-trained ins2vec embeddings...')
word_index, embeddings_matrix = construct_ins_embedding()
# step 3: construct the textcnn model
print('setting up the neural model...')
rnn_models = RNNModels(MAX_SEQUENCE_LENGTH,
len(embeddings_matrix),
EMBEDDING_DIM,
CLASS_NUMBER,
'softmax',
att_type=att_type)
if model_type == 'cudnn_ver':
model, fn = rnn_models.get_cudnn_version_model(embeddings_matrix,
DROP_OUT)
elif model_type == 'cnn_rnn':
model, fn = rnn_models.get_cnn_rnn_model(embeddings_matrix, DROP_OUT)
elif model_type == 'bidirectional_cudnn':
model, fn = rnn_models.get_bidirectional_cudnn_model(
embeddings_matrix, DROP_OUT)
elif model_type == 'naive':
model, fn = rnn_models.get_naive_version_model(embeddings_matrix,
DROP_OUT)
elif model_type == 'bidirectional':
model = rnn_models.get_bidirectional_model(embeddings_matrix, DROP_OUT)
elif model_type == 'multilayer_stateful':
model = rnn_models.get_multilayer_stateful_model(
embeddings_matrix, DROP_OUT)
elif model_type == 'multilayer':
model = rnn_models.get_multilayer_model(embeddings_matrix, DROP_OUT)
else:
print('!!!!!!!!!!NO SUCH MODEL: ' + model_type)
# 搭建神经网络完成后，这一步相当于编译它
opti = optimizers.Adam()
model.compile(opti, 'categorical_crossentropy',
metrics=['accuracy']) # 多分类
# step 4: train the model with available dataset
print('---------------------Step 4---------------------------')
if unique_fun_stg == 'Intact':
texts, labels = prepare_data(corpus_path, path_stg_suffix, cp_family)
else:
texts, labels = prepare_data_with_unique(corpus_path, path_stg_suffix,
unique_fun_path, cp_family)
print('train-test-data split and shuffle...')
x_train, x_test, y_train, y_test = train_test_split(
texts,
labels,
test_size=dataset_split_ratio,
random_state=SEED,
shuffle=True)
print(len(x_train), 'train sequences')
print(len(x_test), 'test sequences')
# train the neural network
print('training...')
start = time.clock()
model = neural_net_train(model,
x_train=x_train,
y_train=y_train,
fig_prefix=fig_prefix)
end = time.clock()
print('neural-net training takes: %s seconds' % (end - start))
# model is got and stored
# step 5: evaluate the model on test data
model_evaluaiton(model, fig_prefix)
def main(dataset_name):
dataset = load_dataset()
raw_data = np.asarray(dataset['raw']['data'])
raw_label = np.asarray(dataset['raw']['label'])
num_classes = len(np.unique(raw_label))
rskf = RepeatedStratifiedKFold(n_splits=k_folds,
n_repeats=k_fold_reps,
random_state=42)
for e2efs_class in e2efs_classes:
print('E2EFS-Method : ', e2efs_class.__name__)
cont_seed = 0
nfeats = []
accuracies = []
model_accuracies = []
svc_accuracies = []
fs_time = []
BAs = []
svc_BAs = []
model_BAs = []
mAPs = []
svc_mAPs = []
model_mAPs = []
mus = []
name = dataset_name + '_' + kernel + '_mu_' + str(mu)
print(name)
for j, (train_index,
test_index) in enumerate(rskf.split(raw_data, raw_label)):
print('k_fold', j, 'of', k_folds * k_fold_reps)
train_data, train_labels = raw_data[train_index], raw_label[
train_index]
test_data, test_labels = raw_data[test_index], raw_label[
test_index]
train_labels = to_categorical(train_labels,
num_classes=num_classes)
test_labels = to_categorical(test_labels, num_classes=num_classes)
valid_features = np.where(np.abs(train_data).sum(axis=0) > 0)[0]
if len(valid_features) < train_data.shape[1]:
print('Removing', train_data.shape[1] - len(valid_features),
'zero features')
train_data = train_data[:, valid_features]
test_data = test_data[:, valid_features]
model_kwargs = {
'mu': mu / len(train_data),
'kernel': kernel,
'degree': 3
}
svc_kwargs = {'C': 1.0, 'solver': 0.}
for i, n_features in enumerate([10, 50, 100, 150, 200]):
n_accuracies = []
n_svc_accuracies = []
n_model_accuracies = []
n_BAs = []
n_svc_BAs = []
n_model_BAs = []
n_mAPs = []
n_svc_mAPs = []
n_model_mAPs = []
n_train_accuracies = []
n_time = []
print('n_features : ', n_features)
heatmaps = []
weight = train_labels[:, -1].mean()
for r in range(reps):
np.random.seed(cont_seed)
K.tf.set_random_seed(cont_seed)
cont_seed += 1
model = train_Keras(
train_data,
train_labels,
test_data,
test_labels,
model_kwargs,
e2efs_class=e2efs_class,
n_features=n_features,
)
heatmaps.append(K.eval(model.heatmap))
n_time.append(model.fs_time)
test_data_norm = model.normalization.transform(test_data)
train_data_norm = model.normalization.transform(train_data)
test_pred = model.predict(test_data_norm)
n_model_accuracies.append(
model.evaluate(test_data_norm, test_labels,
verbose=0)[-1])
n_model_BAs.append(balance_accuracy(
test_labels, test_pred))
n_model_mAPs.append(
average_precision_score(test_labels[:, -1], test_pred))
train_acc = model.evaluate(train_data_norm,
train_labels,
verbose=0)[-1]
print('n_features : ', n_features, ', accuracy : ',
n_model_accuracies[-1], ', BA : ', n_model_BAs[-1],
', mAP : ', n_model_mAPs[-1], ', train_accuracy : ',
train_acc, ', time : ', n_time[-1], 's')
del model
K.clear_session()
heatmap = np.mean(heatmaps, axis=0)
best_features = np.argsort(heatmap)[::-1][:n_features]
svc_train_data = train_data[:, best_features]
svc_test_data = test_data[:, best_features]
norm = normalization_func()
svc_train_data_norm = norm.fit_transform(svc_train_data)
svc_test_data_norm = norm.transform(svc_test_data)
bestcv = -1
bestc = None
bestSolver = None
for s in [0, 1, 2, 3]:
for my_c in [
0.001, 0.1, 0.5, 1.0, 1.4, 1.5, 1.6, 2.0, 2.5, 5.0,
100.0
]:
cmd = '-v 5 -s ' + str(s) + ' -c ' + str(my_c) + ' -q'
cv = liblinearutil.train(
(2 * train_labels[:, -1] - 1).tolist(),
svc_train_data_norm.tolist(), cmd)
if cv > bestcv:
# print('Best -> C:', my_c, ', s:', s, ', acc:', cv)
bestcv = cv
bestc = my_c
bestSolver = s
svc_kwargs['C'] = bestc
svc_kwargs['solver'] = bestSolver
print('Best -> C:', bestc, ', s:', bestSolver, ', acc:',
bestcv)
for r in range(reps):
np.random.seed(cont_seed)
K.tf.set_random_seed(cont_seed)
cont_seed += 1
model = train_SVC(svc_train_data_norm, train_labels,
svc_kwargs)
_, accuracy, test_pred = liblinearutil.predict(
(2 * test_labels[:, -1] - 1).tolist(),
svc_test_data_norm.tolist(), model, '-q')
test_pred = np.asarray(test_pred)
n_svc_accuracies.append(accuracy[0])
n_svc_BAs.append(balance_accuracy(test_labels, test_pred))
n_svc_mAPs.append(
average_precision_score(test_labels[:, -1], test_pred))
del model
model = train_Keras(svc_train_data, train_labels,
svc_test_data, test_labels,
model_kwargs)
train_data_norm = model.normalization.transform(
svc_train_data)
test_data_norm = model.normalization.transform(
svc_test_data)
test_pred = model.predict(test_data_norm)
n_BAs.append(balance_accuracy(test_labels, test_pred))
n_mAPs.append(
average_precision_score(test_labels[:, -1], test_pred))
n_accuracies.append(
model.evaluate(test_data_norm, test_labels,
verbose=0)[-1])
n_train_accuracies.append(
model.evaluate(train_data_norm,
train_labels,
verbose=0)[-1])
del model
K.clear_session()
print(
'n_features : ',
n_features,
', acc : ',
n_accuracies[-1],
', BA : ',
n_BAs[-1],
', mAP : ',
n_mAPs[-1],
', train_acc : ',
n_train_accuracies[-1],
', svc_acc : ',
n_svc_accuracies[-1],
', svc_BA : ',
n_svc_BAs[-1],
', svc_mAP : ',
n_svc_mAPs[-1],
)
if i >= len(accuracies):
accuracies.append(n_accuracies)
svc_accuracies.append(n_svc_accuracies)
model_accuracies.append(n_model_accuracies)
BAs.append(n_BAs)
mAPs.append(n_mAPs)
fs_time.append(n_time)
svc_BAs.append(n_svc_BAs)
svc_mAPs.append(n_svc_mAPs)
model_BAs.append(n_model_BAs)
model_mAPs.append(n_model_mAPs)
nfeats.append(n_features)
mus.append(model_kwargs['mu'])
else:
accuracies[i] += n_accuracies
svc_accuracies[i] += n_svc_accuracies
model_accuracies[i] += n_model_accuracies
fs_time[i] += n_time
BAs[i] += n_BAs
mAPs[i] += n_mAPs
svc_BAs[i] += n_svc_BAs
svc_mAPs[i] += n_svc_mAPs
model_BAs[i] += n_model_BAs
model_mAPs[i] += n_model_mAPs
output_filename = directory + 'LinearSVC_' + kernel + '_' + e2efs_class.__name__ + '.json'
if not os.path.isdir(directory):
os.makedirs(directory)
info_data = {
'kernel': kernel,
'reps': reps,
'classification': {
'mus':
mus,
'n_features':
nfeats,
'accuracy':
accuracies,
'mean_accuracy':
np.array(accuracies).mean(axis=1).tolist(),
'svc_accuracy':
svc_accuracies,
'mean_svc_accuracy':
np.array(svc_accuracies).mean(axis=1).tolist(),
'model_accuracy':
model_accuracies,
'mean_model_accuracy':
np.array(model_accuracies).mean(axis=1).tolist(),
'BA':
BAs,
'mean_BA':
np.array(BAs).mean(axis=1).tolist(),
'mAP':
mAPs,
'mean_mAP':
np.array(mAPs).mean(axis=1).tolist(),
'svc_BA':
svc_BAs,
'svc_mean_BA':
np.array(svc_BAs).mean(axis=1).tolist(),
'svc_mAP':
svc_mAPs,
'svc_mean_mAP':
np.array(svc_mAPs).mean(axis=1).tolist(),
'model_BA':
model_BAs,
'model_mean_BA':
np.array(model_BAs).mean(axis=1).tolist(),
'model_mAP':
model_mAPs,
'model_mean_mAP':
np.array(model_mAPs).mean(axis=1).tolist(),
'fs_time':
fs_time
}
}
for k, v in info_data['classification'].items():
if 'mean' in k:
print(k, v)
with open(output_filename, 'w') as outfile:
json.dump(info_data, outfile)
def explain_with_saliency_map(fig_prefix,
model_type,
cp_family,
att_type,
input_files=None,
func_list=None,
top_k=10):
"""
get the prediction result explained with saliency map analysis
:param model_path:
:param input_files: list of files (each corresponds to an executable)
:param func_list: list of functions (names) to be analyzed
:param top_k:
:return:
"""
global ins2vec_model, word_index, x_test, y_test, func_names
# clear existing tf graph at the start of each iteration
kb.clear_session()
# re-setup
sess_setup()
# step 1: load pre-trained ins2vec model
print('loading pre-trained ins2vec model...')
ins2vec_model = Word2Vec.load(dic_file_path)
# step 2: construct ins2vec embeddings
print('constructing pre-trained ins2vec embeddings...')
word_index, embeddings_matrix = construct_ins_embedding()
# step 3: construct the textcnn model
print('setting up the neural model...')
rnn_models = RNNModels(MAX_SEQUENCE_LENGTH,
len(embeddings_matrix),
EMBEDDING_DIM,
CLASS_NUMBER,
'softmax',
att_type=att_type)
if model_type == 'cudnn_ver':
model, fn = rnn_models.get_cudnn_version_model(embeddings_matrix,
DROP_OUT)
elif model_type == 'cnn_rnn':
model, fn = rnn_models.get_cnn_rnn_model(embeddings_matrix, DROP_OUT)
elif model_type == 'bidirectional_cudnn':
model, fn = rnn_models.get_bidirectional_cudnn_model(
embeddings_matrix, DROP_OUT)
elif model_type == 'naive':
model, fn = rnn_models.get_naive_version_model(embeddings_matrix,
DROP_OUT)
elif model_type == 'bidirectional':
model = rnn_models.get_bidirectional_model(embeddings_matrix, DROP_OUT)
elif model_type == 'multilayer_stateful':
model = rnn_models.get_multilayer_stateful_model(
embeddings_matrix, DROP_OUT)
elif model_type == 'multilayer':
model = rnn_models.get_multilayer_model(embeddings_matrix, DROP_OUT)
else:
print('!!!!!!!!!!NO SUCH MODEL: ' + model_type)
if unique_fun_stg == 'Intact':
x_test, y_test, func_names = prepare_data(corpus_path, path_stg_suffix,
cp_family, True)
else:
x_test, y_test, func_names = prepare_data_with_unique(
corpus_path, path_stg_suffix, unique_fun_path, cp_family, True)
print(len(x_test), 'test sequences')
# load the trained model
print('loading...' + model_store_path + '\n')
# step 5: evaluate the model on test data
model_evaluaiton_for_explain(model, fig_prefix, fn, top_k)
def main(args):
train_df = data.load_data.load_custom_text_as_pd(args.train_data,sep='\t',header=True, \
text_column=['Text'],target_column=['Label'])
val_df = data.load_data.load_custom_text_as_pd(args.val_data,sep='\t', header=False, \
text_column=['Text'],target_column=['Label'])
train_df = pd.DataFrame(train_df, copy=False)
val_df = pd.DataFrame(val_df, copy=False)
val_df.columns = train_df.columns
model_save_dir = args.model_save_path
try:
os.makedirs(model_save_dir)
except OSError:
pass
nli_train_df = pd.DataFrame()
nli_val_df = pd.DataFrame()
mnli = data.data_utils.mnli_data(train_df.labels.unique())
df_ = pd.DataFrame()
for i in tqdm(range(train_df.shape[0])):
df_['Id'], df_['words'], df_['text2'], df_['labels'], df_['orig_label'] = mnli.convert_to_mnli_format(\
train_df.Id.iloc[i], train_df.words.iloc[i], train_df.labels.iloc[i])
nli_train_df = pd.concat([nli_train_df, df_])
for i in tqdm(range(val_df.shape[0])):
df_['Id'], df_['words'], df_['text2'], df_['labels'], df_['orig_label'] = mnli.convert_to_mnli_format(\
val_df.Id.iloc[i], val_df.words.iloc[i], val_df.labels.iloc[i])
nli_val_df = pd.concat([nli_val_df, df_])
nli_train_df.labels, label2idx = data.data_utils.convert_categorical_label_to_int(nli_train_df.labels, \
save_path=os.path.join(model_save_dir,'label2idx.pkl'))
nli_val_df.labels, _ = data.data_utils.convert_categorical_label_to_int(nli_val_df.labels, \
save_path=os.path.join(model_save_dir,'label2idx.pkl'))
#print (nli_train_df.head(5))
#print (nli_val_df.head(5))
print("Tokenization")
if 'bertweet' in args.transformer_model_name.lower():
bertweettokenizer = True
else:
bertweettokenizer = False
if bertweettokenizer == True:
tokenizer = data.custom_tokenizers.BERTweetTokenizer(
args.transformer_model_name)
else:
tokenizer = AutoTokenizer.from_pretrained(args.transformer_model_name)
trainX = data.data_utils.compute_transformer_input_arrays(
nli_train_df, 'words', tokenizer, args.max_text_len, 'text2',
bertweettokenizer)
valX = data.data_utils.compute_transformer_input_arrays(
nli_val_df, 'words', tokenizer, args.max_text_len, 'text2',
bertweettokenizer)
outputs = data.data_utils.compute_output_arrays(nli_train_df, 'labels')
val_outputs = data.data_utils.compute_output_arrays(nli_val_df, 'labels')
outputs = outputs[:, np.newaxis]
val_outputs = val_outputs[:, np.newaxis]
print("Modelling")
model = models.tf_models.transformer_base_model_cls_token(
args.transformer_model_name, args.max_text_len, dropout=args.dropout)
print(model.summary())
optimizer = tf.keras.optimizers.Adam(learning_rate=args.lr) #SGD
model.compile(loss='binary_crossentropy',
optimizer=optimizer,
metrics='accuracy') #binary_crossentropy
early = tf.keras.callbacks.EarlyStopping(monitor='val_accuracy', patience=5, \
verbose=1, mode='auto', restore_best_weights=True)
lr = tf.keras.callbacks.ReduceLROnPlateau(monitor='val_accuracy', factor=0.7, \
patience=3, verbose=1, mode='auto', min_lr=0.000001)
f1callback = models.tf_utils.F1Callback(model,
valX,
val_outputs,
filename=os.path.join(
model_save_dir, 'model.h5'),
patience=5)
snapshot = models.tf_utils.Snapshot(model, valX, val_outputs)
config = {
'text_max_len': args.max_text_len,
'epochs': args.epochs,
"learning_rate": args.lr,
"batch_size": args.train_batch_size,
"dropout": args.dropout,
"model_description": args.transformer_model_name + ' with NLI'
}
with open(os.path.join(model_save_dir, 'config.pkl'), 'wb') as handle:
pickle.dump(config, handle, protocol=pickle.HIGHEST_PROTOCOL)
K.clear_session()
if os.path.exists(os.path.join(model_save_dir, 'model.h5')) == False:
if _has_wandb and args.wandb_logging:
wandb.init(project='wnut-task2', config=config)
model.fit(trainX, outputs, validation_data=(valX, val_outputs), epochs=args.epochs,\
batch_size=args.train_batch_size, callbacks=[early, lr, f1callback, WandbCallback(), snapshot], verbose=1)
else:
model.fit(trainX, outputs, validation_data=(valX, val_outputs), epochs=args.epochs,\
batch_size=args.train_batch_size, callbacks=[early,lr, f1callback, snapshot], verbose=1)
model.load_weights(os.path.join(model_save_dir, 'model.h5'))
#model_json = model.to_json()
#with open(os.path.join(model_save_dir,"model.json"), "w") as json_file:
# json_file.write(model_json)
val_pred = np.round(model.predict(valX))[:, 0]
print("NLI Evaluation")
f1 = f1_score(nli_val_df.labels, val_pred)
precision = precision_score(nli_val_df.labels, val_pred)
recall = recall_score(nli_val_df.labels, val_pred)
print("NLI scores: \nF1 {}, Precision {} and Recall {}".format(
f1, precision, recall))
print("Original Evaluation")
nli_val_df['pred_proba'] = model.predict(valX)[:, 0]
nli_val_df_ = nli_val_df.sort_values(['Id', 'pred_proba'],
ascending=[True, True
]).reset_index(drop=True)
nli_val_df_ = nli_val_df_.drop_duplicates(subset=['Id']).reset_index(
drop=True)
val_df = val_df.drop_duplicates(subset=['Id']).reset_index(drop=True)
assert nli_val_df_.shape[0] == val_df.shape[0]
val_df = pd.merge(val_df, nli_val_df_[['Id', 'orig_label']], how='inner')
#train_df.labels, _ = data.data_utils.convert_categorical_label_to_int(train_df.labels, \
# save_path=os.path.join(model_save_dir,'label2idx_orig.pkl'))
print(val_df.iloc[0])
val_df.labels, _ = data.data_utils.convert_categorical_label_to_int(val_df.labels, \
save_path=os.path.join(model_save_dir,'label2idx_orig.pkl'))
val_df.orig_label, _ = data.data_utils.convert_categorical_label_to_int(val_df.orig_label, \
save_path=os.path.join(model_save_dir,'label2idx_orig.pkl'))
print(val_df.iloc[0])
f1 = f1_score(val_df.labels, val_df.orig_label)
precision = precision_score(val_df.labels, val_df.orig_label)
recall = recall_score(val_df.labels, val_df.orig_label)
'''
snapshot_val_pred = np.round(np.concatenate(snapshot.best_scoring_snapshots, axis=-1).mean(-1))
print ("Snapshot Evaluation")
f1_snapshot = f1_score(val_df.labels, snapshot_val_pred)
precision_snapshot = precision_score(val_df.labels, snapshot_val_pred)
recall_snapshot = recall_score(val_df.labels, snapshot_val_pred)
'''
results_ = pd.DataFrame()
results_['description'] = [args.transformer_model_name + ' with NLI']
results_['f1'] = [f1]
results_['precision'] = [precision]
results_['recall'] = [recall]
print(results_.iloc[0])
#print ("Snapshot scores: \nF1 {}, Precision {} and Recall {}".format(f1_snapshot, precision_snapshot, recall_snapshot))
if os.path.exists('../results/result.csv'):
results = pd.read_csv('../results/result.csv')
results = pd.concat([results, results_], axis=0)
results.to_csv('../results/result.csv', index=False)
else:
results_.to_csv('../results/result.csv', index=False)
def scan_round(self):
'''The main operational function that manages the experiment
on the level of execution of each round.'''
# determine the parameters for the particular execution
self.round_params = round_params(self)
# print round params
if self.print_params is True:
print(self.round_params)
# set start time
round_start = strftime('%H%M%S')
start = time()
# fit the model
try:
_hr_out, self.keras_model = ingest_model(self)
except TypeError as err:
if err.args[0] == "unsupported operand type(s) for +: 'int' and 'numpy.str_'":
raise TalosTypeError("Activation should be as object and not string in params")
else:
raise TalosReturnError("Make sure that input model returns 'out, model' where out is history object from model.fit()")
# set end time and log
round_end = strftime('%H%M%S')
round_seconds = time() - start
self.round_times.append([round_start, round_end, round_seconds])
# create log and other stats
try:
self.epoch_entropy.append(epoch_entropy(_hr_out))
except (TypeError, AttributeError):
raise TalosReturnError("Make sure that input model returns in the order 'out, model'")
if self.round_counter == 0:
_for_header = create_header(self, _hr_out)
self.result.append(_for_header)
save_result(self)
_hr_out = run_round_results(self, _hr_out)
self.result.append(_hr_out)
save_result(self)
# apply reduction
if self.reduction_method is not None:
if (self.round_counter + 1) % self.reduction_interval == 0:
len_before_reduce = len(self.param_log)
self = reduce_run(self)
total_reduced = len_before_reduce - len(self.param_log)
# update the progress bar
self.pbar.update(total_reduced)
# save model and weights
self.saved_models.append(self.keras_model.to_json())
self.saved_weights.append(self.keras_model.get_weights())
# clear tensorflow sessions (maybe)
if self.clear_tf_session is True:
K.clear_session()
self.round_counter += 1
return self
