def main():
# parser config
config_file = "./config.ini"
cp = ConfigParser()
cp.read(config_file)
# default config
output_dir = cp["DEFAULT"].get("output_dir")
base_model_name = cp["DEFAULT"].get("base_model_name")
class_names = cp["DEFAULT"].get("class_names").split(",")
image_source_dir = cp["DEFAULT"].get("image_source_dir")
# train config
image_dimension = cp["TRAIN"].getint("image_dimension")
# test config
batch_size = cp["TEST"].getint("batch_size")
test_steps = cp["TEST"].get("test_steps")
use_best_weights = cp["TEST"].getboolean("use_best_weights")
# parse weights file path
output_weights_name = cp["TRAIN"].get("output_weights_name")
weights_path = os.path.join(output_dir, output_weights_name)
best_weights_path = os.path.join(output_dir, f"best_{output_weights_name}")
# get test sample count
test_counts, _ = get_sample_counts(output_dir, "test", class_names)
# compute steps
if test_steps == "auto":
test_steps = int(test_counts / batch_size)
else:
try:
test_steps = int(test_steps)
except ValueError:
raise ValueError(f"""
test_steps: {test_steps} is invalid,
please use 'auto' or integer.
""")
print(f"** test_steps: {test_steps} **")
print("** load model **")
if use_best_weights:
print("** use best weights **")
model_weights_path = best_weights_path
else:
print("** use last weights **")
model_weights_path = weights_path
model_factory = ModelFactory()
print("** weight path is {} **".format(model_weights_path))
model = model_factory.get_model(
class_names,
model_name=base_model_name,
use_base_weights=False,
weights_path=model_weights_path)
print("** load test generator **")
test_ids = PatientInfo('stage_1_test_images/', train=True)
train=False,
img_dir='stage_1_test_images/*' #image_source_dir+'*', #'stage_1_test_images/*',
def load_ori_model(config_file="./config.ini"):
cp = ConfigParser()
cp.read(config_file)
# default config
output_dir = cp["DEFAULT"].get("output_dir")
base_model_name = cp["DEFAULT"].get("base_model_name")
class_names = cp["DEFAULT"].get("class_names").split(",")
image_source_dir = cp["DEFAULT"].get("image_source_dir")
# train config
image_dimension = cp["TRAIN"].getint("image_dimension")
# test config
batch_size = cp["TEST"].getint("batch_size")
test_steps = cp["TEST"].get("test_steps")
use_best_weights = cp["TEST"].getboolean("use_best_weights")
# parse weights file path
output_weights_name = cp["TRAIN"].get("output_weights_name")
weights_path = os.path.join(output_dir, output_weights_name)
best_weights_path = os.path.join(output_dir, f"best_{output_weights_name}")
# get test sample count
test_counts, _ = get_sample_counts(output_dir, "test", class_names)
# compute steps
if test_steps == "auto":
test_steps = int(test_counts / batch_size)
else:
try:
test_steps = int(test_steps)
except ValueError:
raise ValueError(f"""
test_steps: {test_steps} is invalid,
please use 'auto' or integer.
""")
print(f"** test_steps: {test_steps} **")
print("** load model **")
if use_best_weights:
print("** use best weights **")
model_weights_path = best_weights_path
else:
print("** use last weights **")
model_weights_path = weights_path
# load CheXNet model:
model_factory = ModelFactory()
model = model_factory.get_model(class_names,
model_name=base_model_name,
use_base_weights=False,
weights_path=model_weights_path)
return model
def main():
# parser config
config_file = "./config.ini"
cp = ConfigParser()
cp.read(config_file)
# default config
output_dir = cp["DEFAULT"].get("output_dir")
image_source_dir = cp["DEFAULT"].get("image_source_dir")
train_patient_count = cp["DEFAULT"].getint("train_patient_count")
dev_patient_count = cp["DEFAULT"].getint("dev_patient_count")
data_entry_file = cp["DEFAULT"].get("data_entry_file")
class_names = cp["DEFAULT"].get("class_names").split(",")
# train config
use_base_model_weights = cp["TRAIN"].getboolean("use_base_model_weights")
use_trained_model_weights = cp["TRAIN"].getboolean(
"use_trained_model_weights")
use_best_weights = cp["TRAIN"].getboolean("use_best_weights")
output_weights_name = cp["TRAIN"].get("output_weights_name")
epochs = cp["TRAIN"].getint("epochs")
batch_size = cp["TRAIN"].getint("batch_size")
initial_learning_rate = cp["TRAIN"].getfloat("initial_learning_rate")
train_steps = cp["TRAIN"].get("train_steps")
patience_reduce_lr = cp["TRAIN"].getint("patience_reduce_lr")
validation_steps = cp["TRAIN"].get("validation_steps")
positive_weights_multiply = cp["TRAIN"].getfloat(
"positive_weights_multiply")
use_class_balancing = cp["TRAIN"].getboolean("use_class_balancing")
use_default_split = cp["TRAIN"].getboolean("use_default_split")
# if previously trained weights is used, never re-split
if use_trained_model_weights:
# resuming mode
print(
"** use trained model weights, turn on use_skip_split automatically **"
)
use_skip_split = True
# load training status for resuming
training_stats_file = os.path.join(output_dir, ".training_stats.json")
if os.path.isfile(training_stats_file):
# TODO: add loading previous learning rate?
training_stats = json.load(open(training_stats_file))
else:
training_stats = {}
else:
# start over
use_skip_split = cp["TRAIN"].getboolean("use_skip_split ")
training_stats = {}
split_dataset_random_state = cp["TRAIN"].getint(
"split_dataset_random_state")
show_model_summary = cp["TRAIN"].getboolean("show_model_summary")
# end parser config
# check output_dir, create it if not exists
if not os.path.isdir(output_dir):
os.makedirs(output_dir)
running_flag_file = os.path.join(output_dir, ".training.lock")
if os.path.isfile(running_flag_file):
raise RuntimeError("A process is running in this directory!!!")
else:
open(running_flag_file, "a").close()
try:
print(f"backup config file to {output_dir}")
shutil.copy(config_file,
os.path.join(output_dir,
os.path.split(config_file)[1]))
# split train/dev/test
if use_default_split:
datasets = ["train", "dev", "test"]
for dataset in datasets:
shutil.copy(f"./data/default_split/{dataset}.csv", output_dir)
elif not use_skip_split:
print("** split dataset **")
split_data(
data_entry_file,
class_names,
train_patient_count,
dev_patient_count,
output_dir,
split_dataset_random_state,
)
# get train/dev sample counts
train_counts, train_pos_counts = get_sample_counts(
output_dir, "train", class_names)
dev_counts, _ = get_sample_counts(output_dir, "dev", class_names)
# compute steps
if train_steps == "auto":
train_steps = int(train_counts / batch_size)
else:
try:
train_steps = int(train_steps)
except ValueError:
raise ValueError(f"""
train_steps: {train_steps} is invalid,
please use 'auto' or integer.
""")
print(f"** train_steps: {train_steps} **")
if validation_steps == "auto":
validation_steps = int(dev_counts / batch_size)
else:
try:
validation_steps = int(validation_steps)
except ValueError:
raise ValueError(f"""
validation_steps: {validation_steps} is invalid,
please use 'auto' or integer.
""")
print(f"** validation_steps: {validation_steps} **")
# compute class weights
print("** compute class weights from training data **")
class_weights = get_class_weights(
train_counts,
train_pos_counts,
multiply=positive_weights_multiply,
use_class_balancing=use_class_balancing)
print("** class_weights **")
for c, w in class_weights.items():
print(f" {c}: {w}")
print("** load model **")
if use_base_model_weights:
base_model_weights_file = cp["TRAIN"].get(
"base_model_weights_file")
else:
base_model_weights_file = None
if use_trained_model_weights:
if use_best_weights:
model_weights_file = os.path.join(
output_dir, f"best_{output_weights_name}")
else:
model_weights_file = os.path.join(output_dir,
output_weights_name)
else:
model_weights_file = None
model = get_model(class_names, base_model_weights_file,
model_weights_file)
if show_model_summary:
print(model.summary())
# recreate symlink folder for ImageDataGenerator
symlink_dir_name = "image_links"
create_symlink(image_source_dir, output_dir, symlink_dir_name)
print("** create image generators **")
train_data_path = f"{output_dir}/{symlink_dir_name}/train/"
train_generator = custom_image_generator(
ImageDataGenerator(horizontal_flip=True, rescale=1. / 255),
train_data_path,
batch_size=batch_size,
class_names=class_names,
)
dev_data_path = f"{output_dir}/{symlink_dir_name}/dev/"
dev_generator = custom_image_generator(
ImageDataGenerator(horizontal_flip=True, rescale=1. / 255),
dev_data_path,
batch_size=batch_size,
class_names=class_names,
)
output_weights_path = os.path.join(output_dir, output_weights_name)
print(f"** set output weights path to: {output_weights_path} **")
print("** check multiple gpu availability **")
gpus = len(os.getenv("CUDA_VISIBLE_DEVICES", "1").split(","))
if gpus > 1:
print(f"** multi_gpu_model is used! gpus={gpus} **")
model_train = multi_gpu_model(model, gpus)
# FIXME: currently (Keras 2.1.2) checkpoint doesn't work with multi_gpu_model
checkpoint = MultiGPUModelCheckpoint(
filepath=output_weights_path,
base_model=model,
)
else:
model_train = model
checkpoint = ModelCheckpoint(output_weights_path)
print("** compile model with class weights **")
optimizer = Adam(lr=initial_learning_rate)
model_train.compile(optimizer=optimizer, loss="binary_crossentropy")
auroc = MultipleClassAUROC(
generator=dev_generator,
steps=validation_steps,
class_names=class_names,
weights_path=output_weights_path,
stats=training_stats,
)
callbacks = [
checkpoint,
TensorBoard(log_dir=os.path.join(output_dir, "logs"),
batch_size=batch_size),
ReduceLROnPlateau(monitor='val_loss',
factor=0.1,
patience=patience_reduce_lr,
verbose=1),
auroc,
]
print("** training start **")
history = model_train.fit_generator(
generator=train_generator,
steps_per_epoch=train_steps,
epochs=epochs,
validation_data=dev_generator,
validation_steps=validation_steps,
callbacks=callbacks,
class_weight=class_weights,
)
# dump history
print("** dump history **")
with open(os.path.join(output_dir, "history.pkl"), "wb") as f:
pickle.dump({
"history": history.history,
"auroc": auroc.aurocs,
}, f)
print("** done! **")
finally:
os.remove(running_flag_file)
def train_rsna_clf(train_data=None, validation_data=None, remove_running=True):
# parser config
config_file = "./config.ini"
cp = ConfigParser()
cp.read(config_file)
# default config
output_dir = cp["DEFAULT"].get("output_dir")
image_source_dir = cp["DEFAULT"].get("image_source_dir")
base_model_name = cp["DEFAULT"].get("base_model_name")
class_names1 = cp["DEFAULT"].get("class_names1").split(",")
class_names2 = cp["DEFAULT"].get("class_names2").split(",")
# train config
train_image_source_dir = cp["TRAIN"].get("train_image_source_dir")
train_class_info = cp["TRAIN"].get("train_class_info")
train_box_info = cp["TRAIN"].get("train_box_info")
use_base_model_weights = cp["TRAIN"].getboolean("use_base_model_weights")
use_trained_model_weights = cp["TRAIN"].getboolean(
"use_trained_model_weights")
use_best_weights = cp["TRAIN"].getboolean("use_best_weights")
input_weights_name = cp["TRAIN"].get("input_weights_name")
output_weights_name = cp["TRAIN"].get("output_weights_name")
epochs = cp["TRAIN"].getint("epochs")
batch_size = cp["TRAIN"].getint("batch_size")
initial_learning_rate = cp["TRAIN"].getfloat("initial_learning_rate")
generator_workers = cp["TRAIN"].getint("generator_workers")
image_dimension = cp["TRAIN"].getint("image_dimension")
train_steps = cp["TRAIN"].get("train_steps")
patience_reduce_lr = cp["TRAIN"].getint("patience_reduce_lr")
min_lr = cp["TRAIN"].getfloat("min_lr")
validation_steps = cp["TRAIN"].get("validation_steps")
positive_weights_multiply = cp["TRAIN"].getfloat(
"positive_weights_multiply")
dataset_csv_dir = cp["TRAIN"].get("dataset_csv_dir")
# if previously trained weights is used, never re-split
if use_trained_model_weights:
# resuming mode
print("** use trained model weights **")
# load training status for resuming
training_stats_file = os.path.join(output_dir, ".training_stats.json")
if os.path.isfile(training_stats_file):
# TODO: add loading previous learning rate?
training_stats = json.load(open(training_stats_file))
else:
training_stats = {}
else:
# start over
training_stats = {}
show_model_summary = cp["TRAIN"].getboolean("show_model_summary")
# end parser config
# check output_dir, create it if not exists
if not os.path.isdir(output_dir):
os.makedirs(output_dir)
running_flag_file = os.path.join(output_dir, ".training.lock")
if os.path.isfile(running_flag_file):
if remove_running:
os.remove(running_flag_file)
open(running_flag_file, "a").close()
else:
raise RuntimeError("A process is running in this directory!!!")
else:
open(running_flag_file, "a").close()
try:
print(f"backup config file to {output_dir}")
shutil.copy(config_file,
os.path.join(output_dir,
os.path.split(config_file)[1]))
# get train/dev sample counts
train_counts, train_pos_counts = get_sample_counts(
train_data.df, class_names2)
validation_counts, _ = get_sample_counts(validation_data.df,
class_names2)
# compute steps
if train_steps == "auto":
train_steps = int(train_counts / batch_size)
else:
try:
train_steps = int(train_steps)
except ValueError:
raise ValueError(f"""
train_steps: {train_steps} is invalid,
please use 'auto' or integer.
""")
print(f"** train_steps: {train_steps} **")
if validation_steps == "auto":
validation_steps = int(validation_counts / batch_size)
else:
try:
validation_steps = int(validation_steps)
except ValueError:
raise ValueError(f"""
validation_steps: {validation_steps} is invalid,
please use 'auto' or integer.
""")
print(f"** validation_steps: {validation_steps} **")
# compute class weights
print("** compute class weights from training data **")
class_weights = get_class_weights(
train_counts,
train_pos_counts,
multiply=positive_weights_multiply,
)
print("** class_weights **")
print(class_weights)
print("** load model **")
if use_trained_model_weights:
if use_best_weights:
model_weights_file = os.path.join(
output_dir, f"best_{input_weights_name}")
else:
model_weights_file = os.path.join(output_dir,
input_weights_name)
else:
model_weights_file = None
model_factory = ModelFactory()
model = model_factory.get_model(
class_names1,
model_name=base_model_name,
use_base_weights=use_base_model_weights,
weights_path=model_weights_file,
input_shape=(image_dimension, image_dimension, 3))
model = modify_last_layer(model, class_names2)
if show_model_summary:
print(model.summary())
train_sq = AugmentedLabelSequence_clf(
train_data,
batch_size=batch_size,
target_size=(image_dimension, image_dimension),
augmenter=augmenter,
steps=train_steps,
)
validation_sq = AugmentedLabelSequence_clf(
validation_data,
batch_size=batch_size,
target_size=(image_dimension, image_dimension),
augmenter=augmenter,
steps=validation_steps,
)
output_weights_path = os.path.join(output_dir, output_weights_name)
print(f"** set output weights path to: {output_weights_path} **")
print("** check multiple gpu availability **")
gpus = len(os.getenv("CUDA_VISIBLE_DEVICES", "1").split(","))
if gpus > 1:
print(f"** multi_gpu_model is used! gpus={gpus} **")
model_train = multi_gpu_model(model, gpus)
# FIXME: currently (Keras 2.1.2) checkpoint doesn't work with multi_gpu_model
checkpoint = MultiGPUModelCheckpoint(
filepath=output_weights_path,
base_model=model,
)
else:
model_train = model
checkpoint = ModelCheckpoint(
output_weights_path,
save_weights_only=True,
save_best_only=True,
verbose=1,
)
print("** compile model with class weights **")
optimizer = Adam(lr=initial_learning_rate)
model_train.compile(optimizer=optimizer, loss="binary_crossentropy")
auroc = MultipleClassAUROC(
sequence=validation_sq,
class_names=class_names2,
weights_path=output_weights_path,
stats=training_stats,
workers=generator_workers,
)
callbacks = [
checkpoint,
TensorBoard(log_dir=os.path.join(output_dir, "logs"),
batch_size=batch_size),
ReduceLROnPlateau(monitor='val_loss',
factor=0.1,
patience=patience_reduce_lr,
verbose=1,
mode="min",
min_lr=min_lr),
auroc,
]
print("** start training **")
history = model_train.fit_generator(
generator=train_sq,
steps_per_epoch=train_steps,
epochs=epochs,
validation_data=validation_sq,
validation_steps=validation_steps,
callbacks=callbacks,
class_weight=class_weights,
workers=generator_workers,
shuffle=False,
)
# dump history
print("** dump history **")
with open(os.path.join(output_dir, "history.pkl"), "wb") as f:
pickle.dump({
"history": history.history,
"auroc": auroc.aurocs,
}, f)
print("** done! **")
finally:
os.remove(running_flag_file)
def main(fold,gender_train,gender_test):
# parser config
config_file = 'config_file.ini'
cp = ConfigParser()
cp.read(config_file)
root_output_dir= cp["DEFAULT"].get("output_dir")
# default config
print(root_output_dir,gender_train)
output_dir= root_output_dir + gender_train+'/Fold_'+str(fold)+'/output/'
base_model_name = cp["DEFAULT"].get("base_model_name")
class_names = cp["DEFAULT"].get("class_names").split(",")
image_source_dir = cp["DEFAULT"].get("image_source_dir")
# train config
image_dimension = cp["TRAIN"].getint("image_dimension")
# test config
batch_size = cp["TEST"].getint("batch_size")
test_steps = cp["TEST"].get("test_steps")
use_best_weights = cp["TEST"].getboolean("use_best_weights")
# parse weights file path
output_weights_name = cp["TRAIN"].get("output_weights_name")
weights_path = os.path.join(output_dir, output_weights_name)
best_weights_path = os.path.join(output_dir, f"best_{output_weights_name}")
# get test sample count
test_counts, _ = get_sample_counts(root_output_dir+gender_train+'/Fold_'+str(fold),str(gender_test), class_names)
# compute steps
if test_steps == "auto":
test_steps = int(test_counts / batch_size)
else:
try:
test_steps = int(test_steps)
except ValueError:
raise ValueError(f"""
test_steps: {test_steps} is invalid,
please use 'auto' or integer.
""")
print(f"** test_steps: {test_steps} **")
print("** load model **")
if use_best_weights:
print("** use best weights **")
model_weights_path = best_weights_path
else:
print("** use last weights **")
model_weights_path = weights_path
model_factory = ModelFactory()
model = model_factory.get_model(
class_names,
model_name=base_model_name,
use_base_weights=False,
weights_path=model_weights_path)
print("** load test generator **")
test_sequence = AugmentedImageSequence(
dataset_csv_file=os.path.join(root_output_dir+gender_train+'/Fold_'+str(fold), str(gender_test)+".csv"),
class_names=class_names,
source_image_dir=image_source_dir,
batch_size=batch_size,
target_size=(image_dimension, image_dimension),
augmenter=None,
steps=test_steps,
shuffle_on_epoch_end=False,
)
print("** make prediction **")
y_hat = model.predict_generator(test_sequence, verbose=1)
y = test_sequence.get_y_true()
y_pred_dir = output_dir + "y_pred_run_" + str(fold)+"_train"+gender_train+"_"+gender_test+ ".csv"
y_true_dir = output_dir + "y_true_run_" + str(fold)+"_train"+gender_train+"_"+gender_test+ ".csv"
np.savetxt(y_pred_dir, y_hat, delimiter=",")
np.savetxt(y_true_dir, y, delimiter=",")
def cxpl(model_dir, data_dir, results_subdir, random_seed, resolution):
np.random.seed(random_seed)
tf.set_random_seed(np.random.randint(1 << 31))
session_conf = tf.ConfigProto(intra_op_parallelism_threads=1, inter_op_parallelism_threads=1)
sess = tf.Session(graph=tf.get_default_graph(), config=session_conf)
set_session(sess)
# parser config
config_file = model_dir+ "/config.ini"
print("Config File Path:", config_file,flush=True)
assert os.path.isfile(config_file)
cp = ConfigParser()
cp.read(config_file)
output_dir = os.path.join(results_subdir, "classification_results/test")
print("Output Directory:", output_dir,flush=True)
if not os.path.isdir(output_dir):
os.makedirs(output_dir)
# default config
image_dimension = cp["TRAIN"].getint("image_dimension")
gan_resolution = resolution
batch_size = cp["TEST"].getint("batch_size")
use_best_weights = cp["TEST"].getboolean("use_best_weights")
if use_best_weights:
print("** Using BEST weights",flush=True)
model_weights_path = os.path.join(results_subdir, "classification_results/train/best_weights.h5")
else:
print("** Using LAST weights",flush=True)
model_weights_path = os.path.join(results_subdir, "classification_results/train/weights.h5")
print("** DenseNet Input Resolution:", image_dimension, flush=True)
print("** GAN Image Resolution:", gan_resolution, flush=True)
# get test sample count
test_dir = os.path.join(results_subdir, "inference/test")
shutil.copy(test_dir+"/test.csv", output_dir)
# Get class names
class_names = get_class_names(output_dir,"test")
tfrecord_dir_te = os.path.join(data_dir, "test")
test_counts, _ = get_sample_counts(output_dir, "test", class_names)
# get indicies (all of csv file for validation)
print("** test counts:", test_counts, flush=True)
# compute steps
test_steps = int(np.floor(test_counts / batch_size))
print("** test_steps:", test_steps, flush=True)
log2_record = int(np.log2(gan_resolution))
record_file_ending = "*"+ np.str(log2_record)+ ".tfrecords"
print("** resolution ", gan_resolution, " corresponds to ", record_file_ending, " TFRecord file.", flush=True)
# Get Model
# ------------------------------------
input_shape=(image_dimension, image_dimension, 3)
img_input = Input(shape=input_shape)
base_model = DenseNet121(
include_top = False,
weights = None,
input_tensor = img_input,
input_shape = input_shape,
pooling = "avg")
x = base_model.output
predictions = Dense(len(class_names), activation="sigmoid", name="predictions")(x)
model = Model(inputs=img_input, outputs = predictions)
print(" ** load model from:", model_weights_path, flush=True)
model.load_weights(model_weights_path)
# ------------------------------------
print("** load test generator **", flush=True)
test_seq = TFWrapper(
tfrecord_dir=tfrecord_dir_te,
record_file_endings = record_file_ending,
batch_size = batch_size,
model_target_size = (image_dimension, image_dimension),
steps = None,
augment=False,
shuffle=False,
prefetch=True,
repeat=False)
print("** make prediction **", flush=True)
test_seq.initialise()
x_all, y_all = test_seq.get_all_test_data()
print("X-Test Shape:", x_all.shape,flush=True)
print("Y-Test Shape:", y_all.shape,flush=True)
print("----------------------------------------", flush=True)
print("Test Model AUROC", flush=True)
y_pred = model.predict(x_all)
current_auroc = []
for i in range(len(class_names)):
try:
score = roc_auc_score(y_all[:, i], y_pred[:, i])
except ValueError:
score = 0
current_auroc.append(score)
print(i+1,class_names[i],": ", score, flush=True)
mean_auroc = np.mean(current_auroc)
print("Mean auroc: ", mean_auroc,flush=True)
print("----------------------------------------", flush=True)
downscale_factor = 8
num_models_to_use = 3
num_test_images = 100
print("Number of Models to use:", num_models_to_use, flush=True)
print("Number of Test images:", num_test_images, flush=True)
x_tr, y_tr = x_all[num_test_images:], y_all[num_test_images:]
x_te, y_te = x_all[0:num_test_images], y_all[0:num_test_images]
downsample_factors = (downscale_factor,downscale_factor)
print("Downsample Factors:", downsample_factors,flush=True)
model_builder = UNetModelBuilder(downsample_factors, num_layers=2, num_units=8, activation="relu",
p_dropout=0.0, verbose=0, batch_size=32, learning_rate=0.001)
print("Model build done.",flush=True)
masking_operation = ZeroMasking()
loss = categorical_crossentropy
explainer = CXPlain(model, model_builder, masking_operation, loss,
num_models=num_models_to_use, downsample_factors=downsample_factors, flatten_for_explained_model=False)
print("Explainer build done.",flush=True)
explainer.fit(x_tr, y_tr);
print("Explainer fit done.",flush=True)
try:
attr, conf = explainer.explain(x_te, confidence_level=0.80)
np.save(output_dir+"/x_cxpl.npy", x_te)
np.save(output_dir+"/y_cxpl.npy", y_te)
np.save(output_dir+"/attr.npy", attr)
np.save(output_dir+"/conf.npy", conf)
print("Explainer explain done and saved.",flush=True)
except Exception as ef: print(ef,flush=True)
from tokenizer_wrapper import TokenizerWrapper
import matplotlib.pyplot as plt
config_file = "./config.ini"
cp = ConfigParser()
cp.read(config_file)
class_names = cp["Captioning_Model"].get("class_names").split(",")
image_source_dir = cp["Data"].get("image_source_dir")
data_dir = cp["Data"].get("data_dir")
all_data_csv = cp['Data'].get('all_data_csv')
training_csv = cp['Data'].get('training_set_csv')
image_dimension = cp["Chexnet_Default"].getint("image_dimension")
batch_size = cp["Captioning_Model_Train"].getint("batch_size")
training_counts = get_sample_counts(data_dir, training_csv)
EPOCHS = cp["Captioning_Model_Train"].getint("epochs")
max_sequence_length = cp['Captioning_Model'].getint('max_sequence_length')
tokenizer_vocab_size = cp['Captioning_Model'].getint('tokenizer_vocab_size')
BUFFER_SIZE = cp["Captioning_Model"].getint("buffer_size")
embedding_dim = cp["Captioning_Model"].getint("embedding_dim")
units = cp["Captioning_Model"].getint("units")
checkpoint_path = cp["Captioning_Model_Train"].get("ckpt_path")
continue_from_last_ckpt = cp["Captioning_Model_Train"].getboolean(
"continue_from_last_ckpt")
# compute steps
steps = int(training_counts / batch_size)
print(f"** train_steps: {steps} **")
def test(model_dir, data_dir, results_subdir, random_seed, resolution):
np.random.seed(random_seed)
tf.set_random_seed(np.random.randint(1 << 31))
session_conf = tf.ConfigProto(intra_op_parallelism_threads=1,
inter_op_parallelism_threads=1)
sess = tf.Session(graph=tf.get_default_graph(), config=session_conf)
set_session(sess)
# parser config
config_file = model_dir + "/config.ini"
print("Config File Path:", config_file, flush=True)
assert os.path.isfile(config_file)
cp = ConfigParser()
cp.read(config_file)
# default config
image_dimension = cp["TRAIN"].getint("image_dimension")
batch_size = cp["TEST"].getint("batch_size")
use_best_weights = cp["TEST"].getboolean("use_best_weights")
print("** DenseNet input resolution:", image_dimension, flush=True)
print("** GAN image resolution:", resolution, flush=True)
log2_record = int(np.log2(resolution))
record_file_ending = "*" + np.str(log2_record) + ".tfrecords"
print("** Resolution ",
resolution,
" corresponds to ",
record_file_ending,
" TFRecord file.",
flush=True)
output_dir = os.path.join(
results_subdir,
"classification_results_res_" + np.str(2**log2_record) + "/test")
print("Output Directory:", output_dir, flush=True)
if not os.path.isdir(output_dir):
os.makedirs(output_dir)
if use_best_weights:
print("** Using BEST weights", flush=True)
model_weights_path = os.path.join(
results_subdir, "classification_results_res_" +
np.str(2**log2_record) + "/train/best_weights.h5")
else:
print("** Using LAST weights", flush=True)
model_weights_path = os.path.join(
results_subdir, "classification_results_res_" +
np.str(2**log2_record) + "/train/weights.h5")
# get test sample count
shutil.copy(results_subdir[:-4] + "/test/test.csv", output_dir)
tfrecord_dir_te = os.path.join(data_dir, "test")
class_names = get_class_names(output_dir, "test")
test_counts, _ = get_sample_counts(output_dir, "test", class_names)
# get indicies (all of csv file for validation)
print("** test counts:", test_counts, flush=True)
# compute steps
test_steps = int(np.floor(test_counts / batch_size))
print("** test_steps:", test_steps, flush=True)
# Get Model
# ------------------------------------
input_shape = (image_dimension, image_dimension, 3)
img_input = Input(shape=input_shape)
base_model = DenseNet121(include_top=False,
weights=None,
input_tensor=img_input,
input_shape=input_shape,
pooling="avg")
x = base_model.output
predictions = Dense(len(class_names),
activation="sigmoid",
name="predictions")(x)
model = Model(inputs=img_input, outputs=predictions)
print(" ** load model from:", model_weights_path, flush=True)
model.load_weights(model_weights_path)
# ------------------------------------
print("** load test generator **", flush=True)
test_seq = TFWrapper(tfrecord_dir=tfrecord_dir_te,
record_file_endings=record_file_ending,
batch_size=batch_size,
model_target_size=(image_dimension, image_dimension),
steps=None,
augment=False,
shuffle=False,
prefetch=True,
repeat=False)
print("** make prediction **", flush=True)
test_seq.initialise() #MAKE SURE REINIT
y_hat = model.predict_generator(test_seq, workers=0)
test_seq.initialise() #MAKE SURE REINIT
y = test_seq.get_y_true()
test_log_path = os.path.join(output_dir, "test.log")
print("** write log to", test_log_path, flush=True)
aurocs = []
tpr_fpr_thr = []
with open(test_log_path, "w") as f:
for i in range(len(class_names)):
tpr, fpr, thr = roc_curve(y[:, i], y_hat[:, i])
roc_rates = np.concatenate(
(fpr.reshape(-1, 1), tpr.reshape(-1, 1), thr.reshape(-1, 1)),
axis=1)
tpr_fpr_thr.append(roc_rates)
try:
score = roc_auc_score(y[:, i], y_hat[:, i])
if score < 0.5:
score = 1. - score
aurocs.append(score)
except ValueError:
score = 0
f.write(np.str(class_names[i]) + " : " + np.str(score) + "\n")
mean_auroc = np.mean(aurocs)
f.write("-------------------------\n")
f.write("mean auroc: " + np.str(mean_auroc) + "\n")
print("mean auroc:", mean_auroc, flush=True)
roc_char = np.asarray(tpr_fpr_thr)
np.save(output_dir + "/roc_char.npy", roc_char)
print("Saved ROC data (TPR, FPR, THR) to:",
output_dir + "/roc_char.npy",
flush=True)
def nn(model_dir, data_dir, results_subdir, random_seed, resolution):
np.random.seed(random_seed)
tf.set_random_seed(np.random.randint(1 << 31))
session_conf = tf.ConfigProto(intra_op_parallelism_threads=1,
inter_op_parallelism_threads=1)
sess = tf.Session(graph=tf.get_default_graph(), config=session_conf)
set_session(sess)
# parser config
config_file = model_dir + "/config.ini"
print("Config File Path:", config_file, flush=True)
assert os.path.isfile(config_file)
cp = ConfigParser()
cp.read(config_file)
output_dir = os.path.join(results_subdir, "classification_results/nn")
train_outdir = os.path.join(results_subdir, "classification_results/train")
print("Output Directory:", output_dir, flush=True)
# default config
image_dimension = cp["TRAIN"].getint("image_dimension")
gan_resolution = resolution
batch_size = cp["TEST"].getint("batch_size")
use_best_weights = cp["TEST"].getboolean("use_best_weights")
if use_best_weights:
print("** Using BEST weights", flush=True)
model_weights_path = os.path.join(
results_subdir, "classification_results/nn/best_weights.h5")
else:
print("** Using LAST weights", flush=True)
model_weights_path = os.path.join(
results_subdir, "classification_results/nn/weights.h5")
print("** DenseNet Input Resolution:", image_dimension, flush=True)
print("** GAN Image Resolution:", gan_resolution, flush=True)
tfrecord_dir_tr = os.path.join(data_dir, "train")
tfrecord_dir_te = os.path.join(results_subdir, "inference/test")
# Get class names
class_names = get_class_names(train_outdir, "train")
counts, _ = get_sample_counts(train_outdir, "train", class_names)
# get indicies (all of csv file for validation)
print("** counts:", counts, flush=True)
# compute steps
train_steps = int(np.floor(counts / batch_size))
print("** t_steps:", train_steps, flush=True)
log2_record = int(np.log2(gan_resolution))
record_file_ending = "*" + np.str(log2_record) + ".tfrecords"
print("** resolution ",
gan_resolution,
" corresponds to ",
record_file_ending,
" TFRecord file.",
flush=True)
# Get Model
# ------------------------------------
input_shape = (image_dimension, image_dimension, 3)
img_input = Input(shape=input_shape)
base_model = DenseNet121(include_top=False,
weights=None,
input_tensor=img_input,
input_shape=input_shape,
pooling="avg")
x = base_model.output
predictions = Dense(len(class_names),
activation="sigmoid",
name="predictions")(x)
model = Model(inputs=img_input, outputs=predictions)
print(" ** load model from:", model_weights_path, flush=True)
model.load_weights(model_weights_path)
# ------------------------------------
# Extract representation layer output:
layer_name = 'avg_pool'
intermediate_layer_model = Model(
inputs=model.input, outputs=model.get_layer(layer_name).output)
#intermediate_output = intermediate_layer_model(data)
def renorm_and_save_npy(x, name):
imagenet_mean = np.array([0.485, 0.456, 0.406])
imagenet_std = np.array([0.229, 0.224, 0.225])
x = x * imagenet_std + imagenet_mean
save_path = output_dir + "/" + name + ".npy"
np.save(save_path, x)
print("** save npy images under: ", save_path, flush=True)
def save_array(x, name):
save_path = output_dir + "/" + name + ".npy"
np.save(save_path, x)
print("** save npy images under: ", save_path, flush=True)
# Load test Inference images
test_bs = 200
print("** load inference images, save random n=", test_bs, flush=True)
test_seq = TFWrapper(tfrecord_dir=tfrecord_dir_te,
record_file_endings=record_file_ending,
batch_size=test_bs,
model_target_size=(image_dimension, image_dimension),
steps=None,
augment=False,
shuffle=False,
prefetch=True,
repeat=False)
test_seq.initialise()
x, x_orig, x_label = test_seq.__getitem__(0)
renorm_and_save_npy(x, name="real_inf_224")
renorm_and_save_npy(x_orig, name="real_inf_256")
save_array(x_label, name="real_inf_label")
print("** Compute inf latent rep **", flush=True)
x_latrep = intermediate_layer_model.predict(x)
print("** Latent Size: ", x_latrep.shape, flush=True)
# Load train Inference images
print("** load train generator **", flush=True)
train_seq = TFWrapper(tfrecord_dir=tfrecord_dir_tr,
record_file_endings=record_file_ending,
batch_size=batch_size,
model_target_size=(image_dimension, image_dimension),
steps=train_steps,
augment=False,
shuffle=False,
prefetch=True,
repeat=False)
train_seq.initialise()
print("** generator loaded **", flush=True)
# Loop through training data and compute minimums
H, H_orig = image_dimension, 256
W, W_orig = image_dimension, 256
D = 3
BS = batch_size
n = test_bs
LS = x_latrep.shape[1]
cur_nn_imgs = np.zeros((n, H, W, D)) #Current nn images
cur_nn_imgs_orig = np.zeros((n, H_orig, W_orig, D))
cur_nn_labels = np.zeros((n, x_label.shape[1]))
cur_cos_min = np.ones((n, 1)) * 10000 #Current minimum cosine distance
time_old = time.time()
print("** Start nn determination **", flush=True)
for i in range(0, train_steps):
# Get batch images and lat. reps
y, y_orig, y_label = train_seq.__getitem__(i) #[BS,H,W,D]
y_latrep = intermediate_layer_model.predict(y) #[BS,LS]
#y_reshaped = y.reshape([BS,1,H,W,D]) #Reshape for tiling [BS,1,H,W,D]
#y_orig_reshaped = y_orig.reshape([BS,1,H_orig,W_orig,D])
#y_label_reshaped = y_label.reshape([BS,1,x_label.shape[1]])
y_tiled = np.tile(y, [1, n, 1, 1, 1]) #Tile: [BS,n,H,W,D]
y_orig_tiled = np.tile(y_orig, [1, n, 1, 1, 1])
y_label_tiled = np.tile(y_label, [1, n, 1])
cosdis = np.ones(
(n, BS)) - cosine_similarity(x_latrep, y_latrep) #[n,BS]
argmin_cosdis = np.argmin(cosdis, axis=1) #[n,1]
min_cosdis = np.min(cosdis, axis=1).reshape(n, 1) #[n,1]
min_y = y_tiled[:, argmin_cosdis].reshape(
n, H, W, D) #[n,H,W,D]: Min. Cosdis for each inf_img from batch
min_y_orig = y_orig_tiled[:,
argmin_cosdis].reshape(n, H_orig, W_orig, D)
min_ylabel = y_label_tiled[:, argmin_cosdis].reshape(
(n, x_label.shape[1]))
t = np.where(
min_cosdis < cur_cos_min
) #Indicies where min. cosdistance is smaller then current
cur_cos_min[t[0]] = min_cosdis[t[0]] #Update current cosdis minima
cur_nn_imgs[t[0]] = min_y[t[0]] #Update current nn images
cur_nn_imgs_orig[t[0]] = min_y_orig[t[0]]
cur_nn_labels[t[0]] = min_ylabel[t[0]]
if i % 100 == 0 and i > 0:
time_new = time.time()
print("Iteration ", i, "/", train_steps,
"took %.2f seconds" % (time_new - time_old))
time_old = time_new
print("Current mean cos-distance:", np.mean(cur_cos_min))
print("** Loop Done **", flush=True)
renorm_and_save_npy(cur_nn_imgs, name="nn_images_224")
renorm_and_save_npy(cur_nn_imgs_orig, name="nn_images_256")
save_array(cur_cos_min, name="cosdistance_minimum")
save_array(cur_nn_labels, name="nn_labels")
def main():
# parser config
config_file = "./config.ini"
cp = ConfigParser()
cp.read(config_file)
# default config
weights_dir = cp["DEFAULT"].get("weights_dir")
base_model_name = cp["DEFAULT"].get("base_model_name")
chexnet_class_names = cp["DEFAULT"].get("chexnet_class_names").split(",")
class_names = cp["DEFAULT"].get("class_names").split(",")
image_source_dir = cp["DEFAULT"].get("image_source_dir")
data_dir = cp["DEFAULT"].get("data_dir")
# train config
image_dimension = cp["TRAIN"].getint("image_dimension")
# test config
batch_size = cp["TEST"].getint("batch_size")
test_steps = cp["TEST"].get("test_steps")
# parse weights file path
output_weights_name = cp["TRAIN"].get("output_weights_name")
weights_path = os.path.join(weights_dir, output_weights_name)
# get test sample count
test_counts = get_sample_counts(data_dir, "all_data", class_names)
# compute steps
if test_steps == "auto":
test_steps = int(test_counts / batch_size)
else:
try:
test_steps = int(test_steps)
except ValueError:
raise ValueError(f"""
test_steps: {test_steps} is invalid,
please use 'auto' or integer.
""")
print(f"** test_steps: {test_steps} **")
print("** load model **")
model_weights_path = weights_path
model_factory = ModelFactory()
model = model_factory.get_model(chexnet_class_names,
model_name=base_model_name,
use_base_weights=False,
weights_path=model_weights_path,
pop_last_layer=True)
model.summary()
print("** load test generator **")
test_sequence = AugmentedImageSequence(
dataset_csv_file=os.path.join(data_dir, "all_data.csv"),
class_names=class_names,
source_image_dir=image_source_dir,
batch_size=batch_size,
target_size=(image_dimension, image_dimension),
augmenter=None,
steps=test_steps,
shuffle_on_epoch_end=False,
)
print("** make prediction **")
image, y = test_sequence.__getitem__(4)
y_hat = model.predict(image)
# y_hat = model.predict_generator(test_sequence, verbose=1)
# y = test_sequence.get_y_true()
print(y_hat.shape)
def main():
# Instantiate config parser
# as long as a configuration file is in the local directory of this training code
# it will be utilized by the training script
# TODO : Add a README for the configuration file used to configure this training cycle
config_file = "./sample_config.ini"
cp = ConfigParser()
cp.read(config_file)
# set a bunch of default config
output_directory = cp["DEFAULT"].get("output_directory")
image_source_directory = cp["DEFAULT"].get("image_source_directory")
# TODO tie in base model name for model verioning for SavedModels
base_model_name = cp["DEFAULT"].get("base_model_name")
# Class names are passed in as array within the configuration script
class_names = cp["DEFAULT"].get("class_names").split(",")
model_version = cp["DEFAULT"].get("model_version")
tensorboard_log_dir = cp["DEFAULT"].get("tensorboard_log_dir")
# training configuration
# See sample_config.ini for explanation of all of the parameters
use_base_model_weights = cp["TRAIN"].getboolean("use_base_model_weights")
use_trained_model_weights = cp["TRAIN"].getboolean("use_trained_model_weights")
use_best_weights = cp["TRAIN"].getboolean("use_best_weights")
output_weights_name = cp["TRAIN"].get("output_weights_name")
epochs = cp["TRAIN"].getint("epochs")
batch_size = cp["TRAIN"].getint("batch_size")
initial_learning_rate = cp["TRAIN"].getfloat("initial_learning_rate")
generator_workers = cp["TRAIN"].getint("generator_workers")
image_dimension = cp["TRAIN"].getint("image_dimension")
train_steps = cp["TRAIN"].get("train_steps")
patience_reduce_lr = cp["TRAIN"].getint("patience_reduce_lr")
min_learning_rate = cp["TRAIN"].getfloat("min_learning_rate")
validation_steps = cp["TRAIN"].get("validation_steps")
positive_weights_multiply = cp["TRAIN"].getfloat("positive_weights_multiply")
dataset_csv_dir = cp["TRAIN"].get("dataset_csv_dir")
if use_trained_model_weights:
print("<<< Using pretrained model weights! >>>")
training_stats_file = os.path.join(output_directory, ".training_stats.json")
if os.path.isfile(training_stats_file):
training_stats = json.load(open(training_stats_file))
else:
training_stats = {}
else:
# start over again
training_stats = {}
show_model_summary = cp["TRAIN"].getboolean("show_model_summary")
# end configuration parser
utility.check_create_output_dir(output_directory)
utility.create_tensorboard_log_dir(tensorboard_log_dir)
try:
utility.backup_config_file(output_directory, config_file)
datasets = ["train", "validation", "test"]
for dataset in datasets:
shutil.copy(os.path.join(dataset_csv_dir, f"{dataset}.csv"), output_directory)
train_counts, train_pos_counts = utility.get_sample_counts(output_directory, "train", class_names)
validation_counts, _ = utility.get_sample_counts(output_directory, "validation", class_names)
# compute steps
# train steps var defined in config ini file
# if set to standard auto, normalize train_steps
# wrt batch_size, otherwise take user input
if train_steps == "auto":
train_steps = int(train_counts / batch_size)
else:
try:
train_steps = int(train_steps)
except:
raise ValueError(f"""
train_steps : {train_steps} is invalid,
please use 'auto' or specify an integer.
""")
print(f" <<< train_steps : {train_steps} >>>")
if validation_steps == "auto":
validation_steps = int(validation_counts / batch_size)
else:
try:
validation_steps = int(validation_steps)
except:
raise ValueError(f"""
validation_steps : {validation_steps} is invalid,
please use 'auto' or specify an integer.
""")
print(f" <<< validation_steps : {validation_steps} >>>")
# class weights
class_weights = utility.get_class_weights(
train_counts,
train_pos_counts,
multiply=positive_weights_multiply,
)
print(f"class_weights : {class_weights}")
print(" <<< Loading Model >>>")
if use_trained_model_weights:
if use_best_weights:
model_weights_file = os.path.join(output_directory, f"best_{output_weights_name}")
else:
model_weights_file = os.path.join(output_directory, output_weights_name)
else:
model_weights_file = None
model_factory = modelwrap.Models()
model = model_factory.get_model(
class_names=class_names,
use_base_weights=use_base_model_weights,
weights_path=model_weights_file,
input_shape=(image_dimension,image_dimension,3)
)
if show_model_summary:
print(model.summary())
print(" <<< Creating Image Generators >>> ")
train_sequence = generator.AugmentedImageSequence(
dataset_csv_file=os.path.join(output_directory, "train.csv"),
class_names=class_names,
source_image_dir=image_source_directory,
batch_size=batch_size,
target_size=(image_dimension, image_dimension),
augmenter=utility.augmenter(),
steps=train_steps,
)
validation_sequence = generator.AugmentedImageSequence(
dataset_csv_file=os.path.join(output_directory, "validation.csv"),
class_names=class_names,
source_image_dir=image_source_directory,
batch_size=batch_size,
target_size=(image_dimension, image_dimension),
augmenter=utility.augmenter(),
steps=validation_steps,
shuffle_on_epoch_end=False,
)
output_weights_path = os.path.join(output_directory, output_weights_name)
print(f" <<< Set Output Weights Path to : {output_weights_path}")
# TODO implement multi-gpu support
model_train = model
checkpoint = ModelCheckpoint(
output_weights_path,
save_weights_only=True,
save_best_only=True,
verbose=1
)
print(" <<< Compile model and class weights >>>")
optimizer = Adam(lr=initial_learning_rate)
model_train.compile(
optimizer=optimizer, loss="binary_crossentropy"
)
auroc = MultiClassAUROC(
sequence=validation_sequence,
class_names=class_names,
weights_path=output_weights_path,
stats=training_stats,
workers=generator_workers,
)
# serving_checkpoint = ServingCheckpoint(
# output_directory=output_directory,
# model=model,
# model_version=model_version,
# )
callbacks =[
checkpoint,
TensorBoard(log_dir=os.path.join(tensorboard_log_dir), batch_size=batch_size),
ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=patience_reduce_lr,
verbose=1, mode="min", min_lr=min_learning_rate),
auroc
]
print(" <<< Starting Model Training >>> ")
history = model_train.fit_generator(
generator=train_sequence,
steps_per_epoch=train_steps,
epochs=epochs,
validation_data=validation_sequence,
validation_steps=validation_steps,
callbacks=callbacks,
class_weight=class_weights,
workers=generator_workers,
shuffle=False,
)
model_class_weights = tf.convert_to_tensor(model_train.layers[-1].get_weights()[0], tf.float32)
model_final_conv_layer = utility.get_output_layer(model_train, "bn")
tensor_info_input = tf.saved_model.utils.build_tensor_info(model_train.input)
tensor_info_output = tf.saved_model.utils.build_tensor_info(model_train.output)
tensor_info_class_weights = tf.saved_model.utils.build_tensor_info(model_class_weights)
tensor_info_final_conv_layer = tf.saved_model.utils.build_tensor_info(model_final_conv_layer.output)
# export model for serving
export_base_path = output_directory
export_path = os.path.join(
tf.compat.as_bytes(export_base_path),
tf.compat.as_bytes(model_version)
)
prediction_signature = (
tf.saved_model.signature_def_utils.build_signature_def(
inputs={'images': tensor_info_input},
outputs={'prediction': tensor_info_output, 'class_weights': tensor_info_class_weights, 'final_conv_layer': tensor_info_final_conv_layer},
method_name=tf.saved_model.signature_constants.PREDICT_METHOD_NAME
)
)
print(f" <<< Exporting Trained Model to {export_path} >>> ")
builder = tf.saved_model.builder.SavedModelBuilder(export_path)
with K.get_session() as sess:
builder.add_meta_graph_and_variables(
sess=sess,
tags=[tf.saved_model.tag_constants.SERVING],
signature_def_map={'predict': prediction_signature}
)
builder.save()
print(" <<< Export History >>>")
with open(os.path.join(output_directory, "history.pkl"), "wb") as f:
pickle.dump({
"history": history.history,
"auroc": auroc.aurocs,
}, f)
print(" <<< Export Complete! >>> ")
finally:
utility.delete_training_lock(output_directory)
def main():
# parser config
config_file = "./config.ini"
cp = ConfigParser()
cp.read(config_file)
# default config
output_dir = cp["DEFAULT"].get("output_dir")
if not os.path.exists(output_dir):
os.mkdir(output_dir)
base_model_name = cp["DEFAULT"].get("base_model_name")
class_names = cp["DEFAULT"].get("class_names").split(",")
image_source_dir = cp["DEFAULT"].get("image_source_dir")
data_set_dir = cp["TRAIN"].get("dataset_csv_dir")
# train config
image_dimension = cp["TRAIN"].getint("image_dimension")
# test config
batch_size = cp["TEST"].getint("batch_size")
test_steps = cp["TEST"].get("test_steps")
use_best_weights = cp["TEST"].getboolean("use_best_weights")
# parse weights file path
input_weights_name = cp["TRAIN"].get("input_weights_name")
weights_path = os.path.join(data_set_dir, input_weights_name)
best_weights_path = os.path.join(data_set_dir,
"best_{}".format(input_weights_name))
# get test sample count
test_counts, _ = get_sample_counts(data_set_dir, "test", class_names)
# compute steps
if test_steps == "auto":
test_steps = int(test_counts / batch_size)
else:
try:
test_steps = int(test_steps)
except ValueError:
raise ValueError("""
test_steps: {} is invalid,
please use 'auto' or integer.
""".format(test_steps))
print("** test_steps: {} **".format(test_steps))
print("** load model **")
if use_best_weights:
print("** use best weights **")
model_weights_path = best_weights_path
else:
print("** use last weights **")
model_weights_path = weights_path
model_factory = ModelFactory()
model = model_factory.get_model(class_names,
model_name=base_model_name,
use_base_weights=False,
weights_path=model_weights_path)
print("** load test generator **")
test_sequence = AugmentedImageSequence(
# dataset_csv_file=os.path.join(output_dir, "dev.csv"),
dataset_csv_file=os.path.join(data_set_dir, "test.csv"),
# dataset_csv_file=os.path.join(data_set_dir, "MIMIC_dataset.csv"),
class_names=class_names,
source_image_dir=image_source_dir,
batch_size=batch_size,
target_size=(image_dimension, image_dimension),
augmenter=None,
steps=test_steps,
shuffle_on_epoch_end=False,
)
# test_sequence.dataset_df.to_csv(os.path.join(output_dir, 'test_data_frame.csv'))
print("** make prediction **")
y_hat = model.predict_generator(test_sequence, verbose=1)
y = test_sequence.get_y_true()
# np.savetxt(os.path.join(output_dir, 'y_hat_1205_default_weight.txt'), y_hat)
np.savetxt(os.path.join(output_dir, 'y_0430.txt'), y)
test_log_path = os.path.join(output_dir, "test.log")
print("** write log to {} **".format(test_log_path))
aurocs = []
with open(test_log_path, "w") as f:
for i in range(len(class_names)):
try:
score = roc_auc_score(y[:, i], y_hat[:, i])
aurocs.append(score)
except ValueError:
score = 0
f.write("{}: {}\n".format(class_names[i], score))
mean_auroc = np.mean(aurocs)
f.write("-------------------------\n")
f.write("mean auroc: {}\n".format(mean_auroc))
print("mean auroc: {}".format(mean_auroc))
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--model_epoch', type=int, default=0)
args = parser.parse_args()
# Set Parameter #
base_model_name = "DenseNet121"
use_base_model_weights = True
weights_path = None
image_dimension = 224
batch_size = 32
epochs = 20
class_names = ["Nodule", "Pneumothorax"]
csv_path = './data/classification'
image_source_dir = '/media/nfs/CXR/NIH/chest_xrays/NIH/data/images_1024x1024/'
augmenter = None
# If train_steps is set to None, will calculate train steps by len(train)/batch_size
train_steps = None
positive_weights_multiply = 1
outputs_path = './experiments/ae'
weights_name = f'weights{args.model_epoch}.h5'
output_weights_path = os.path.join(outputs_path, weights_name)
initial_learning_rate = 0.0001
training_stats = {}
# Get Sample and Total Count From Training Data and Compute Class Weights #
train_counts, train_pos_counts = get_sample_counts(csv_path, "train",
class_names)
if train_steps == None:
train_steps = int(train_counts / batch_size)
dev_counts, _ = get_sample_counts(csv_path, "test", class_names)
validation_steps = int(dev_counts / batch_size)
print('***Compute Class Weights***')
class_weights = get_class_weights(train_counts,
train_pos_counts,
multiply=positive_weights_multiply)
print(class_weights)
# Create Image Sequence #
train_sequence = AugmentedImageSequence(
dataset_csv_file=os.path.join(csv_path, "train.csv"),
class_names=class_names,
source_image_dir=image_source_dir,
batch_size=batch_size,
target_size=(image_dimension, image_dimension),
augmenter=augmenter,
steps=train_steps,
model_epoch=args.model_epoch)
validation_sequence = AugmentedImageSequence(
dataset_csv_file=os.path.join(csv_path, "test.csv"),
class_names=class_names,
source_image_dir=image_source_dir,
batch_size=batch_size,
target_size=(image_dimension, image_dimension),
augmenter=augmenter,
steps=validation_steps,
shuffle_on_epoch_end=False,
model_epoch=args.model_epoch)
# Build Model #
factory = ModelFactory()
model = factory.get_model(class_names,
model_name=base_model_name,
use_base_weights=use_base_model_weights,
weights_path=None,
input_shape=(image_dimension, image_dimension,
3))
print("** check multiple gpu availability **")
gpus = len(os.getenv("CUDA_VISIBLE_DEVICES", "1").split(","))
if gpus > 1:
print("** multi_gpu_model is used! gpus={gpus} **")
model_train = multi_gpu_model(model, gpus)
# FIXME: currently (Keras 2.1.2) checkpoint doesn't work with multi_gpu_model
checkpoint = MultiGPUModelCheckpoint(
filepath=output_weights_path,
base_model=model,
)
else:
model_train = model
checkpoint = ModelCheckpoint(
output_weights_path,
save_weights_only=True,
save_best_only=True,
verbose=1,
)
auroc = MultipleClassAUROC(sequence=validation_sequence,
class_names=class_names,
weights_path=output_weights_path,
stats=training_stats,
workers=8,
model_epoch=args.model_epoch)
callbacks = [
checkpoint,
TensorBoard(log_dir=os.path.join(outputs_path, "logs"),
batch_size=batch_size),
ReduceLROnPlateau(monitor='val_loss',
factor=0.1,
patience=1,
verbose=1,
mode="min",
min_lr=1e-8),
auroc,
]
# Compile Model #
print('*** Start Compiling ***')
optimizer = Adam(lr=initial_learning_rate)
model_train.compile(optimizer=optimizer, loss="binary_crossentropy")
# Train #
print("** start training **")
history = model_train.fit_generator(
generator=train_sequence,
steps_per_epoch=train_steps,
epochs=epochs,
validation_data=validation_sequence,
validation_steps=validation_steps,
callbacks=callbacks,
class_weight=class_weights,
workers=8,
shuffle=False,
)
# dump history
print("** dump history **")
with open(os.path.join(outputs_path, f"history{args.model_epoch}.pkl"),
"wb") as f:
pickle.dump({
"history": history.history,
"auroc": auroc.aurocs,
}, f)
print("** done! **")
from PIL import Image
config_file = "./config.ini"
cp = ConfigParser()
cp.read(config_file)
class_names = cp["Captioning_Model"].get("class_names").split(",")
image_source_dir = cp["Data"].get("image_source_dir")
data_dir = cp["Data"].get("data_dir")
all_data_csv = cp['Data'].get('all_data_csv')
testing_csv = cp['Data'].get('training_set_csv')
image_dimension = cp["Chexnet_Default"].getint("image_dimension")
batch_size = cp["Captioning_Model_Inference"].getint("batch_size")
testing_counts = get_sample_counts(data_dir, testing_csv)
max_sequence_length = cp['Captioning_Model'].getint('max_sequence_length')
tokenizer_vocab_size = cp['Captioning_Model'].getint('tokenizer_vocab_size')
# These two variables represent that vector shape
features_shape = cp["Captioning_Model"].getint("features_shape")
attention_features_shape = cp["Captioning_Model"].getint(
"attention_features_shape")
BUFFER_SIZE = cp["Captioning_Model"].getint("buffer_size")
embedding_dim = cp["Captioning_Model"].getint("embedding_dim")
units = cp["Captioning_Model"].getint("units")
checkpoint_path = cp["Captioning_Model_Train"].get("ckpt_path")
def main():
# parser config
config_file = "../../config.ini"
cp = ConfigParser()
cp.read(config_file)
# test config
output_dir = cp["TEST"].get("output_dir")
batch_size = cp["TEST"].getint("batch_size")
model_name = cp["TEST"].get("model_name")
# parse weights file path
output_weights_name = cp["TRAIN"].get("output_weights_name")
weights_path = os.path.join(output_dir, output_weights_name)
time_steps = cp["TRAIN"].getint("time_steps")
input_dim_openface = cp["TRAIN"].getint("input_dim_openface")
input_dim_openpose = cp["TRAIN"].getint("input_dim_openpose")
input_dim_rgb400 = cp["TRAIN"].getint("input_dim_rgb400")
csv_source_dir_openface = cp["DEFAULT"].get("csv_source_dir_openface")
csv_source_dir_openpose = cp["DEFAULT"].get("csv_source_dir_openpose")
csv_source_dir_rgb400 = cp["DEFAULT"].get("csv_source_dir_rgb400")
featrue_name = cp["TEST"].get("featrue_name")
cuDNN = cp["TRAIN"].getboolean("cuDNN")
bidirect = cp["TEST"].getboolean("bidirect")
units = cp["TEST"].getint("units")
layers = cp["TEST"].getint("layers")
with_FC_dropout_layers = cp["TEST"].getboolean("with_FC_dropout_layers")
initializer = cp["TEST"].get("initializer")
regularizers_fuc = cp["TEST"].get("regularizers_fuc")
regularizers_l1 = cp["TEST"].getfloat("regularizers_l1")
regularizers_l2 = cp["TEST"].getfloat("regularizers_l2")
if regularizers_fuc == 'l1':
regularizers_fuc = regularizers.l1(regularizers_l1)
elif regularizers_fuc == 'l2':
regularizers_fuc = regularizers.l2(regularizers_l2)
elif regularizers_fuc == 'l1_l2':
regularizers_fuc = regularizers.l1_l2(regularizers_l1, regularizers_l2)
else:
regularizers_fuc = None
units_layers = []
[units_layers.append(units) for i in range(layers)]
test_csv = cp["TEST"].get("test_csv")
# compute steps
# test_counts = get_sample_counts("test")
test_counts = get_sample_counts("val")
test_steps = ceil(test_counts / batch_size)
print(f"** test_steps: {test_steps} **")
print("** load model **")
model_factory = ModelFactory(regularizers=regularizers_fuc,
initializer=initializer,
with_FC_dropout_layers=with_FC_dropout_layers)
model_fun = getattr(model_factory, f'get_model_{model_name}')
model = model_fun(
TIME_STEPS=time_steps,
INPUT_DIM=eval(f'input_dim_{featrue_name}'),
weights_path=weights_path,
CuDNN=cuDNN,
bidirect=bidirect,
units=units_layers,
)
print("** load test generator **")
test_sequence = FeatruesSequence(
dataset_csv_file=os.path.join('data', test_csv),
csv_source_dir=eval(f'csv_source_dir_{featrue_name}'),
batch_size=batch_size,
shuffle_on_epoch_end=False,
test=True)
gpus = len(os.getenv("CUDA_VISIBLE_DEVICES", "0").split(","))
if gpus > 1:
print(f"** multi_gpu_model is used! gpus={gpus} **")
model_train = multi_gpu_model(model, gpus)
else:
model_train = model
print("** make prediction **")
# model_train.compile(optimizer=Adam(), loss="mean_squared_error")
prob_array = model_train.predict_generator(test_sequence,
steps=test_steps,
max_queue_size=8,
workers=8,
use_multiprocessing=True,
verbose=1)
print(prob_array)
list(map(out_put_prediction, prob_array))
print(pred_list)
df_test = pd.read_csv(os.path.join('data', test_csv))
df_test['predictions'] = pred_list
df_test['probs'] = prob_array
df_test[['openface_file', 'predictions',
'probs']].to_csv(os.path.join(output_dir, 'result.csv'),
index=False)
def main():
# parser config
config_file = "./config.ini"
cp = ConfigParser()
cp.read(config_file)
# default config
output_dir = cp["DEFAULT"].get("output_dir")
class_names = cp["DEFAULT"].get("class_names").split(",")
# test config
batch_size = cp["TEST"].getint("batch_size")
use_best_weights = cp["TEST"].getboolean("use_best_weights")
# parse weights file path
output_weights_name = cp["TRAIN"].get("output_weights_name")
weights_path = os.path.join(output_dir, output_weights_name)
best_weights_path = os.path.join(output_dir, f"best_{output_weights_name}")
# get test sample count
test_counts, _ = get_sample_counts(output_dir, "test", class_names)
symlink_dir_name = "image_links"
test_data_path = f"{output_dir}/{symlink_dir_name}/test/"
step_test = int(test_counts / batch_size)
print("** load test generator **")
test_generator = custom_image_generator(
ImageDataGenerator(horizontal_flip=True, rescale=1. / 255),
test_data_path,
batch_size=batch_size,
class_names=class_names, cam=False
)
test_generator_orig = custom_image_generator(
ImageDataGenerator(horizontal_flip=True, rescale=1. / 255),
test_data_path,
batch_size=batch_size,
class_names=class_names, cam=True)
x, y = load_generator_data(test_generator, step_test, len(class_names))
x_orig, _ = load_generator_data(test_generator_orig, step_test, len(class_names))
print("** load model **")
model = get_model(class_names)
if use_best_weights:
print("** use best weights **")
model.load_weights(best_weights_path)
else:
print("** use last weights **")
model.load_weights(weights_path)
print("** make prediction **")
y_hat = model.predict(x, verbose=1)
test_log_path = os.path.join(output_dir, "test.log")
print(f"** write log to {test_log_path} **")
aurocs = []
with open(test_log_path, "w") as f:
for i in range(len(class_names)):
try:
score = roc_auc_score(y[i], y_hat[i])
aurocs.append(score)
except ValueError:
score = 0
f.write(f"{class_names[i]}: {score}\n")
mean_auroc = np.mean(aurocs)
f.write("-------------------------\n")
f.write(f"mean auroc: {mean_auroc}\n")
grad_cam(model, class_names, y, y_hat, x, x_orig)
def train(model_dir, results_subdir, random_seed, resolution):
np.random.seed(random_seed)
tf.set_random_seed(np.random.randint(1 << 31))
session_conf = tf.ConfigProto(intra_op_parallelism_threads=1,
inter_op_parallelism_threads=1)
session_conf.gpu_options.allow_growth = True
sess = tf.Session(graph=tf.get_default_graph(), config=session_conf)
set_session(sess)
# parser config
config_file = model_dir + "/config.ini"
print("Config File Path:", config_file, flush=True)
assert os.path.isfile(config_file)
cp = ConfigParser()
cp.read(config_file)
# default config
base_model_name = cp["DEFAULT"].get("base_model_name")
# train config
path_model_base_weights = cp["TRAIN"].get("path_model_base_weights")
use_trained_model_weights = cp["TRAIN"].getboolean(
"use_trained_model_weights")
use_best_weights = cp["TRAIN"].getboolean("use_best_weights")
output_weights_name = cp["TRAIN"].get("output_weights_name")
epochs = cp["TRAIN"].getint("epochs")
batch_size = cp["TRAIN"].getint("batch_size")
initial_learning_rate = cp["TRAIN"].getfloat("initial_learning_rate")
image_dimension = cp["TRAIN"].getint("image_dimension")
patience_reduce_lr = cp["TRAIN"].getint("patience_reduce_lr")
min_lr = cp["TRAIN"].getfloat("min_lr")
positive_weights_multiply = cp["TRAIN"].getfloat(
"positive_weights_multiply")
patience = cp["TRAIN"].getint("patience")
samples_per_epoch = cp["TRAIN"].getint("samples_per_epoch")
reduce_lr = cp["TRAIN"].getfloat("reduce_lr")
print("** DenseNet input resolution:", image_dimension, flush=True)
print("** GAN image resolution:", resolution, flush=True)
print("** Patience epochs", patience, flush=True)
print("** Samples per epoch:", samples_per_epoch, flush=True)
log2_record = int(np.log2(resolution))
record_file_ending = "*" + np.str(log2_record) + ".tfrecords"
print("** Resolution ",
resolution,
" corresponds to ",
record_file_ending,
" TFRecord file.",
flush=True)
output_dir = os.path.join(
results_subdir,
"classification_results_res_" + np.str(2**log2_record) + "/train")
print("Output Directory:", output_dir, flush=True)
if not os.path.isdir(output_dir):
os.makedirs(output_dir)
# if previously trained weights is used, never re-split
if use_trained_model_weights:
print("** use trained model weights **", flush=True)
training_stats_file = os.path.join(output_dir, ".training_stats.json")
if os.path.isfile(training_stats_file):
# TODO: add loading previous learning rate?
training_stats = json.load(open(training_stats_file))
else:
training_stats = {}
else:
# start over
training_stats = {}
show_model_summary = cp["TRAIN"].getboolean("show_model_summary")
running_flag_file = os.path.join(output_dir, ".training.lock")
if os.path.isfile(running_flag_file):
raise RuntimeError("A process is running in this directory!!!")
else:
open(running_flag_file, "a").close()
try:
print("backup config file to", output_dir, flush=True)
shutil.copy(config_file,
os.path.join(output_dir,
os.path.split(config_file)[1]))
tfrecord_dir_tr = os.path.join(results_subdir[:-4], "train")
tfrecord_dir_vl = os.path.join(results_subdir[:-4], "valid")
shutil.copy(tfrecord_dir_tr + "/train.csv", output_dir)
shutil.copy(tfrecord_dir_vl + "/valid.csv", output_dir)
# Get class names
class_names = get_class_names(output_dir, "train")
# get train sample counts
train_counts, train_pos_counts = get_sample_counts(
output_dir, "train", class_names)
valid_counts, _ = get_sample_counts(output_dir, "valid", class_names)
print("Total Training Data:", train_counts, flush=True)
print("Total Validation Data:", valid_counts, flush=True)
train_steps = int(min(samples_per_epoch, train_counts) / batch_size)
print("** train_steps:", train_steps, flush=True)
validation_steps = int(np.floor(valid_counts / batch_size))
print("** validation_steps:", validation_steps, flush=True)
# compute class weights
print("** compute class weights from training data **", flush=True)
class_weights = get_class_weights(
train_counts,
train_pos_counts,
multiply=positive_weights_multiply,
)
print("** class_weights **", flush=True)
print(class_weights)
print("** load model **", flush=True)
if use_trained_model_weights:
if use_best_weights:
model_weights_file = os.path.join(
output_dir, "best_" + output_weights_name)
else:
model_weights_file = os.path.join(output_dir,
output_weights_name)
else:
model_weights_file = None
# Use downloaded weights
if os.path.isfile(path_model_base_weights):
base_weights = path_model_base_weights
print("** Base weights will be loaded.", flush=True)
else:
base_weights = None
print("** No Base weights.", flush=True)
# Get Model
# ------------------------------------
input_shape = (image_dimension, image_dimension, 3)
img_input = Input(shape=input_shape)
base_model = DenseNet121(include_top=False,
weights=base_weights,
input_tensor=img_input,
input_shape=input_shape,
pooling="avg")
x = base_model.output
predictions = Dense(len(class_names),
activation="sigmoid",
name="predictions")(x)
model = Model(inputs=img_input, outputs=predictions)
if use_trained_model_weights and model_weights_file != None:
print("** load model weights_path:",
model_weights_file,
flush=True)
model.load_weights(model_weights_file)
# ------------------------------------
if show_model_summary:
print(model.summary())
print("** create image generators", flush=True)
train_seq = TFWrapper(tfrecord_dir=tfrecord_dir_tr,
record_file_endings=record_file_ending,
batch_size=batch_size,
model_target_size=(image_dimension,
image_dimension),
steps=train_steps,
augment=True,
shuffle=True,
prefetch=True,
repeat=True)
valid_seq = TFWrapper(tfrecord_dir=tfrecord_dir_vl,
record_file_endings=record_file_ending,
batch_size=batch_size,
model_target_size=(image_dimension,
image_dimension),
steps=None,
augment=False,
shuffle=False,
prefetch=True,
repeat=True)
# Initialise train and valid iterats
print("** Initialise train and valid iterators", flush=True)
train_seq.initialise()
valid_seq.initialise()
output_weights_path = os.path.join(output_dir, output_weights_name)
print("** set output weights path to:",
output_weights_path,
flush=True)
print("** SINGLE_gpu_model is used!", flush=True)
model_train = model
checkpoint = ModelCheckpoint(
output_weights_path,
save_weights_only=True,
save_best_only=False,
verbose=1,
)
print("** compile model with class weights **", flush=True)
optimizer = Adam(lr=initial_learning_rate)
model_train.compile(optimizer=optimizer, loss="binary_crossentropy")
auroc = MultipleClassAUROC(sequence=valid_seq,
class_names=class_names,
weights_path=output_weights_path,
stats=training_stats,
early_stop_p=patience,
learn_rate_p=patience_reduce_lr,
learn_rate_f=reduce_lr,
min_lr=min_lr,
workers=0)
callbacks = [
checkpoint,
TensorBoard(log_dir=os.path.join(output_dir, "logs"),
batch_size=batch_size), auroc
]
print("** start training **", flush=True)
history = model_train.fit_generator(
generator=train_seq,
steps_per_epoch=train_steps,
epochs=epochs,
validation_data=valid_seq,
validation_steps=validation_steps,
callbacks=callbacks,
class_weight=class_weights,
workers=0,
shuffle=False,
)
# dump history
print("** dump history **", flush=True)
with open(os.path.join(output_dir, "history.pkl"), "wb") as f:
pickle.dump({
"history": history.history,
"auroc": auroc.aurocs,
}, f)
print("** done! **", flush=True)
finally:
os.remove(running_flag_file)
def main():
# parser config
config_file = "./config.ini"
cp = ConfigParser()
cp.read(config_file)
# default config
output_dir = cp["DEFAULT"].get("output_dir")
if not os.path.exists(output_dir):
os.mkdir(output_dir)
base_model_name = cp["DEFAULT"].get("base_model_name")
class_names = cp["DEFAULT"].get("class_names").split(",")
image_source_dir = cp["DEFAULT"].get("image_source_dir")
image_dimension = cp["TRAIN"].getint("image_dimension")
batch_size = cp["TEST"].getint("batch_size")
test_steps = cp["TEST"].get("test_steps")
# parse weights file path
output_weights_name = cp["TRAIN"].get("output_weights_name")
weights_path = os.path.join(output_dir, output_weights_name)
data_set_dir = cp["TRAIN"].get("dataset_csv_dir")
input_weights_name = cp["TRAIN"].get("input_weights_name")
best_weights_path = os.path.join(data_set_dir,
f"best_{input_weights_name}")
# get test sample count
test_counts, _ = get_sample_counts(data_set_dir, "test", class_names)
# compute steps
if test_steps == "auto":
test_steps = int(test_counts / batch_size)
else:
try:
test_steps = int(test_steps)
except ValueError:
raise ValueError(f"""
test_steps: {test_steps} is invalid,
please use 'auto' or integer.
""")
# CAM config
bbox_list_file = cp["CAM"].get("bbox_list_file")
use_best_weights = cp["CAM"].getboolean("use_best_weights")
print("** load model **")
if use_best_weights:
print("** use best weights **")
model_weights_path = best_weights_path
else:
print("** use last weights **")
model_weights_path = weights_path
model_factory = ModelFactory()
model = model_factory.get_model(class_names,
model_name=base_model_name,
use_base_weights=False,
weights_path=model_weights_path)
print("read bbox list file")
df_images = pd.read_csv(bbox_list_file, header=None, skiprows=1)
df_images.columns = ["file_name", "label", "x", "y", "w", "h"]
print("create a generator for loading transformed images")
cam_sequence = AugmentedImageSequence(
dataset_csv_file=os.path.join(data_set_dir, "test.csv"),
class_names=class_names,
source_image_dir=image_source_dir,
batch_size=batch_size,
target_size=(image_dimension, image_dimension),
augmenter=None,
steps=test_steps,
shuffle_on_epoch_end=False,
)
image_output_dir = os.path.join(output_dir, "cam")
if not os.path.isdir(image_output_dir):
os.makedirs(image_output_dir)
print("create CAM")
df_images.apply(
lambda g: create_cam(
df_g=g,
output_dir=image_output_dir,
image_source_dir=image_source_dir,
model=model,
generator=cam_sequence,
class_names=class_names,
),
axis=1,
)
def modelpredict():
# parser config
config_file = "./config.ini"
cp = ConfigParser()
cp.read(config_file)
# default config
output_dir = cp["DEFAULT"].get("output_dir")
class_names = cp["DEFAULT"].get("class_names").split(",")
# test config
batch_size = cp["DEV"].getint("batch_size")
use_best_weights = cp["DEV"].getboolean("use_best_weights")
# parse weights file path
output_weights_name = cp["TRAIN"].get("output_weights_name")
weights_path = os.path.join(output_dir, output_weights_name)
best_weights_path = os.path.join(output_dir, f"best_{output_weights_name}")
# get test sample count
test_counts, _ = get_sample_counts(output_dir, "dev", class_names)
symlink_dir_name = "image_links"
dev_data_path = f"{output_dir}/{symlink_dir_name}/dev/"
step_test = int(test_counts / batch_size)
print("** load dev generator **")
test_generator = custom_image_generator(
ImageDataGenerator(horizontal_flip=True, rescale=1. / 255),
dev_data_path,
batch_size=batch_size,
class_names=class_names,
)
x, y = load_generator_data(test_generator, step_test)
print("** load model **")
model = get_model(class_names)
if use_best_weights:
print("** use best weights **")
model.load_weights(best_weights_path)
else:
print("** use last weights **")
model.load_weights(weights_path)
print("** make prediction **")
y_hat = model.predict(x)
# print(y_hat)
# dev_log_path = os.path.join(output_dir, "dev.log")
# print(f"** write log to {dev_log_path} **")
# aurocs = []
# with open(dev_log_path, "w") as f:
# for i in range(len(class_names)):
# try:
# score = roc_auc_score(y[i], y_hat[i])
# aurocs.append(score)
# except ValueError:
# score = 0
# f.write(f"{class_names[i]}: {score}\n")
# mean_auroc = np.mean(aurocs)
# f.write("-------------------------\n")
# f.write(f"mean auroc: {mean_auroc}\n")
return y, y_hat
def load_model(config_file="./config.ini", change_arch=False, compile_=True):
cp = ConfigParser()
cp.read(config_file)
# default config
output_dir = cp["DEFAULT"].get("output_dir")
base_model_name = cp["DEFAULT"].get("base_model_name")
class_names = cp["DEFAULT"].get("class_names").split(",")
image_source_dir = cp["DEFAULT"].get("image_source_dir")
# train config
image_dimension = cp["TRAIN"].getint("image_dimension")
# test config
batch_size = cp["TEST"].getint("batch_size")
test_steps = cp["TEST"].get("test_steps")
use_best_weights = cp["TEST"].getboolean("use_best_weights")
# parse weights file path
output_weights_name = cp["TRAIN"].get("output_weights_name")
weights_path = os.path.join(output_dir, output_weights_name)
best_weights_path = os.path.join(output_dir, f"best_{output_weights_name}")
# get test sample count
test_counts, _ = get_sample_counts(output_dir, "test", class_names)
# compute steps
if test_steps == "auto":
test_steps = int(test_counts / batch_size)
else:
try:
test_steps = int(test_steps)
except ValueError:
raise ValueError(f"""
test_steps: {test_steps} is invalid,
please use 'auto' or integer.
""")
print(f"** test_steps: {test_steps} **")
print("** load model **")
if use_best_weights:
print("** use best weights **")
model_weights_path = best_weights_path
else:
print("** use last weights **")
model_weights_path = weights_path
# load CheXNet model:
model_factory = ModelFactory()
model = model_factory.get_model(class_names,
model_name=base_model_name,
use_base_weights=False,
weights_path=model_weights_path)
if change_arch:
#return model
# input layer, output layer:
input_layer = model.get_layer(index=0)
chex_output = model.get_layer(index=-1)
# add second last layer:
intermediate_layer = model.get_layer(index=-2)
rsna_add_layer = Dense(10, activation='relu', name='rsna_add_layer')(
intermediate_layer.output) # params are tentative
rsna_clf_output = Dense(3,
activation='softmax',
name='rsna_clf_output')(concatenate(
[rsna_add_layer, chex_output.output]))
model = Model(inputs=[input_layer.input], outputs=[rsna_clf_output])
losses = {'rsna_clf_output': 'categorical_crossentropy'}
if compile_:
print('** compile **')
model.compile(optimizer='rmsprop', loss=losses, loss_weights=[1.])
else:
if compile_:
print('** compile **')
model.compile(optimizer='rmsprop', loss=losses, loss_weights=[1.])
return model
def main():
# parser config
config_file = "./config.ini"
cp = ConfigParser()
cp.read(config_file)
# default config
output_dir = cp["DEFAULT"].get("output_dir")
base_model_name = cp["DEFAULT"].get("base_model_name")
class_names = cp["DEFAULT"].get("class_names").split(",")
image_source_dir = cp["DEFAULT"].get("image_source_dir")
# train config
image_dimension = cp["TRAIN"].getint("image_dimension")
# test config
batch_size = cp["TEST"].getint("batch_size")
test_steps = cp["TEST"].get("test_steps")
use_best_weights = cp["TEST"].getboolean("use_best_weights")
# parse weights file path
output_weights_name = cp["TRAIN"].get("output_weights_name")
weights_path = os.path.join(output_dir, output_weights_name)
best_weights_path = os.path.join(output_dir, f"best_{output_weights_name}")
# get test sample count
test_counts, _ = get_sample_counts(output_dir, "test", class_names)
# compute steps
if test_steps == "auto":
test_steps = int(test_counts / batch_size)
else:
try:
test_steps = int(test_steps)
except ValueError:
raise ValueError(f"""
test_steps: {test_steps} is invalid,
please use 'auto' or integer.
""")
print(f"** test_steps: {test_steps} **")
print("** load model **")
if use_best_weights:
print("** use best weights **")
model_weights_path = best_weights_path
else:
print("** use last weights **")
model_weights_path = weights_path
model_factory = ModelFactory()
model = model_factory.get_model(class_names,
model_name=base_model_name,
use_base_weights=False,
weights_path=model_weights_path)
print("** load test generator **")
test_sequence = AugmentedImageSequence(
dataset_csv_file=os.path.join(output_dir, "test.csv"),
class_names=class_names,
source_image_dir=image_source_dir,
batch_size=batch_size,
target_size=(image_dimension, image_dimension),
augmenter=None,
steps=test_steps,
shuffle_on_epoch_end=False,
)
print("** make prediction **")
logs = []
starttime = time()
y_hat = model.predict_generator(test_sequence, verbose=1)
logs.append(time() - starttime)
print("time: " + str(logs))
y = test_sequence.get_y_true()
test_log_path = os.path.join(output_dir, "test.log")
print(f"** write log to {test_log_path} **")
aurocs = []
with open(test_log_path, "w") as f:
for i in range(len(class_names)):
try:
score = roc_auc_score(y[:, i], y_hat[:, i])
aurocs.append(score)
except ValueError:
score = 0
f.write(f"{class_names[i]}: {score}\n")
mean_auroc = np.mean(aurocs)
f.write("-------------------------\n")
f.write(f"mean auroc: {mean_auroc}\n")
print(f"mean auroc: {mean_auroc}")
def main():
# parser config
config_file = "./config.ini"
cp = ConfigParser()
cp.read(config_file)
# default config
output_dir = cp["DEFAULT"].get("output_dir")
image_source_dir = cp["DEFAULT"].get("image_source_dir")
train_patient_count = cp["DEFAULT"].getint("train_patient_count")
dev_patient_count = cp["DEFAULT"].getint("dev_patient_count")
data_entry_file = cp["DEFAULT"].get("data_entry_file")
class_names = cp["DEFAULT"].get("class_names").split(",")
# train config
use_base_model_weights = cp["TRAIN"].getboolean("use_base_model_weights")
use_trained_model_weights = cp["TRAIN"].getboolean("use_trained_model_weights")
use_best_weights = cp["TRAIN"].getboolean("use_best_weights")
output_weights_name = cp["TRAIN"].get("output_weights_name")
epochs = cp["TRAIN"].getint("epochs")
batch_size = cp["TRAIN"].getint("batch_size")
initial_learning_rate = cp["TRAIN"].getfloat("initial_learning_rate")
train_steps = cp["TRAIN"].get("train_steps")
patience_reduce_lr = cp["TRAIN"].getint("patience_reduce_lr")
validation_steps = cp["TRAIN"].get("validation_steps")
positive_weights_multiply = cp["TRAIN"].getfloat("positive_weights_multiply")
use_class_balancing = cp["TRAIN"].getboolean("use_class_balancing")
use_default_split = cp["TRAIN"].getboolean("use_default_split")
# if previously trained weights is used, never re-split
if use_trained_model_weights:
# resuming mode
print("** use trained model weights, turn on use_skip_split automatically **")
use_skip_split = True
# load training status for resuming
training_stats_file = os.path.join(output_dir, ".training_stats.json")
if os.path.isfile(training_stats_file):
# TODO: add loading previous learning rate?
training_stats = json.load(open(training_stats_file))
else:
training_stats = {}
else:
# start over
use_skip_split = cp["TRAIN"].getboolean("use_skip_split ")
training_stats = {}
split_dataset_random_state = cp["TRAIN"].getint("split_dataset_random_state")
show_model_summary = cp["TRAIN"].getboolean("show_model_summary")
# end parser config
# check output_dir, create it if not exists
if not os.path.isdir(output_dir):
os.makedirs(output_dir)
running_flag_file = os.path.join(output_dir, ".training.lock")
if os.path.isfile(running_flag_file):
raise RuntimeError("A process is running in this directory!!!")
else:
open(running_flag_file, "a").close()
try:
print(f"backup config file to {output_dir}")
shutil.copy(config_file, os.path.join(output_dir, os.path.split(config_file)[1]))
# split train/dev/test
if use_default_split:
datasets = ["train", "dev", "test"]
for dataset in datasets:
shutil.copy(f"./data/default_split/{dataset}.csv", output_dir)
elif not use_skip_split:
print("** split dataset **")
split_data(
data_entry_file,
class_names,
train_patient_count,
dev_patient_count,
output_dir,
split_dataset_random_state,
)
# get train/dev sample counts
train_counts, train_pos_counts = get_sample_counts(output_dir, "train", class_names)
dev_counts, _ = get_sample_counts(output_dir, "dev", class_names)
# compute steps
if train_steps == "auto":
train_steps = int(train_counts / batch_size)
else:
try:
train_steps = int(train_steps)
except ValueError:
raise ValueError(f"""
train_steps: {train_steps} is invalid,
please use 'auto' or integer.
""")
print(f"** train_steps: {train_steps} **")
if validation_steps == "auto":
validation_steps = int(dev_counts / batch_size)
else:
try:
validation_steps = int(validation_steps)
except ValueError:
raise ValueError(f"""
validation_steps: {validation_steps} is invalid,
please use 'auto' or integer.
""")
print(f"** validation_steps: {validation_steps} **")
# compute class weights
print("** compute class weights from training data **")
class_weights = get_class_weights(
train_counts,
train_pos_counts,
multiply=positive_weights_multiply,
use_class_balancing=use_class_balancing
)
print("** class_weights **")
for c, w in class_weights.items():
print(f" {c}: {w}")
print("** load model **")
if use_base_model_weights:
base_model_weights_file = cp["TRAIN"].get("base_model_weights_file")
else:
base_model_weights_file = None
if use_trained_model_weights:
if use_best_weights:
model_weights_file = os.path.join(output_dir, f"best_{output_weights_name}")
else:
model_weights_file = os.path.join(output_dir, output_weights_name)
else:
model_weights_file = None
model = get_model(class_names, base_model_weights_file, model_weights_file)
if show_model_summary:
print(model.summary())
# recreate symlink folder for ImageDataGenerator
symlink_dir_name = "image_links"
create_symlink(image_source_dir, output_dir, symlink_dir_name)
print("** create image generators **")
train_data_path = f"{output_dir}/{symlink_dir_name}/train/"
train_generator = custom_image_generator(
ImageDataGenerator(horizontal_flip=True, rescale=1./255),
train_data_path,
batch_size=batch_size,
class_names=class_names,
)
dev_data_path = f"{output_dir}/{symlink_dir_name}/dev/"
dev_generator = custom_image_generator(
ImageDataGenerator(horizontal_flip=True, rescale=1./255),
dev_data_path,
batch_size=batch_size,
class_names=class_names,
)
output_weights_path = os.path.join(output_dir, output_weights_name)
print(f"** set output weights path to: {output_weights_path} **")
print("** check multiple gpu availability **")
gpus = len(os.getenv("CUDA_VISIBLE_DEVICES", "1").split(","))
if gpus > 1:
print(f"** multi_gpu_model is used! gpus={gpus} **")
model_train = multi_gpu_model(model, gpus)
# FIXME: currently (Keras 2.1.2) checkpoint doesn't work with multi_gpu_model
checkpoint = MultiGPUModelCheckpoint(
filepath=output_weights_path,
base_model=model,
)
else:
model_train = model
checkpoint = ModelCheckpoint(output_weights_path)
print("** compile model with class weights **")
optimizer = Adam(lr=initial_learning_rate)
model_train.compile(optimizer=optimizer, loss="binary_crossentropy")
auroc = MultipleClassAUROC(
generator=dev_generator,
steps=validation_steps,
class_names=class_names,
weights_path=output_weights_path,
stats=training_stats,
)
callbacks = [
checkpoint,
TensorBoard(log_dir=os.path.join(output_dir, "logs"), batch_size=batch_size),
ReduceLROnPlateau(monitor='val_loss', factor=0.1, patience=patience_reduce_lr, verbose=1),
auroc,
]
print("** training start **")
history = model_train.fit_generator(
generator=train_generator,
steps_per_epoch=train_steps,
epochs=epochs,
validation_data=dev_generator,
validation_steps=validation_steps,
callbacks=callbacks,
class_weight=class_weights,
)
# dump history
print("** dump history **")
with open(os.path.join(output_dir, "history.pkl"), "wb") as f:
pickle.dump({
"history": history.history,
"auroc": auroc.aurocs,
}, f)
print("** done! **")
finally:
os.remove(running_flag_file)