def train(img_size, start_layer, learning_rate, n_epoch, data_x, data_y):
model, base_model = _build_regressor(img_size, 1, start_layer, learning_rate)
weights_history = []
get_weights_cb = LambdaCallback(on_batch_end=lambda batch,
logs: weights_history.append(model.layers[-1].get_weights()[0]))
data_gen = ImageDataGenerator(rotation_range=180, zoom_range=0.1, horizontal_flip=True)
data_gen.fit(data_x)
history = model.fit_generator(data_gen.flow(data_x, data_y, batch_size=batch_size),
validation_data=(data_x[:1000], data_y[:1000]),
workers=4,
callbacks=[get_weights_cb],
use_multiprocessing=True,
epochs=n_epoch)
save_obj(weights_history, name=model_out_dir+"/weights.pkl")
base_model.save(model_out_dir+"/base_model.h5")
model.save(model_out_dir+"/model.h5")
return base_model, weights_history
def train(n_epoch=N_TRAINING_EPOCH,
img_size=299,
sex=0,
batch_size=16,
num_gpu=1,
start_layer=-1):
assert start_layer in [
-1, XCEPTION_EXIT_START, XCEPTION_MID_START, XCEPTION_ENTRY_START, 0
]
assert sex in [0, 1, 2]
# model file path
if start_layer != -1:
model_file = model_out_dir + "/model.h5"
else:
model_file = None
# learning rate
if start_layer == -1:
learning_rate = 1E-3
elif start_layer == XCEPTION_EXIT_START:
learning_rate = 1E-4
elif start_layer == XCEPTION_MID_START:
learning_rate = 5E-5
elif start_layer == XCEPTION_ENTRY_START:
learning_rate = 1E-5
else:
learning_rate = 5E-6
model = _build_regressor(img_size, num_gpu, start_layer, model_file,
learning_rate)
print "[x] load data ..."
data_x, data_y = load_data_sex(sex, img_size, xception_input)
print "[x] loaded data_x {}, data_y {} data".format(
data_x.shape, data_y.shape)
data_gen = ImageDataGenerator(rotation_range=180,
zoom_range=0.1,
horizontal_flip=True)
x_train, x_test, y_train, y_test = train_test_split(data_x, data_y)
data_gen.fit(x_train)
model_callback = ModelCheckpoint(filepath=model_out_dir + "/model.h5",
verbose=1,
save_best_only=True)
loss_callback = LossHistory()
model.fit_generator(data_gen.flow(x_train, y_train, batch_size=batch_size),
validation_data=(x_test, y_test),
workers=4,
callbacks=[model_callback, loss_callback],
use_multiprocessing=True,
epochs=n_epoch)
save_obj(obj=loss_callback.losses,
name=metric_out_dir + "/losses_S{}.pkl".format(start_layer))
y_true = []
y_pred = []
for mini_batch in range(int(data_x.shape[0] // batch_size)):
pred_y = model.predict_on_batch(
data_x[mini_batch * batch_size:(mini_batch + 1) * batch_size])
for i in range(batch_size):
y_true.append(data_y[mini_batch * batch_size + i] * SCALE)
y_pred.append(pred_y[i] * SCALE)
evs, mae, mse, meae, r2s, ccc = regression_metric(np.array(y_true),
np.array(y_pred))
save_obj(
{
"evs": evs,
"mae": mae,
"mse": mse,
"meae": meae,
"r2s": r2s,
"ccc": ccc
},
name=metric_out_dir + "/evaluate_S{}.pkl".format(start_layer))
def train(n_epoch=N_TRAINING_EPOCH,
img_size=299,
sex=0,
batch_size=16,
num_gpu=1,
start_layer=-1,
start_epoch=0,
data_thread=10):
assert start_layer in [
-1, XCEPTION_EXIT_START, XCEPTION_MID_START, XCEPTION_ENTRY_START, 0
]
assert sex in [0, 1, 2]
# model file path
if start_layer != -1:
model_file = model_out_dir + "/model.h5"
else:
model_file = None
# learning rate
if start_layer == -1:
learning_rate = 1E-3
elif start_layer == XCEPTION_EXIT_START:
learning_rate = 1E-4
elif start_layer == XCEPTION_MID_START:
learning_rate = 5E-5
elif start_layer == XCEPTION_ENTRY_START:
learning_rate = 1E-5
else:
learning_rate = 5E-6
model = _build_regressor(img_size, num_gpu, start_layer, model_file,
learning_rate)
best_mae = np.inf
tolerance = 0
data_ids = load_sex_ids(sex)
for i in range(data_thread):
train_data_thread = DataLoadingThread(
name="training_data_thread_{}".format(i))
train_data_thread.set_data(train_data_queue, True)
train_data_thread.setDaemon(True)
train_data_thread.start()
test_data_thread = DataLoadingThread(
name="test_data_thread_{}".format(i))
test_data_thread.set_data(test_data_queue, False)
test_data_thread.setDaemon(True)
test_data_thread.start()
for epoch in tqdm(range(start_epoch + 1, start_epoch + n_epoch + 1)):
print "[x] epoch {} -------------------------------------------".format(
epoch)
for mini_batch in range(len(data_ids) // batch_size):
while train_data_queue.qsize() <= 0:
time.sleep(1)
if DEBUG_MODEL:
print "train data queue, qsize = %d " % train_data_queue.qsize(
)
data_dict = train_data_queue.get()
loss = model.train_on_batch(x=data_dict["x"], y=data_dict["y"])
if mini_batch % 50 == 0:
print "--epoch {}, mini_batch {}, loss {}".format(
epoch, mini_batch, loss)
# test
print "[x] test in epoch {}".format(epoch)
losses = 0.0
for mini_batch in range(int(0.2 * len(data_ids) // batch_size)):
while test_data_queue.qsize() <= 0:
time.sleep(1)
data_dict = test_data_queue.get()
loss = model.test_on_batch(data_dict["x"], data_dict["y"])
losses += loss
losses = losses / (int(0.3 * len(data_ids) // batch_size))
print "== epoch {}, test loss {}".format(epoch, losses)
# test and metric
print "[x] predict in epoch {}".format(epoch)
y_true = []
y_pred = []
for mini_batch in range(int(0.2 * len(data_ids) // batch_size)):
while test_data_queue.qsize() <= 0:
time.sleep(1)
data_dict = test_data_queue.get()
pred_y = model.predict_on_batch(data_dict["x"])
for i in range(batch_size):
y_true.append(data_dict["y"][i] * SCALE)
y_pred.append(pred_y[i] * SCALE)
evs, mae, mse, meae, r2s, ccc = regression_metric(
np.array(y_true), np.array(y_pred))
save_obj(
{
"evs": evs,
"mae": mae,
"mse": mse,
"meae": meae,
"r2s": r2s,
"ccc": ccc,
"loss": losses
},
name=metric_out_dir + "/epoch_{}.pkl".format(epoch))
if mae < best_mae:
best_mae = mae
tolerance = 0
model.save_weights(model_out_dir + "/model.h5")
else:
tolerance += 1
print "[x] epoch {}, evs {}, mae {}, mse {}, meae {}, r2s {}, ccc {}".format(
epoch, evs, mae, mse, meae, r2s, ccc)
if tolerance > TOLERANCE:
break
feed_dict={
x_1: batch_x,
is_train: False
})
binary_preds_0, dice_coeff_0, acc_0, sensitivity_0, specificity_0 = metric_all_value(
batch_y, batch_segmented_0)
binary_preds_1, dice_coeff_1, acc_1, sensitivity_1, specificity_1 = metric_all_value(
batch_y, batch_segmented_1)
utils_data.save_obj(
{
"step": step,
"batch_loss_0": batch_loss_0,
"batch_loss_1": batch_loss_1,
"dice_coeff_0": dice_coeff_0,
"acc_0": acc_0,
"sensitivity_0": sensitivity_0,
"specificity_0": specificity_0,
"dice_coeff_1": dice_coeff_1,
"acc_1": acc_1,
"sensitivity_1": sensitivity_1,
"specificity_1": specificity_1
},
name=metrics_out_dir + "/step_{}_loss.pkl".format(step))
if step % 500 == 0:
output_dir = img_out_dir + "/step_{}".format(step)
if not os.path.isdir(output_dir):
os.makedirs(output_dir)
for i in range(batch_size):
data_index = shuffled_indexes[mini_batch * batch_size + i]
data_id = annotated_ids[data_index]
def train(n_epoch=N_TRAINING_EPOCH,
img_size=299,
sex=0,
batch_size=16,
num_gpu=1,
start_layer=-1,
start_epoch=0):
assert start_layer in [
-1, XCEPTION_EXIT_START, XCEPTION_MID_START, XCEPTION_ENTRY_START, 0
]
assert sex in [0, 1, 2]
# model file path
if start_layer != -1:
model_file = model_out_dir + "/model.h5"
else:
model_file = None
# learning rate
if start_layer == -1:
learning_rate = 1E-2
elif start_layer == XCEPTION_EXIT_START:
learning_rate = 1E-3
elif start_layer == XCEPTION_MID_START:
learning_rate = 5E-4
elif start_layer == XCEPTION_ENTRY_START:
learning_rate = 1E-4
else:
learning_rate = 5E-5
model = _build_regressor(img_size, num_gpu, start_layer, model_file,
learning_rate)
best_mae = np.inf
tolerance = 0
data_ids = load_sex_ids(sex)
for epoch in tqdm(range(start_epoch + 1, start_epoch + n_epoch + 1)):
print "[x] epoch {} -------------------------------------------".format(
epoch)
for mini_batch in range(len(data_ids) // batch_size):
batch_x, batch_y = load_data(sex=sex,
img_size=img_size,
batch_size=batch_size,
augment_times=7)
loss = model.train_on_batch(x=batch_x, y=batch_y)
if mini_batch % 50 == 0:
print "--epoch {}, mini_batch {}, loss {}".format(
epoch, mini_batch, loss)
# test
print "[x] test in epoch {}".format(epoch)
losses = 0.0
for mini_batch in range(int(0.2 * len(data_ids) // batch_size)):
batch_x, batch_y = load_data(sex=sex,
img_size=img_size,
batch_size=batch_size,
augment_times=0)
loss = model.test_on_batch(batch_x, batch_y)
losses += loss
losses = losses / (int(0.3 * len(data_ids) // batch_size))
print "== epoch {}, test loss {}".format(epoch, losses)
# test and metric
print "[x] predict in epoch {}".format(epoch)
y_true = []
y_pred = []
for mini_batch in range(int(0.2 * len(data_ids) // batch_size)):
batch_x, batch_y = load_data(sex=sex,
img_size=img_size,
batch_size=batch_size,
augment_times=0)
pred_y = model.predict_on_batch(batch_x)
for i in range(batch_size):
y_true.append(batch_y[i] * SCALE)
y_pred.append(pred_y[i] * SCALE)
evs, mae, mse, meae, r2s, ccc = regression_metric(
np.array(y_true), np.array(y_pred))
save_obj(
{
"evs": evs,
"mae": mae,
"mse": mse,
"meae": meae,
"r2s": r2s,
"ccc": ccc,
"loss": losses
},
name=metric_out_dir + "/epoch_{}.pkl".format(epoch))
if mae < best_mae:
best_mae = mae
tolerance = 0
model.save_weights(model_out_dir + "/model.h5")
else:
tolerance += 1
print "[x] epoch {}, evs {}, mae {}, mse {}, meae {}, r2s {}, ccc {}".format(
epoch, evs, mae, mse, meae, r2s, ccc)
if tolerance > TOLERANCE:
break
def train(model_name,
weights="imagenet",
n_epoch=N_TRAINING_EPOCH,
sex=0,
batch_size=16,
augment=False,
num_gpu=1,
fine_tune=""):
model, input_shape, base_model = _build_regresser(model_name, weights,
num_gpu, fine_tune)
if model_name == "inception_v3":
preprocess_fn = inception_v3_input
elif model_name == "inception_resnet_v2":
preprocess_fn = inception_resnet_input
elif model_name == "xception":
preprocess_fn = xception_input
else:
raise ValueError("Not a supported model name")
data_x, data_y = load_data(sex, input_shape[0], augment, preprocess_fn)
print "[x] total data size {} G".format(sys.getsizeof(data_x) / 1024**3)
best_loss = np.inf
for epoch in tqdm(range(n_epoch)):
print "========================================================================================================"
x_train, x_test, y_train, y_test = train_test_split(data_x,
data_y,
test_size=0.3,
random_state=0,
shuffle=True)
for mini_batch in range(x_train.shape[0] // batch_size):
batch_x = x_train[mini_batch * batch_size:(mini_batch + 1) *
batch_size]
batch_x_cvt = _batch_cvt(batch_x)
batch_y = y_train[mini_batch * batch_size:(mini_batch + 1) *
batch_size]
loss = model.train_on_batch(x=batch_x_cvt, y=batch_y)
if mini_batch % 100 == 0:
print "--epoch {}, mini_batch {}, loss {}".format(
epoch, mini_batch, loss)
# test
print "[x] test in epoch {}".format(epoch)
losses = 0.0
for mini_batch in range(x_test.shape[0] // batch_size):
batch_x = x_test[mini_batch * (batch_size):(mini_batch + 1) *
batch_size]
batch_y = y_test[mini_batch * (batch_size):(mini_batch + 1) *
batch_size]
batch_x_cvt = _batch_cvt(batch_x)
loss = model.test_on_batch(batch_x_cvt, batch_y)
losses += loss
losses = losses / (x_test.shape[0] // batch_size)
if losses < best_loss:
best_loss = losses
model.save_weights(model_out_dir + "/epoch_{}.h5".format(epoch))
print "== epoch {}, test loss {}".format(epoch, losses)
# test and metric
print "[x] predict in epoch {}".format(epoch)
y_true = []
y_pred = []
for mini_batch in range(x_test.shape[0] // batch_size):
batch_x = x_test[mini_batch * (batch_size):(mini_batch + 1) *
batch_size]
batch_y = y_test[mini_batch * (batch_size):(mini_batch + 1) *
batch_size]
batch_x_cvt = _batch_cvt(batch_x)
pred_y = model.predict_on_batch(batch_x_cvt)
for i in range(batch_size):
y_true.append(batch_y[i] * SCALE)
y_pred.append(pred_y[i] * SCALE)
evs, mae, mse, meae, r2s, ccc = regression_metric(
np.array(y_true), np.array(y_pred))
save_obj(
{
"evs": evs,
"mae": mae,
"mse": mse,
"meae": meae,
"r2s": r2s,
"ccc": ccc,
"loss": losses
},
name=metric_out_dir + "/epoch_{}.pkl".format(epoch))
print "[x] epoch {}, evs {}, mae {}, mse {}, meae {}, r2s {}, ccc {}".format(
epoch, evs, mae, mse, meae, r2s, ccc)
batch_size = FLAGS.batch_size * num_gpu
model_name = FLAGS.model_name
exp_name = FLAGS.exp
sex = FLAGS.sex
augment_data = True if FLAGS.augment == "true" else False
fine_tune = FLAGS.fine_tune
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
set_session(tf.Session(config=config))
metric_out_dir = "E{}_M{}_S{}_A{}_F{}/metric".format(
exp_name, model_name, sex, augment_data, fine_tune)
model_out_dir = "E{}_M{}_S{}_A{}_F{}/model".format(exp_name, model_name,
sex, augment_data,
fine_tune)
if not os.path.isdir(metric_out_dir):
os.makedirs(metric_out_dir)
if not os.path.isdir(model_out_dir):
os.makedirs(model_out_dir)
# training
weights_history, base_model, img_size, pre_fn = \
train(model_name=model_name, weights="imagenet", n_epoch=N_TRAINING_EPOCH, sex=sex,
batch_size=batch_size, augment=augment_data, num_gpu=num_gpu, fine_tune=fine_tune)
# save weight and base model
save_obj(weights_history, name=model_out_dir + "/weights_history.pkl")
base_model.save_weights(filepath=model_out_dir + "/base_model.h5")