def export_model(config, model_creator):
if not config.export_path:
raise ValueError("Export path must be specified.")
if not config.model_version:
raise ValueError("Export model version must be specified.")
utils.makedir(config.export_path)
# Create model
model = model_helper.create_model(model_creator, config, mode="infer")
# TensorFlow model
config_proto = utils.get_config_proto()
sess = tf.Session(config=config_proto, graph=model.graph)
with model.graph.as_default():
loaded_model, global_step = model_helper.create_or_load_model(
model.model, config.best_eval_loss_dir, sess, "infer")
export_dir = os.path.join(config.export_path, config.model_version)
builder = tf.saved_model.builder.SavedModelBuilder(export_dir)
inputs = {
"word_ids1":
tf.saved_model.utils.build_tensor_info(loaded_model.word_ids1),
"word_ids2":
tf.saved_model.utils.build_tensor_info(loaded_model.word_ids2),
"word_len1":
tf.saved_model.utils.build_tensor_info(loaded_model.word_len1),
"word_len2":
tf.saved_model.utils.build_tensor_info(loaded_model.word_len2),
"char_ids1":
tf.saved_model.utils.build_tensor_info(loaded_model.char_ids1),
"char_ids2":
tf.saved_model.utils.build_tensor_info(loaded_model.char_ids2),
"char_len1":
tf.saved_model.utils.build_tensor_info(loaded_model.char_len1),
"char_len2":
tf.saved_model.utils.build_tensor_info(loaded_model.char_len2)
}
outputs = {
"simscore":
tf.saved_model.utils.build_tensor_info(loaded_model.simscore)
}
prediction_signature = (
tf.saved_model.signature_def_utils.build_signature_def(
inputs=inputs,
outputs=outputs,
method_name=tf.saved_model.signature_constants.
PREDICT_METHOD_NAME))
builder.add_meta_graph_and_variables(
sess, [tf.saved_model.tag_constants.SERVING], {
tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY:
prediction_signature
})
builder.save()
logger.info("Export model succeed.")
def __init__(self, args, net_input_shape):
'''
Create evaluation model and load the pre-train weights for inference.
'''
self.net_input_shape = net_input_shape
weights_path = join(args.weights_path)
# Create model object in inference mode but Disable decoder layer.
_, eval_model, _ = create_model(args, net_input_shape, enable_decoder=False)
# Load weights trained on MS-COCO by name because part of output layers are disable.
eval_model.load_weights(weights_path, by_name=True)
self.model = eval_model
def test(config, model_creator):
# for metric in config.metrics.split(","):
best_metric_label = "best_eval_loss"
model_dir = getattr(config, best_metric_label + "_dir")
logger.info("Start evaluating saved best model on training-set.")
eval_model = model_helper.create_model(model_creator, config, mode="eval")
session_config = utils.get_config_proto()
eval_sess = tf.Session(config=session_config, graph=eval_model.graph)
run_test(config, eval_model, eval_sess, config.train_file, model_dir)
logger.info("Start evaluating saved best model on dev-set.")
eval_model = model_helper.create_model(model_creator, config, mode="eval")
session_config = utils.get_config_proto()
eval_sess = tf.Session(config=session_config, graph=eval_model.graph)
run_test(config, eval_model, eval_sess, config.dev_file, model_dir)
logger.info("Start evaluating saved best model on test-set.")
eval_model = model_helper.create_model(model_creator, config, mode="eval")
session_config = utils.get_config_proto()
eval_sess = tf.Session(config=session_config, graph=eval_model.graph)
run_test(config, eval_model, eval_sess, config.test_file, model_dir)
def inference(config, model_creator):
output_file = "output_" + os.path.split(config.infer_file)[-1].split(".")[0]
# Inference output directory
pred_file = os.path.join(config.model_dir, output_file)
utils.makedir(pred_file)
# Inference
model_dir = config.best_eval_loss_dir
# Create model
# model_creator = my_model.MyModel
infer_model = model_helper.create_model(model_creator, config, mode="infer")
# TensorFlow model
sess_config = utils.get_config_proto()
infer_sess = tf.Session(config=sess_config, graph=infer_model.graph)
with infer_model.graph.as_default():
loaded_infer_model, _ = model_helper.create_or_load_model(
infer_model.model, model_dir, infer_sess, "infer")
run_infer(config, loaded_infer_model, infer_sess, pred_file)
def main(args):
net_input_shape = (RESOLUTION, RESOLUTION, 3
) # Only access RGB 3 channels.
# Create the model for training/testing/manipulation
# enable_decoder = False only for SegCaps R3 to disable recognition image output on evaluation model
# to speed up performance.
model_list = create_model(args=args,
input_shape=net_input_shape,
enable_decoder=True)
#print_summary(model=model_list[0], positions=[.38, .65, .75, 1.])
args.output_name = 'split-' + str(args.split_num) + '_batch-' + str(args.batch_size) + \
'_shuff-' + str(args.shuffle_data) + '_aug-' + str(args.aug_data) + \
'_loss-' + str(args.loss) + '_slic-' + str(args.slices) + \
'_sub-' + str(args.subsamp) + '_strid-' + str(args.stride) + \
'_lr-' + str(args.initial_lr) + '_recon-' + str(args.recon_wei)
# args.output_name = 'sh-' + str(args.shuffle_data) + '_a-' + str(args.aug_data)
args.time = time
if platform.system() == 'Windows':
args.use_multiprocessing = False
else:
args.use_multiprocessing = True
args.check_dir = join(args.data_root, 'saved_models', args.net)
try:
makedirs(args.check_dir)
except:
pass
args.log_dir = join(args.data_root, 'logs', args.net)
try:
makedirs(args.log_dir)
except:
pass
args.tf_log_dir = join(args.log_dir, 'tf_logs')
try:
makedirs(args.tf_log_dir)
except:
pass
args.output_dir = join(args.data_root, 'plots', args.net)
try:
makedirs(args.output_dir)
except:
pass
#if args.train == True:
from train import train
# Run training
train(args, model_list[0], net_input_shape)
if args.test == True:
from test import test
# Run testing
test(args, net_input_shape)
if args.manip == True:
from manip import manip
# Run manipulation of segcaps
manip(args, test_list, model_list, net_input_shape)
def main(args):
# Ensure training, testing, and manip are not all turned off
assert (
args.train or args.test or args.manip
), 'Cannot have train, test, and manip all set to 0, Nothing to do.'
# Load the training, validation, and testing data
try:
train_list, val_list, test_list = load_data(args.data_root_dir,
args.split_num)
except:
# Create the training and test splits if not found
logging.info(
'\nNo existing training, validate, test files...System will generate it.'
)
split_data(args.data_root_dir, num_splits=args.Kfold)
train_list, val_list, test_list = load_data(args.data_root_dir,
args.split_num)
# Get image properties from first image. Assume they are all the same.
logging.info('\nRead image files...%s' %
(join(args.data_root_dir, 'imgs', train_list[0][0])))
# Get image shape from the first image.
image = sitk.GetArrayFromImage(
sitk.ReadImage(join(args.data_root_dir, 'imgs', train_list[0][0])))
img_shape = image.shape # # (x, y, channels)
if args.dataset == 'luna16':
net_input_shape = (img_shape[1], img_shape[2], args.slices)
else:
args.slices = 1
if GRAYSCALE:
net_input_shape = (RESOLUTION, RESOLUTION, 1) # only one channel
else:
net_input_shape = (RESOLUTION, RESOLUTION, 3
) # Only access RGB 3 channels.
# Create the model for training/testing/manipulation
# enable_decoder = False only for SegCaps R3 to disable recognition image output on evaluation model
# to speed up performance.
model_list = create_model(args=args,
input_shape=net_input_shape,
enable_decoder=True)
print_summary(model=model_list[0], positions=[.38, .65, .75, 1.])
args.output_name = 'split-' + str(args.split_num) + '_batch-' + str(args.batch_size) + \
'_shuff-' + str(args.shuffle_data) + '_aug-' + str(args.aug_data) + \
'_loss-' + str(args.loss) + '_slic-' + str(args.slices) + \
'_sub-' + str(args.subsamp) + '_strid-' + str(args.stride) + \
'_lr-' + str(args.initial_lr) + '_recon-' + str(args.recon_wei)
# args.output_name = 'sh-' + str(args.shuffle_data) + '_a-' + str(args.aug_data)
args.time = time
if platform.system() == 'Windows':
args.use_multiprocessing = False
else:
args.use_multiprocessing = True
args.check_dir = join(args.data_root_dir, 'saved_models', args.net)
try:
makedirs(args.check_dir)
except:
pass
args.log_dir = join(args.data_root_dir, 'logs', args.net)
try:
makedirs(args.log_dir)
except:
pass
args.tf_log_dir = join(args.log_dir, 'tf_logs')
try:
makedirs(args.tf_log_dir)
except:
pass
args.output_dir = join(args.data_root_dir, 'plots', args.net)
try:
makedirs(args.output_dir)
except:
pass
if args.train:
from train import train
# Run training
train(args, train_list, val_list, model_list[0], net_input_shape)
if args.test:
from test import test
# Run testing
test(args, test_list, model_list, net_input_shape)
if args.manip:
from manip import manip
# Run manipulation of segcaps
manip(args, test_list, model_list, net_input_shape)
def train(config, model_creator):
steps_per_stats = config.steps_per_stats
steps_per_eval = config.steps_per_eval
model_dir = config.model_dir
log_dir = config.log_dir
ckpt_name = config.ckpt_name
ckpt_path = os.path.join(model_dir, ckpt_name)
# Create model
train_model = model_helper.create_model(model_creator, config, "train")
eval_model = model_helper.create_model(model_creator, config, "eval")
# infer_model = model_helper.create_model(model_creator, config, "infer")
train_data = data_helper.load_data(config.train_file,
config.word_vocab_file,
config.char_vocab_file,
w_max_len1=config.max_word_len1,
w_max_len2=config.max_word_len2,
c_max_len1=config.max_char_len1,
c_max_len2=config.max_char_len2,
text_split="|",
split="\t")
train_iterator = data_helper.batch_iterator(train_data,
batch_size=config.batch_size,
shuffle=True)
eval_data = data_helper.load_data(config.dev_file,
config.word_vocab_file,
config.char_vocab_file,
w_max_len1=config.max_word_len1,
w_max_len2=config.max_word_len2,
c_max_len1=config.max_char_len1,
c_max_len2=config.max_char_len2,
text_split="|",
split="\t")
# eval_iterator = data_helper.batch_iterator(eval_data, batch_size=config.batch_size, shuffle=False)
# TensorFlow model
session_config = utils.get_config_proto()
train_sess = tf.Session(config=session_config, graph=train_model.graph)
eval_sess = tf.Session(config=session_config, graph=eval_model.graph)
# infer_sess = tf.Session(config=config, graph=infer_model.graph)
# Summary Writer
train_summary_writer = tf.summary.FileWriter(
os.path.join(log_dir, "train_log"), train_model.graph)
eval_summary_writer = tf.summary.FileWriter(
os.path.join(log_dir, "eval_log"), eval_model.graph)
with train_model.graph.as_default():
loaded_train_model, global_step = model_helper.create_or_load_model(
train_model.model, model_dir, train_sess, "train")
local_initializer = tf.local_variables_initializer()
# running_vars = tf.get_collection(tf.GraphKeys.LOCAL_VARIABLES, scope="metrics")
# running_vars_initializer = tf.variables_initializer(var_list=running_vars)
step_time, train_loss, train_acc, train_rec, train_pre, train_f1, train_auc, gN = 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0
lr = loaded_train_model.learning_rate.eval(session=train_sess)
last_stat_step = global_step
last_eval_step = global_step
logger.info("# Start step %d" % global_step)
epoch_idx = 0
while epoch_idx < config.num_train_epochs:
start_time = time.time()
try:
# TODO: tf.metrics
# train_sess.run(running_vars_initializer)
train_sess.run(local_initializer)
batch = next(train_iterator)
b_word_ids1, b_word_ids2, b_word_len1, b_word_len2, b_char_ids1, b_char_ids2, b_char_len1, b_char_len2, b_labels = batch
# for b in batch:
# print(b)
train_summary1, pred, step_loss, _, acc_op, rec_op, pre_op, auc_op, global_step, grad_norm, lr = \
loaded_train_model.train(train_sess, b_word_ids1, b_word_ids2, b_word_len1, b_word_len2,
b_char_ids1, b_char_ids2, b_char_len1, b_char_len2, b_labels)
train_summary2, step_acc, step_rec, step_pre, step_auc = \
train_sess.run([loaded_train_model.train_summary2,
loaded_train_model.accuracy,
loaded_train_model.recall,
loaded_train_model.precision,
loaded_train_model.auc])
config.epoch_step += 1
except StopIteration:
# Finished going through the training dataset. Go to next epoch.
epoch_idx += 1
config.epoch_step = 0
train_iterator = data_helper.batch_iterator(
train_data, batch_size=config.batch_size, shuffle=True)
continue
step_time += (time.time() - start_time)
train_loss += step_loss
train_acc += step_acc
train_rec += step_rec
train_pre += step_pre
train_auc += step_auc
gN += grad_norm
if global_step - last_stat_step >= steps_per_stats:
last_stat_step = global_step
step_time /= steps_per_stats
train_loss /= steps_per_stats
train_acc /= steps_per_stats
train_rec /= steps_per_stats
train_pre /= steps_per_stats
train_f1 = (2 * train_rec * train_pre) / (train_rec + train_pre +
0.00000001)
gN /= steps_per_stats
logger.info(
" step %d lr %g step_time %.2fs loss %.4f acc %.4f rec %.4f pre %.4f f1 %.4f auc %.4f gN %.2f"
% (global_step, lr, step_time, train_loss, train_acc,
train_rec, train_pre, train_f1, train_auc, grad_norm))
train_summary_writer.add_summary(train_summary1,
global_step=global_step)
train_summary_writer.add_summary(train_summary2,
global_step=global_step)
step_time, train_loss, train_acc, train_rec, train_pre, train_f1, train_auc, gN = 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0
if global_step - last_eval_step >= steps_per_eval:
last_eval_step = global_step
# Save checkpoint
loaded_train_model.saver.save(train_sess,
ckpt_path,
global_step=global_step)
# Evaluate on dev
run_eval(config,
eval_model,
eval_sess,
eval_data,
model_dir,
ckpt_name,
eval_summary_writer,
save_on_best=True)
logger.info("# Finished epoch %d, step %d." % (epoch_idx, global_step))
# Done training
loaded_train_model.saver.save(train_sess,
ckpt_path,
global_step=global_step)
logger.info(
"# Final, step %d lr %g step_time %.2fs loss %.4f acc %.4f rec %.4f pre %.4f f1 %.4f auc %.4f gN %.2f"
% (global_step, lr, step_time, train_loss, train_acc, train_rec,
train_pre, train_f1, train_auc, gN))
logger.info("# Done training!")
train_summary_writer.close()
eval_summary_writer.close()