def init_callbacks(self):
self.callbacks.append(
ModelCheckpoint(
filepath=os.path.join(
self.config.callbacks.checkpoint_dir,
'%s-{epoch:02d}-{val_loss:.2f}.hdf5' %
self.config.exp.name),
monitor=self.config.callbacks.checkpoint_monitor,
mode=self.config.callbacks.checkpoint_mode,
save_best_only=self.config.callbacks.checkpoint_save_best_only,
save_weights_only=self.config.callbacks.
checkpoint_save_weights_only,
verbose=self.config.callbacks.checkpoint_verbose,
))
self.callbacks.append(
TensorBoard(
log_dir=self.config.callbacks.tensorboard_log_dir,
write_graph=self.config.callbacks.tensorboard_write_graph,
))
if hasattr(self.config, "comet_api_key"):
from comet_ml import Experiment
experiment = Experiment(api_key=self.config.comet_api_key,
project_name=self.config.exp_name)
experiment.disable_mp()
experiment.log_multiple_params(self.config)
self.callbacks.append(experiment.get_keras_callback())
def init_callbacks(self):
self.callbacks.append(
ModelCheckpoint(
filepath=os.path.join(
self.checkpoint_dir, '%s-{epoch:02d}-{val_loss:.2f}.hdf5' %
self.config['exp_name']),
monitor=self.config['checkpoint_monitor'],
mode=self.config['checkpoint_mode'],
save_best_only=self.config['checkpoint_save_best_only'],
save_weights_only=self.config['checkpoint_save_weights_only'],
verbose=self.config['checkpoint_verbose'],
))
self.callbacks.append(
TensorBoard(
log_dir=self.tensorboard_log_dir,
write_graph=self.config['tensorboard_write_graph'],
histogram_freq=0, # don't compute histograms
write_images=
False # don't write model weights to visualize as image in TensorBoard
))
if hasattr(self.config, "comet_api_key"):
from comet_ml import Experiment
experiment = Experiment(api_key=self.config['comet_api_key'],
project_name=self.config['exp_name'])
experiment.disable_mp()
experiment.log_multiple_params(self.config)
self.callbacks.append(experiment.get_keras_callback())
def init_callbacks(self):
self.callbacks.append(
ModelCheckpoint(
filepath=os.path.join(self.config.checkpoint_dir, '%s-{epoch:03d}-{val_nme:.5f}.hdf5' % self.config.exp_name),
monitor=self.config.checkpoint_monitor,
mode=self.config.checkpoint_mode,
save_best_only=self.config.checkpoint_save_best_only,
save_weights_only=self.config.checkpoint_save_weights_only,
verbose=self.config.checkpoint_verbose,
)
)
self.callbacks.append(
TensorBoard(
log_dir=self.config.tensorboard_log_dir,
write_graph=self.config.tensorboard_write_graph,
)
)
# self.callbacks.append(
# LearningRateScheduler(self.lr_scheduler)
# )
if hasattr(self.config,"comet_api_key"):
from comet_ml import Experiment
experiment = Experiment(api_key=self.config.comet_api_key, project_name=self.config.exp_name)
experiment.disable_mp()
experiment.log_multiple_params(self.config)
self.callbacks.append(experiment.get_keras_callback())
def init_callbacks(self):
experiment = None
if hasattr(self.config, "comet_api_key"):
experiment = Experiment(api_key=self.config.comet_api_key,
project_name=self.config.experiment_name)
experiment.log_parameters(self.config)
self.callbacks.append(experiment.get_keras_callback())
self.callbacks.append(
Evaluater(eval_data=self.data_loader.get_val_generator(),
eval_steps=self.data_loader.get_val_steps(),
ref_data=self.data_loader.get_reference_data(),
config=self.config,
comet_experiment=experiment))
self.callbacks.append(
ReduceLROnPlateau(monitor=self.config.checkpoint_monitor,
factor=self.config.lr_reduce_factor,
patience=self.config.lr_reduce_patience,
verbose=1,
mode=self.config.checkpoint_mode))
self.callbacks.append(
ModelCheckpoint(
filepath=os.path.join(
self.config.checkpoint_dir,
'%s-{epoch:02d}-{val_accuracy:.2f}.hdf5' %
self.config.experiment_name),
monitor=self.config.checkpoint_monitor,
mode=self.config.checkpoint_mode,
save_best_only=self.config.checkpoint_save_best_only,
save_weights_only=self.config.checkpoint_save_weights_only,
verbose=self.config.checkpoint_verbose,
))
self.callbacks.append(
TensorBoard(
log_dir=self.config.tensorboard_log_dir,
write_graph=self.config.tensorboard_write_graph,
))
def init_callbacks(self):
self.callbacks.append(
ModelCheckpoint(
filepath=os.path.join(
self.config.callbacks.checkpoint_dir,
'%s-{epoch:02d}-{val_loss:.2f}.hdf5' %
self.config.exp.name),
monitor=self.config.callbacks.checkpoint_monitor,
mode=self.config.callbacks.checkpoint_mode,
save_best_only=self.config.callbacks.checkpoint_save_best_only,
save_weights_only=self.config.callbacks.
checkpoint_save_weights_only,
verbose=self.config.callbacks.checkpoint_verbose,
))
self.callbacks.append(
TensorBoard(
log_dir=self.config.callbacks.tensorboard_log_dir,
write_graph=self.config.callbacks.tensorboard_write_graph,
))
# if the config has the debug flag on, turn on tfdbg (TODO: make it work)
if hasattr(self.config, "debug"):
if (self.config.debug == True):
import keras.backend
from tensorflow.python import debug as tf_debug
print("#=========== DEBUG MODE ===========#")
sess = keras.backend.get_session()
sess = tf_debug.LocalCLIDebugWrapperSession(sess)
keras.backend.set_session(sess)
# if the config file has a comet_ml key, log on comet
if hasattr(self.config, "comet_api_key"):
from comet_ml import Experiment # PUT the import in main
experiment = Experiment(api_key=self.config.exp.comet_api_key,
project_name=self.config.exp.name)
experiment.disable_mp()
experiment.log_multiple_params(self.config)
self.callbacks.append(experiment.get_keras_callback())
def main(_):
experiment = Experiment(api_key="IowbTppLPOohqhcDtzxw76Cot")#comet_mlに保存
comet_callback = experiment.get_keras_callback()
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.allow_soft_placement = True
sess = tf.Session(config=config)
tf.keras.backend.set_session(sess)
trainingdatalist = ReadSliceDataList(args.trainingdatafile)
testdatalist = None
if args.testfile is not None:
testdatalist = ReadSliceDataList(args.testfile)
testdatalist = random.sample(testdatalist, int(len(testdatalist)*0.1))
(imageshape, labelshape) = GetInputShapes(trainingdatalist[0])
nclasses = 3 # Number of classes
print(imageshape)
print(labelshape)
#with tf.device('/device:GPU:{}'.format(args.gpuid)):##いらない？
####TPU用#######
x = tf.keras.layers.Input(shape=imageshape, name="x")
segmentation = ConstructModel(x, nclasses, not args.nobn, not args.nodropout)
model = tf.keras.models.Model(x, segmentation)
model.summary()
strategy = f.contrib.tpu.TPUDistributionStrategy( /
tf.contrib.cluster_resolver.TPUClusterResolver(tpu='grpc://' + os.environ['COLAB_TPU_ADDR']))
tpu_model = tf.contrib.tpu.keras_to_tpu_model(model, strategy=strategy)
optimizer = tf.keras.optimizers.Adam(lr=args.learningrate)
tpu_model.compile(loss=penalty_categorical, optimizer=optimizer, metrics=[kidney_dice, cancer_dice])
createParentPath(args.modelfile)
with open(args.modelfile, 'w') as f:
f.write(model.to_yaml())
if args.weightfile is None:
initial_epoch = 0
else:
model.load_weights(args.weightfile)
initial_epoch = args.initialepoch
if args.latestfile is None:
latestfile = args.logdir + '/latestweights.hdf5'
else:
latestfile = args.latestfile
createParentPath(latestfile)
tb_cbk = tf.keras.callbacks.TensorBoard(log_dir=args.logdir)
latest_cbk = LatestWeightSaver(latestfile)
callbacks = [tb_cbk, latest_cbk,comet_callback]######comet追加
if testdatalist is not None:
if args.bestfile is None:
bestfile = args.logdir + '/bestweights.hdf5'
else:
bestfile = args.bestfile
createParentPath(bestfile)
chkp_cbk = tf.keras.callbacks.ModelCheckpoint(filepath=bestfile, save_best_only = True, save_weights_only = True)
callbacks.append(chkp_cbk)
if args.weightinterval is not None:
periodic_cbk = PeriodicWeightSaver(logdir=args.logdir, interval=args.weightinterval)
callbacks.append(periodic_cbk)
steps_per_epoch = len(trainingdatalist) / args.batchsize
print ("Batch size: {}".format(args.batchsize))
print ("Number of Epochs: {}".format(args.epochs))
print ("Number of Steps/epoch: {}".format(steps_per_epoch))
#with tf.device('/device:GPU:{}'.format(args.gpuid)):#いらない？
if testdatalist is not None:
historys = tpu_model.fit_generator(ImportBatchArray(trainingdatalist, batch_size = args.batchsize, apply_augmentation = False),
steps_per_epoch = int(steps_per_epoch), epochs = args.epochs,
callbacks=callbacks,
validation_data = ImportBatchArray(testdatalist, batch_size = args.batchsize),
validation_steps = len(testdatalist),
initial_epoch = int(initial_epoch))
else:
historys = tpu_model.fit_generator(ImportBatchArray(trainingdatalist, batch_size = args.batchsize, apply_augmentation = False),
steps_per_epoch = int(steps_per_epoch), epochs = args.epochs,
callbacks=callbacks,
initial_epoch = int(initial_epoch))
loss = historys.history['dice']
val_loss = historys.history['val_dice']
epochs = len(loss)
history_file = open(args.history,"a")
for x in range(epochs):
print("{}\t{}".format(loss[x],val_loss[x]),file = history_file)
print("\n",file=history_file)
history_file.close()
