def init_callbacks(self): self.callbacks.append( ModelCheckpoint( filepath=os.path.join( self.config.callbacks.checkpoint_dir, '%s-{epoch:02d}-{val_loss:.2f}.hdf5' % self.config.exp.name), monitor=self.config.callbacks.checkpoint_monitor, mode=self.config.callbacks.checkpoint_mode, save_best_only=self.config.callbacks.checkpoint_save_best_only, save_weights_only=self.config.callbacks. checkpoint_save_weights_only, verbose=self.config.callbacks.checkpoint_verbose, )) self.callbacks.append( TensorBoard( log_dir=self.config.callbacks.tensorboard_log_dir, write_graph=self.config.callbacks.tensorboard_write_graph, )) if hasattr(self.config, "comet_api_key"): from comet_ml import Experiment experiment = Experiment(api_key=self.config.comet_api_key, project_name=self.config.exp_name) experiment.disable_mp() experiment.log_multiple_params(self.config) self.callbacks.append(experiment.get_keras_callback())
def init_callbacks(self): self.callbacks.append( ModelCheckpoint( filepath=os.path.join( self.checkpoint_dir, '%s-{epoch:02d}-{val_loss:.2f}.hdf5' % self.config['exp_name']), monitor=self.config['checkpoint_monitor'], mode=self.config['checkpoint_mode'], save_best_only=self.config['checkpoint_save_best_only'], save_weights_only=self.config['checkpoint_save_weights_only'], verbose=self.config['checkpoint_verbose'], )) self.callbacks.append( TensorBoard( log_dir=self.tensorboard_log_dir, write_graph=self.config['tensorboard_write_graph'], histogram_freq=0, # don't compute histograms write_images= False # don't write model weights to visualize as image in TensorBoard )) if hasattr(self.config, "comet_api_key"): from comet_ml import Experiment experiment = Experiment(api_key=self.config['comet_api_key'], project_name=self.config['exp_name']) experiment.disable_mp() experiment.log_multiple_params(self.config) self.callbacks.append(experiment.get_keras_callback())
def init_callbacks(self): self.callbacks.append( ModelCheckpoint( filepath=os.path.join(self.config.checkpoint_dir, '%s-{epoch:03d}-{val_nme:.5f}.hdf5' % self.config.exp_name), monitor=self.config.checkpoint_monitor, mode=self.config.checkpoint_mode, save_best_only=self.config.checkpoint_save_best_only, save_weights_only=self.config.checkpoint_save_weights_only, verbose=self.config.checkpoint_verbose, ) ) self.callbacks.append( TensorBoard( log_dir=self.config.tensorboard_log_dir, write_graph=self.config.tensorboard_write_graph, ) ) # self.callbacks.append( # LearningRateScheduler(self.lr_scheduler) # ) if hasattr(self.config,"comet_api_key"): from comet_ml import Experiment experiment = Experiment(api_key=self.config.comet_api_key, project_name=self.config.exp_name) experiment.disable_mp() experiment.log_multiple_params(self.config) self.callbacks.append(experiment.get_keras_callback())
def init_callbacks(self): experiment = None if hasattr(self.config, "comet_api_key"): experiment = Experiment(api_key=self.config.comet_api_key, project_name=self.config.experiment_name) experiment.log_parameters(self.config) self.callbacks.append(experiment.get_keras_callback()) self.callbacks.append( Evaluater(eval_data=self.data_loader.get_val_generator(), eval_steps=self.data_loader.get_val_steps(), ref_data=self.data_loader.get_reference_data(), config=self.config, comet_experiment=experiment)) self.callbacks.append( ReduceLROnPlateau(monitor=self.config.checkpoint_monitor, factor=self.config.lr_reduce_factor, patience=self.config.lr_reduce_patience, verbose=1, mode=self.config.checkpoint_mode)) self.callbacks.append( ModelCheckpoint( filepath=os.path.join( self.config.checkpoint_dir, '%s-{epoch:02d}-{val_accuracy:.2f}.hdf5' % self.config.experiment_name), monitor=self.config.checkpoint_monitor, mode=self.config.checkpoint_mode, save_best_only=self.config.checkpoint_save_best_only, save_weights_only=self.config.checkpoint_save_weights_only, verbose=self.config.checkpoint_verbose, )) self.callbacks.append( TensorBoard( log_dir=self.config.tensorboard_log_dir, write_graph=self.config.tensorboard_write_graph, ))
def init_callbacks(self): self.callbacks.append( ModelCheckpoint( filepath=os.path.join( self.config.callbacks.checkpoint_dir, '%s-{epoch:02d}-{val_loss:.2f}.hdf5' % self.config.exp.name), monitor=self.config.callbacks.checkpoint_monitor, mode=self.config.callbacks.checkpoint_mode, save_best_only=self.config.callbacks.checkpoint_save_best_only, save_weights_only=self.config.callbacks. checkpoint_save_weights_only, verbose=self.config.callbacks.checkpoint_verbose, )) self.callbacks.append( TensorBoard( log_dir=self.config.callbacks.tensorboard_log_dir, write_graph=self.config.callbacks.tensorboard_write_graph, )) # if the config has the debug flag on, turn on tfdbg (TODO: make it work) if hasattr(self.config, "debug"): if (self.config.debug == True): import keras.backend from tensorflow.python import debug as tf_debug print("#=========== DEBUG MODE ===========#") sess = keras.backend.get_session() sess = tf_debug.LocalCLIDebugWrapperSession(sess) keras.backend.set_session(sess) # if the config file has a comet_ml key, log on comet if hasattr(self.config, "comet_api_key"): from comet_ml import Experiment # PUT the import in main experiment = Experiment(api_key=self.config.exp.comet_api_key, project_name=self.config.exp.name) experiment.disable_mp() experiment.log_multiple_params(self.config) self.callbacks.append(experiment.get_keras_callback())
def main(_): experiment = Experiment(api_key="IowbTppLPOohqhcDtzxw76Cot")#comet_mlに保存 comet_callback = experiment.get_keras_callback() config = tf.ConfigProto() config.gpu_options.allow_growth = True config.allow_soft_placement = True sess = tf.Session(config=config) tf.keras.backend.set_session(sess) trainingdatalist = ReadSliceDataList(args.trainingdatafile) testdatalist = None if args.testfile is not None: testdatalist = ReadSliceDataList(args.testfile) testdatalist = random.sample(testdatalist, int(len(testdatalist)*0.1)) (imageshape, labelshape) = GetInputShapes(trainingdatalist[0]) nclasses = 3 # Number of classes print(imageshape) print(labelshape) #with tf.device('/device:GPU:{}'.format(args.gpuid)):##いらない? ####TPU用####### x = tf.keras.layers.Input(shape=imageshape, name="x") segmentation = ConstructModel(x, nclasses, not args.nobn, not args.nodropout) model = tf.keras.models.Model(x, segmentation) model.summary() strategy = f.contrib.tpu.TPUDistributionStrategy( / tf.contrib.cluster_resolver.TPUClusterResolver(tpu='grpc://' + os.environ['COLAB_TPU_ADDR'])) tpu_model = tf.contrib.tpu.keras_to_tpu_model(model, strategy=strategy) optimizer = tf.keras.optimizers.Adam(lr=args.learningrate) tpu_model.compile(loss=penalty_categorical, optimizer=optimizer, metrics=[kidney_dice, cancer_dice]) createParentPath(args.modelfile) with open(args.modelfile, 'w') as f: f.write(model.to_yaml()) if args.weightfile is None: initial_epoch = 0 else: model.load_weights(args.weightfile) initial_epoch = args.initialepoch if args.latestfile is None: latestfile = args.logdir + '/latestweights.hdf5' else: latestfile = args.latestfile createParentPath(latestfile) tb_cbk = tf.keras.callbacks.TensorBoard(log_dir=args.logdir) latest_cbk = LatestWeightSaver(latestfile) callbacks = [tb_cbk, latest_cbk,comet_callback]######comet追加 if testdatalist is not None: if args.bestfile is None: bestfile = args.logdir + '/bestweights.hdf5' else: bestfile = args.bestfile createParentPath(bestfile) chkp_cbk = tf.keras.callbacks.ModelCheckpoint(filepath=bestfile, save_best_only = True, save_weights_only = True) callbacks.append(chkp_cbk) if args.weightinterval is not None: periodic_cbk = PeriodicWeightSaver(logdir=args.logdir, interval=args.weightinterval) callbacks.append(periodic_cbk) steps_per_epoch = len(trainingdatalist) / args.batchsize print ("Batch size: {}".format(args.batchsize)) print ("Number of Epochs: {}".format(args.epochs)) print ("Number of Steps/epoch: {}".format(steps_per_epoch)) #with tf.device('/device:GPU:{}'.format(args.gpuid)):#いらない? if testdatalist is not None: historys = tpu_model.fit_generator(ImportBatchArray(trainingdatalist, batch_size = args.batchsize, apply_augmentation = False), steps_per_epoch = int(steps_per_epoch), epochs = args.epochs, callbacks=callbacks, validation_data = ImportBatchArray(testdatalist, batch_size = args.batchsize), validation_steps = len(testdatalist), initial_epoch = int(initial_epoch)) else: historys = tpu_model.fit_generator(ImportBatchArray(trainingdatalist, batch_size = args.batchsize, apply_augmentation = False), steps_per_epoch = int(steps_per_epoch), epochs = args.epochs, callbacks=callbacks, initial_epoch = int(initial_epoch)) loss = historys.history['dice'] val_loss = historys.history['val_dice'] epochs = len(loss) history_file = open(args.history,"a") for x in range(epochs): print("{}\t{}".format(loss[x],val_loss[x]),file = history_file) print("\n",file=history_file) history_file.close()