def load_models(self, load_epoch=None, stop_on_missing=True, init_layers=False):
if load_epoch == 'latest':
load_epoch = utils.get_latest_epoch_in_dir(self.models_dir)
self.logger.debug('Found latest epoch {} in dir {}'.format(load_epoch, self.models_dir))
if load_epoch is not None and int(load_epoch) > 0:
self.logger.debug('Looking for models in {}'.format(self.models_dir))
found_a_model = False
model_files = os.listdir(self.models_dir)
for m in self.models:
# model_filename = os.path.join(models_dir, '{}_epoch{}.h5'.format(m.name, load_epoch))
model_filename = [os.path.join(self.models_dir, mf) for mf in model_files \
if mf.split('_epoch')[0] == m.name and 'epoch{}'.format(load_epoch) in mf]
if len(model_filename) == 0:
self.logger.debug('Could not find any model files with name {} and epoch {}!'.format(
m.name, load_epoch
))
model_filename = None
if stop_on_missing:
sys.exit()
continue
else:
model_filename = model_filename[0]
if os.path.isfile(model_filename):
self.logger.debug('Loading model {} from {}'.format(m.name, model_filename))
try:
m.load_weights(model_filename)
except ValueError:
self.logger.debug('FAILED TO LOAD WEIGHTS DIRECTLY')
if not init_layers:
sys.exit()
except IndexError:
self.logger.debug('FAILED TO LOAD WEIGHTS DIRECTLY')
if not init_layers:
sys.exit()
found_a_model = True
elif not os.path.isfile(model_filename):
self.logger.debug('Could not find model file {}!'.format(model_filename))
if stop_on_missing:
sys.exit()
if not found_a_model:
self.logger.debug(
'Did not find any models with epoch {} in dir {}!'.format(load_epoch, self.models_dir))
load_epoch = 0
self.latest_epoch = int(load_epoch) + 1
return int(load_epoch) + 1
else:
return 0
def load_models(self,
load_epoch=None,
stop_on_missing=True,
init_layers=False):
if load_epoch == 'latest':
load_epoch = utils.get_latest_epoch_in_dir(self.models_dir)
self.logger.debug('Found latest epoch {} in dir {}'.format(
load_epoch, self.models_dir))
# do this first so we look for the correct (not pre-softmax) model
if load_epoch is not None:
self.update_epoch_count(int(load_epoch))
start_epoch = super(SegmenterTrainer, self).load_models(load_epoch)
return start_epoch