def load_model(cls, path):
for item_entry in os.scandir(path):
if item_entry.name == 'extraction_params.json':
params = data_functions.load_json(item_entry.path)
self.threshold = float(params['threshold'])
self.hist_type = params['hist_type']
if item_entry.name.split('.')[-1] == 'pickle':
self.cls_model = data_functions.load_pickle(item_entry.path)
def load_models(self):
self.unet_model = ClarifruitUnet.load_model(self.unet_model_path)
for item_entry in os.scandir(self.classifier_path):
if item_entry.name == 'extraction_params.json':
params = data_functions.load_json(item_entry.path)
self.threshold = float(params['threshold'])
self.hist_type = params['hist_type']
if item_entry.name.split('.')[-1] == 'pickle':
self.cls_model = data_functions.load_pickle(item_entry.path)
def load_model(cls, src_path, update_dict=None, steps=None):
"""
load a pretrained model located in the src_path
:param src_path: the path containing the pretrained model
:param update_dict: optional, dict, a dictionary of parameter used to
modifiey the loaded model
:param steps:optional,int, the steps from which to load the model,
if None, the latest weights and parameters are loaded, default None
:return: a ClarifruitUnet instance
"""
if steps is not None:
json_file, _ = cls.get_file_via_steps(src_path, steps, 'json',
STEPS_REGEX)
hdf5_file, samples_seen = cls.get_file_via_steps(
src_path, steps, 'hdf5', STEPS_REGEX)
else:
json_file = max(glob.iglob(os.path.join(src_path, '*.json')),
key=os.path.getctime)
hdf5_file = max(glob.iglob(os.path.join(src_path, '*.hdf5')),
key=os.path.getctime)
samples_seen = cls.get_pattern(hdf5_file, STEPS_REGEX)
samples_seen = samples_seen if samples_seen is not None else 0
session_number = cls.get_pattern(hdf5_file, SESS_REGEX)
session_number = session_number if session_number is not None else 1
params_dict = data_functions.load_json(json_file)
params_dict['pretrained_weights'] = hdf5_file
#TODO: try to rearange loading weights
# if 'weights' in os.path.basename(hdf5_file):
# params_dict['pretrained_weights'] = hdf5_file
# else:
# params_dict['checkpoint'] = hdf5_file
params_dict['train_time'] = os.path.basename(src_path)
if update_dict is not None:
if 'pretrained_weights' or 'checkpoint' in update_dict:
params_dict['pretrained_weights'] = None
params_dict['checkpoint'] = None
params_dict.update(update_dict)
model = ClarifruitUnet(**params_dict)
logger.info(f"continuing training from {os.path.basename(hdf5_file)}")
setattr(model, 'samples_seen', samples_seen)
setattr(model, 'params_filepath', json_file)
setattr(model, 'session_number', session_number)
return model
def train_unet(params_dict_path, src_path, update_dict_path, steps):
log_path = data_functions.create_path(LOG_PATH, 'unet_logs')
configure_logger(name="cherry_stem",
console_level='INFO',
file_level='INFO',
out_path=log_path)
logger = logging.getLogger(__name__)
try:
if params_dict_path is not None:
if src_path is not None:
message = 'Ambiguous loading paths, input either' \
' "params_dict_path or "src_path", not both '
raise UnetModelException(message)
else:
params_dict = data_functions.load_json(params_dict_path)
model = unet_model_functions.ClarifruitUnet(**params_dict)
else:
if update_dict_path is not None:
update_dict = data_functions.load_json(update_dict_path)
else:
update_dict = None
model = unet_model_functions.ClarifruitUnet.load_model(
src_path, update_dict, steps)
keras_logs_path = model.set_model_for_train()
logger.info(f"for tensorboard use \n "
f"tensorboard --logdir={keras_logs_path}")
model.fit_unet()
except:
message = 'Error loading Model'
logger.exception(message)
raise Exception
def create_histogrames(path=None):
log_path = data_functions.create_path(LOG_PATH, 'extract_logs')
configure_logger(name="stem_extractor",
console_level='INFO',
file_level='INFO',
out_path=log_path)
logger = logging.getLogger(__name__)
try:
params_dict = data_functions.load_json(path)
stem_extract.create_test_train_obj(**params_dict)
except:
message = 'Error loading Model'
logger.exception(message)
raise Exception
def save_model_params(self):
"""
save the current ClarifruitUnet instance parameters to a json file
:return:
"""
params_dict = self.get_model_params()
if self.params_filepath is not None:
file_params = data_functions.load_json(self.params_filepath)
if file_params != params_dict: # cheking if the parametes for this
# session are diffrent then those
# in the source file
self.session_number += 1
curr_file_name = (self.params_file_name + PARAMS_UPDATE_FORMAT +
'json').format(sess=self.session_number,
steps=self.samples_seen)
data_functions.save_json(params_dict, curr_file_name, self.curr_folder)
self.params_filepath = os.path.join(self.curr_folder, curr_file_name)