def _set_logdir(self): if self.name is None: self.name = datetime.datetime.now().strftime( "%Y-%m-%d-%H-%M-%S.%f") self.logdir = self.basedir / self.name config_file = self.logdir / 'config.json' if self.config is None: if config_file.exists(): config_dict = load_json(str(config_file)) self.config = Config(**config_dict) if not self.config.is_valid(): invalid_attr = self.config.is_valid(True)[1] raise ValueError('Invalid attributes in loaded config: ' + ', '.join(invalid_attr)) else: raise FileNotFoundError("config file doesn't exist: %s" % str(config_file.resolve())) else: if self.logdir.exists(): warnings.warn( 'output path for model already exists, files may be overwritten: %s' % str(self.logdir.resolve())) self.logdir.mkdir(parents=True, exist_ok=True) save_json(vars(self.config), str(config_file))
def Optimize(self): self.Y = [label(y) for y in tqdm(self.Y)] self.X = [normalize(x,1,99.8,axis= (0,1)) for x in tqdm(self.X)] print('Images to apply prediction on',len(self.X)) Yhat_val = [self.Starmodel.predict(x) for x in self.X] opt_prob_thresh, opt_measure, opt_nms_thresh = None, -np.inf, None for _opt_nms_thresh in self.nms_threshs: _opt_prob_thresh, _opt_measure = self.optimize_threshold(self.Y, Yhat_val, model=self.Starmodel, nms_thresh=_opt_nms_thresh) if _opt_measure > opt_measure: opt_prob_thresh, opt_measure, opt_nms_thresh = _opt_prob_thresh, _opt_measure, _opt_nms_thresh opt_threshs = dict(prob=opt_prob_thresh, nms=opt_nms_thresh) self.thresholds = opt_threshs print("Using optimized values: prob_thresh={prob:g}, nms_thresh={nms:g}.", opt_threshs) if self.basedir is not None: print("Saving to 'thresholds.json'.") save_json(opt_threshs, str(self.basedir + '/' + 'thresholds.json')) return opt_threshs
def optimize_thresholds(self, X_val, Y_val, nms_threshs=[0.3,0.4,0.5], iou_threshs=[0.3,0.5,0.7], predict_kwargs=None, optimize_kwargs=None, save_to_json=True): """Optimize two thresholds (probability, NMS overlap) necessary for predicting object instances. Note that the default thresholds yield good results in many cases, but optimizing the thresholds for a particular dataset can further improve performance. The optimized thresholds are automatically used for all further predictions and also written to the model directory. See ``utils.optimize_threshold`` for details and possible choices for ``optimize_kwargs``. Parameters ---------- X_val : list of ndarray (Validation) input images (must be normalized) to use for threshold tuning. Y_val : list of ndarray (Validation) label images to use for threshold tuning. nms_threshs : list of float List of overlap thresholds to be considered for NMS. For each value in this list, optimization is run to find a corresponding prob_thresh value. iou_threshs : list of float List of intersection over union (IOU) thresholds for which the (average) matching performance is considered to tune the thresholds. predict_kwargs: dict Keyword arguments for ``predict`` function of this class. (If not provided, will guess value for `n_tiles` to prevent out of memory errors.) optimize_kwargs: dict Keyword arguments for ``utils.optimize_threshold`` function. """ if predict_kwargs is None: predict_kwargs = {} if optimize_kwargs is None: optimize_kwargs = {} def _predict_kwargs(x): if 'n_tiles' in predict_kwargs: return predict_kwargs else: return {**predict_kwargs, 'n_tiles': self._guess_n_tiles(x), 'show_tile_progress': False} Yhat_val = [self.predict(x, **_predict_kwargs(x)) for x in X_val] opt_prob_thresh, opt_measure, opt_nms_thresh = None, -np.inf, None for _opt_nms_thresh in nms_threshs: _opt_prob_thresh, _opt_measure = optimize_threshold(Y_val, Yhat_val, model=self, nms_thresh=_opt_nms_thresh, iou_threshs=iou_threshs, **optimize_kwargs) if _opt_measure > opt_measure: opt_prob_thresh, opt_measure, opt_nms_thresh = _opt_prob_thresh, _opt_measure, _opt_nms_thresh opt_threshs = dict(prob=opt_prob_thresh, nms=opt_nms_thresh) self.thresholds = opt_threshs print(end='', file=sys.stderr, flush=True) print("Using optimized values: prob_thresh={prob:g}, nms_thresh={nms:g}.".format(prob=self.thresholds.prob, nms=self.thresholds.nms)) if save_to_json and self.basedir is not None: print("Saving to 'thresholds.json'.") save_json(opt_threshs, str(self.logdir / 'thresholds.json')) return opt_threshs
def export_to_dir(dirname): if len(model.inputs) > 1 or len(model.outputs) > 1: warnings.warn('Found multiple input or output layers.') builder = tf.saved_model.builder.SavedModelBuilder(dirname) # use name 'input'/'output' if there's just a single input/output layer inputs = dict(zip(model.input_names,model.inputs)) if len(model.inputs) > 1 else dict(input=model.input) outputs = dict(zip(model.output_names,model.outputs)) if len(model.outputs) > 1 else dict(output=model.output) signature = tf.saved_model.signature_def_utils.predict_signature_def(inputs=inputs, outputs=outputs) signature_def_map = { tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: signature } builder.add_meta_graph_and_variables(K.get_session(), [tf.saved_model.tag_constants.SERVING], signature_def_map=signature_def_map) builder.save() if meta is not None and len(meta) > 0: save_json(meta, os.path.join(dirname,'meta.json'))
def save(self, path): save_json(self._asdict(), path)