def write_set(protos, record_path): logger.info(f"writing {record_path}...") with tf.python_io.TFRecordWriter(record_path) as writer: for i, proto in enumerate(protos): if i % 100 == 0: logger.info(f"writing example {i}") writer.write(proto)
def evaluate(self, art_data, multiscale=False): """Runs evaluation for UNet.""" if tf.executing_eagerly(): tile_labels = [] errors = [] outputs = [] total_num_failed = 0 for i, (batch_tiles, batch_labels) in enumerate(art_data.evaluation_input()): if i % 10 == 0: logger.info( f"evaluating batch {i} / {art_data.steps_per_epoch}") tile_labels += list(batch_labels) outputs += _unbatch_outputs( self.model.predict_on_batch(batch_tiles)) while len(outputs) >= art_data.num_tiles: label = art_data.untile_points( tile_labels[:art_data.num_tiles]) prediction = art_data.analyze_outputs( outputs, multiscale=multiscale) error, num_failed = dat.evaluate_prediction( label, prediction) total_num_failed += num_failed errors += error[error[:, 0] >= 0].tolist() del tile_labels[:art_data.num_tiles] del outputs[:art_data.num_tiles] else: raise NotImplementedError("evaluation on patient execution") return np.array(errors), total_num_failed
def rm(path): if not os.path.exists(path): return if os.path.isfile(path): os.remove(path) elif os.path.isdir(path): shutil.rmtree(path) else: raise RuntimeError(f"bad path: {path}") logger.info(f"removed {path}.")
def predict(self): """Run prediction on the unlabeled set.""" unlabeled_set = self._load_unlabeled() model = self._load_model() start_time = time() predictions = list( model.predict(unlabeled_set, multiscale=self.multiscale)) logger.info(f"ran prediction in {time() - start_time}s.") logger.debug(f"prediction:\n{predictions}") fname = join(self.model_root, 'predictions.npy') np.save(fname, predictions) logger.info(f"saved {len(predictions)} predictions to {fname}.")
def postprocess(self, dataset, mode, cache=False): if "ENUMERATED" in mode: dataset = dataset.apply(tf.data.experimental.enumerate_dataset()) if cache: logger.info("caching this epoch...") dataset = dataset.repeat(-1).take(self.size).cache(self.cache_dir) dataset = dataset.batch(self.batch_size, drop_remainder=True) if mode == ArtificeData.TRAINING: dataset = dataset.shuffle(self.num_shuffle) dataset = dataset.repeat(-1) if mode != ArtificeData.TRAINING: dataset = dataset.take(self.steps_per_epoch) dataset = dataset.prefetch(self.prefetch_buffer_size) return dataset
def run(self, seconds=-1): """Run for at most `seconds`. If `seconds` is negative, run forever.""" start_time = time() for i in itertools.count(): examples = [] idxs = [] for _ in range(self.record_size): idx = self.info.pop() while idx is None: logger.info( "waiting {self.sleep_duration}s for more selections..." ) sleep(self.sleep_duration) idx = self.info.pop() entry = self.data_set.get_entry(idx) logger.info(f"annotating example {idx}...") examples.append(self.annotate(entry)) idxs.append(idx) record_name = self._generate_record_name() dat.write_set(map(dat.proto_from_annotated_example, examples), record_name) self.info.finalize(idxs) logger.info(f"saved {i}'th annotated set to {record_name}.") if time() - start_time > seconds > 0: logger.info(f"finished after {seconds}s.") break
def load_weights(self, checkpoint_path=None): """Update the model weights from the chekpoint file. :param checkpoint_path: checkpoint path to use. If not provided, uses the class name to construct a checkpoint path. """ if checkpoint_path is None: checkpoint_path = self.checkpoint_path if os.path.exists(checkpoint_path): self.model.load_weights(checkpoint_path, by_name=True) # todo: by_name? logger.info(f"loaded model weights from {checkpoint_path}") else: logger.info(f"no checkpoint at {checkpoint_path}")
def accumulate(self, accumulator): """Runs the accumulators across the dataset. An accumulator function should take a `entry` and an `aggregate` object. On the first call, `aggregate` will be None. Afterward, each accumulator will be passed the output from its previous call as `aggregate`, as well as the next entry in the data as 'entry'. On the final call, `entry` will be None, allowing for post-processing. If the accumulator returns None for aggregate, the accumulation is terminated early. :param accumulator: an accumulator function OR a dictionary mapping names to accumulator functions :returns: aggregate from `accumulator` OR a dictionary of aggregates with the same keys as `accumulators`. :rtype: dict """ if type(accumulator) == dict: accumulators = accumulator else: accumulators = {0: accumulator} aggregates = dict.fromkeys(accumulators.keys()) finished = dict([(k, False) for k in accumulators.keys()]) if tf.executing_eagerly(): for entry in self.dataset: if all(finished.values()): break for k, acc in accumulators.items(): if finished[k]: continue agg = acc(tuple(t.numpy() for t in entry), aggregates[k]) if agg is None: finished[k] = True else: aggregates[k] = agg else: raise NotImplementedError logger.info("finished accumulation") for k, acc in accumulators.items(): aggregates[k] = acc(None, aggregates[k]) if type(accumulator) == dict: return aggregates else: return aggregates[0]
def vis_predict(self): """Run prediction on the test set and visualize the output.""" history_files = glob(join(self.model_dir, '*history.json')) hists = dict( (fname, utils.json_load(fname)) for fname in history_files) vis.plot_hist(hists) test_set = self._load_test() model = self._load_model() for image, dist_image, prediction in model.predict_visualization( test_set): fig, axes = vis.plot_image(image, image, dist_image, colorbar=True) axes[0, 1].plot(prediction[:, 1], prediction[:, 0], 'rx') axes[0, 2].plot(prediction[:, 1], prediction[:, 0], 'rx') logger.info(f"prediction:\n{prediction}") vis.show(join(self.figs_dir, 'prediction.pdf')) if not self.show: break
def show(fname=None, save=False): """Show the figure currently in matplotlib or save it, if not self.show. If no fname provided, and self.show is False, then closes the figure. If save is True, figure is saved regardless of show. :param fname: name of the file to save to. :param save: whether to save the file. """ if _show and not save: logger.info("showing figure...") plt.show() elif fname is None: logger.warning("Cannot save figure. Did you forget to set --show?") plt.close() else: plt.savefig(fname) logger.info(f"saved figure to {fname}.")
def train(self, art_data, initial_epoch=0, epochs=1, seconds=0, **kwargs): """Fits the model, saving it along the way, and reloads every epoch. :param art_data: ArtificeData set :param initial_epoch: epoch that training is starting from :param epochs: epoch number to stop at. If -1, training continues forever. :param seconds: seconds after which to stop reloading every epoch. If -1, reload is never stopped. If 0, dataset is loaded only once, at beginning. :returns: history dictionary """ if (initial_epoch > 0 and os.path.exists(self.history_path) and not self.overwrite): hist = utils.json_load(self.history_path) else: hist = {} start_time = time() epoch = initial_epoch while epoch != epochs and time() - start_time > seconds > 0: logger.info("reloading dataset (not cached)...") hist = self.fit(art_data, hist=hist, initial_epoch=epoch, epochs=(epoch + 1), **kwargs) epoch += 1 if epoch != epochs: hist = self.fit(art_data, hist=hist, initial_epoch=epoch, epochs=epochs, **kwargs) self.save() return hist
def predict(self, art_data, multiscale=False): """Run prediction, reassembling tiles, with the Artifice data.""" if tf.executing_eagerly(): outputs = [] for i, batch in enumerate(art_data.prediction_input()): if i % 100 == 0: logger.info(f"batch {i} / {art_data.steps_per_epoch}") outputs += _unbatch_outputs( self.model.predict_on_batch(batch)) while len(outputs) >= art_data.num_tiles: prediction = art_data.analyze_outputs( outputs, multiscale=multiscale) yield prediction del outputs[:art_data.num_tiles] else: raise NotImplementedError( "enable eager execution for eval (remove --patient)") outputs = [] next_batch = ( art_data.prediction_input().make_one_shot_iterator().get_next()) with tf.Session() as sess: sess.run(tf.global_variables_initializer()) for i in itertools.count(): try: batch = sess.run(next_batch) except tf.errors.OutOfRangeError: return if i % 100 == 0: logger.info(f"batch {i} / {art_data.steps_per_epoch}") outputs += _unbatch_outputs( self.model.predict_on_batch(batch)) while len(outputs) >= art_data.num_tiles: prediction = art_data.analyze_outputs( outputs, multiscale=multiscale) yield prediction del outputs[:art_data.num_tiles]
def run(self, seconds=-1): """Run for at most `seconds`. If `seconds` is negative, run forever.""" start_time = time() if tf.executing_eagerly(): dataset = self.data_set.enumerated_prediction_input().repeat(-1) for indices, images in dataset: logger.info(f"evaluating priorities for {indices}...") priorities = list(self.prioritize(images)) self.info.push(list(zip(list(indices), priorities))) logger.info(f"pushed {indices} with priorities {priorities}.") if time() - start_time > seconds > 0: logger.info(f"finished after {seconds}s.") break else: raise NotImplementedError("patient execution")
def main(): _system_checks() parser = argparse.ArgumentParser(description=docs.description) parser.add_argument('commands', nargs='+', help=docs.commands) # file settings parser.add_argument('--data-root', '--input', '-i', nargs=1, default=['data/default'], help=docs.data_root) parser.add_argument('--model-root', '--model-dir', '-m', nargs=1, default=['models/tmp'], help=docs.model_root) parser.add_argument('--overwrite', '-f', action='store_true', help=docs.overwrite) parser.add_argument('--deep', action='store_true', help=docs.deep) parser.add_argument('--figs-dir', '--figures', nargs=1, default=['figs'], help=docs.figs_dir) # data settings parser.add_argument('--convert-mode', nargs='+', default=[0, 4], type=int, help=docs.convert_mode) parser.add_argument('--transformation', '--augment', '-a', nargs='?', default=None, const=0, type=int, help=docs.transformation) parser.add_argument('--identity-prob', nargs=1, default=[0.01], type=float, help=docs.identity_prob) parser.add_argument('--priority-mode', '--priority', nargs=1, default=['random'], help=docs.priority_mode) parser.add_argument('--labeled', action='store_true', help=docs.labeled) # annotation settings parser.add_argument('--annotation-mode', '--annotate', nargs=1, default=['disks'], help=docs.annotation_mode) parser.add_argument('--record-size', nargs=1, default=[10], type=int, help=docs.record_size) parser.add_argument('--annotation-delay', nargs=1, default=[60], type=float, help=docs.annotation_delay) # sizes relating to data parser.add_argument('--image-shape', '--shape', '-s', nargs=3, type=int, default=[500, 500, 1], help=docs.image_shape) parser.add_argument('--data-size', '-N', nargs=1, default=[10000], type=int, help=docs.data_size) parser.add_argument('--test-size', '-T', nargs=1, default=[1000], type=int, help=docs.test_size) parser.add_argument('--batch-size', '-b', nargs=1, default=[16], type=int, help=docs.batch_size) parser.add_argument('--num-objects', '-n', nargs=1, default=[40], type=int, help=docs.num_objects) parser.add_argument('--pose-dim', '-p', nargs=1, default=[2], type=int, help=docs.pose_dim) parser.add_argument('--num-shuffle', nargs=1, default=[1000], type=int, help=docs.num_shuffle) # model architecture parser.add_argument('--base-shape', nargs='+', default=[28], type=int, help=docs.base_shape) parser.add_argument('--level-filters', nargs='+', default=[128, 64, 32], type=int, help=docs.level_filters) parser.add_argument('--level-depth', nargs='+', default=[2], type=int, help=docs.level_depth) # sparse eval and other optimization settings parser.add_argument('--model', '-M', nargs='?', default='unet', help=docs.model) parser.add_argument('--multiscale', action='store_true', help=docs.multiscale) parser.add_argument('--use-var', action='store_true', help=docs.use_var) # model hyperparameters parser.add_argument('--dropout', nargs=1, default=[0.5], type=float, help=docs.dropout) parser.add_argument('--initial-epoch', nargs=1, default=[0], type=int, help=docs.initial_epoch) # todo: get from ckpt parser.add_argument('--epochs', '-e', nargs=1, default=[1], type=int, help=docs.epochs) parser.add_argument('--learning-rate', '-l', nargs=1, default=[0.1], type=float, help=docs.learning_rate) parser.add_argument('--tol', nargs=1, default=[0.1], type=float, help=docs.tol) # runtime settings parser.add_argument('--num-parallel-calls', '--cores', nargs=1, default=[-1], type=int, help=docs.num_parallel_calls) parser.add_argument('--verbose', '-v', nargs='?', const=1, default=2, type=int, help=docs.verbose) parser.add_argument('--keras-verbose', nargs='?', const=2, default=1, type=int, help=docs.keras_verbose) parser.add_argument('--patient', action='store_true', help=docs.patient) parser.add_argument('--show', action='store_true', help=docs.show) parser.add_argument('--cache', action='store_true', help=docs.cache) parser.add_argument('--seconds', '--time', '--reload', '-t', '-r', nargs='?', default=0, const=-1, type=int, help=docs.seconds) args = parser.parse_args() art = Artifice(commands=args.commands, convert_mode=args.convert_mode, transformation=args.transformation, identity_prob=args.identity_prob[0], priority_mode=args.priority_mode[0], labeled=args.labeled, annotation_mode=args.annotation_mode[0], record_size=args.record_size[0], annotation_delay=args.annotation_delay[0], data_root=args.data_root[0], model_root=args.model_root[0], overwrite=args.overwrite, deep=args.deep, figs_dir=args.figs_dir[0], image_shape=args.image_shape, data_size=args.data_size[0], test_size=args.test_size[0], batch_size=args.batch_size[0], num_objects=args.num_objects[0], pose_dim=args.pose_dim[0], num_shuffle=args.num_shuffle[0], base_shape=args.base_shape, level_filters=args.level_filters, level_depth=args.level_depth[0], model=args.model, multiscale=args.multiscale, use_var=args.use_var, dropout=args.dropout[0], initial_epoch=args.initial_epoch[0], epochs=args.epochs[0], learning_rate=args.learning_rate[0], tol=args.tol[0], num_parallel_calls=args.num_parallel_calls[0], verbose=args.verbose, keras_verbose=args.keras_verbose, eager=(not args.patient), show=args.show, cache=args.cache, seconds=args.seconds) logger.info(art) art()
def _ensure_dirs_exist(dirs): for path in dirs: if not exists(path): logger.info(f"creating '{path}'") os.makedirs(path)
def evaluate(self): test_set = self._load_test() model = self._load_model() errors, num_failed = model.evaluate(test_set) if not errors: logger.warning(f"found ZERO objects, num_failed: {num_failed}") return avg_error = errors.mean(axis=0) total_num_objects = self.test_size * self.num_objects num_detected = total_num_objects - num_failed logger.info(f"objects detected: {num_detected} / " f"{total_num_objects}") logger.info(f"avg (euclidean) detection error: {avg_error[0]}") logger.info(f"avg (absolute) pose error: {avg_error[1:]}") logger.info( "note: some objects may be occluded, making detection impossible") logger.info(f"avg: {errors.mean(axis=0)}") logger.info(f"std: {errors.std(axis=0)}") logger.info(f"min: {errors.min(axis=0)}") logger.info(f"max: {errors.max(axis=0)}")