def write_set(protos, record_path):
logger.info(f"writing {record_path}...")
with tf.python_io.TFRecordWriter(record_path) as writer:
for i, proto in enumerate(protos):
if i % 100 == 0:
logger.info(f"writing example {i}")
writer.write(proto)
def evaluate(self, art_data, multiscale=False):
"""Runs evaluation for UNet."""
if tf.executing_eagerly():
tile_labels = []
errors = []
outputs = []
total_num_failed = 0
for i, (batch_tiles,
batch_labels) in enumerate(art_data.evaluation_input()):
if i % 10 == 0:
logger.info(
f"evaluating batch {i} / {art_data.steps_per_epoch}")
tile_labels += list(batch_labels)
outputs += _unbatch_outputs(
self.model.predict_on_batch(batch_tiles))
while len(outputs) >= art_data.num_tiles:
label = art_data.untile_points(
tile_labels[:art_data.num_tiles])
prediction = art_data.analyze_outputs(
outputs, multiscale=multiscale)
error, num_failed = dat.evaluate_prediction(
label, prediction)
total_num_failed += num_failed
errors += error[error[:, 0] >= 0].tolist()
del tile_labels[:art_data.num_tiles]
del outputs[:art_data.num_tiles]
else:
raise NotImplementedError("evaluation on patient execution")
return np.array(errors), total_num_failed
def rm(path):
if not os.path.exists(path):
return
if os.path.isfile(path):
os.remove(path)
elif os.path.isdir(path):
shutil.rmtree(path)
else:
raise RuntimeError(f"bad path: {path}")
logger.info(f"removed {path}.")
def predict(self):
"""Run prediction on the unlabeled set."""
unlabeled_set = self._load_unlabeled()
model = self._load_model()
start_time = time()
predictions = list(
model.predict(unlabeled_set, multiscale=self.multiscale))
logger.info(f"ran prediction in {time() - start_time}s.")
logger.debug(f"prediction:\n{predictions}")
fname = join(self.model_root, 'predictions.npy')
np.save(fname, predictions)
logger.info(f"saved {len(predictions)} predictions to {fname}.")
def postprocess(self, dataset, mode, cache=False):
if "ENUMERATED" in mode:
dataset = dataset.apply(tf.data.experimental.enumerate_dataset())
if cache:
logger.info("caching this epoch...")
dataset = dataset.repeat(-1).take(self.size).cache(self.cache_dir)
dataset = dataset.batch(self.batch_size, drop_remainder=True)
if mode == ArtificeData.TRAINING:
dataset = dataset.shuffle(self.num_shuffle)
dataset = dataset.repeat(-1)
if mode != ArtificeData.TRAINING:
dataset = dataset.take(self.steps_per_epoch)
dataset = dataset.prefetch(self.prefetch_buffer_size)
return dataset
def run(self, seconds=-1):
"""Run for at most `seconds`. If `seconds` is negative, run forever."""
start_time = time()
for i in itertools.count():
examples = []
idxs = []
for _ in range(self.record_size):
idx = self.info.pop()
while idx is None:
logger.info(
"waiting {self.sleep_duration}s for more selections..."
)
sleep(self.sleep_duration)
idx = self.info.pop()
entry = self.data_set.get_entry(idx)
logger.info(f"annotating example {idx}...")
examples.append(self.annotate(entry))
idxs.append(idx)
record_name = self._generate_record_name()
dat.write_set(map(dat.proto_from_annotated_example, examples),
record_name)
self.info.finalize(idxs)
logger.info(f"saved {i}'th annotated set to {record_name}.")
if time() - start_time > seconds > 0:
logger.info(f"finished after {seconds}s.")
break
def load_weights(self, checkpoint_path=None):
"""Update the model weights from the chekpoint file.
:param checkpoint_path: checkpoint path to use. If not provided, uses the
class name to construct a checkpoint path.
"""
if checkpoint_path is None:
checkpoint_path = self.checkpoint_path
if os.path.exists(checkpoint_path):
self.model.load_weights(checkpoint_path,
by_name=True) # todo: by_name?
logger.info(f"loaded model weights from {checkpoint_path}")
else:
logger.info(f"no checkpoint at {checkpoint_path}")
def accumulate(self, accumulator):
"""Runs the accumulators across the dataset.
An accumulator function should take a `entry` and an `aggregate` object. On
the first call, `aggregate` will be None. Afterward, each accumulator will
be passed the output from its previous call as `aggregate`, as well as the
next entry in the data as 'entry'. On the final call, `entry` will be None,
allowing for post-processing.
If the accumulator returns None for aggregate, the accumulation is
terminated early.
:param accumulator: an accumulator function OR a dictionary mapping names
to accumulator functions
:returns: aggregate from `accumulator` OR a dictionary of aggregates with
the same keys as `accumulators`.
:rtype: dict
"""
if type(accumulator) == dict:
accumulators = accumulator
else:
accumulators = {0: accumulator}
aggregates = dict.fromkeys(accumulators.keys())
finished = dict([(k, False) for k in accumulators.keys()])
if tf.executing_eagerly():
for entry in self.dataset:
if all(finished.values()):
break
for k, acc in accumulators.items():
if finished[k]:
continue
agg = acc(tuple(t.numpy() for t in entry), aggregates[k])
if agg is None:
finished[k] = True
else:
aggregates[k] = agg
else:
raise NotImplementedError
logger.info("finished accumulation")
for k, acc in accumulators.items():
aggregates[k] = acc(None, aggregates[k])
if type(accumulator) == dict:
return aggregates
else:
return aggregates[0]
def vis_predict(self):
"""Run prediction on the test set and visualize the output."""
history_files = glob(join(self.model_dir, '*history.json'))
hists = dict(
(fname, utils.json_load(fname)) for fname in history_files)
vis.plot_hist(hists)
test_set = self._load_test()
model = self._load_model()
for image, dist_image, prediction in model.predict_visualization(
test_set):
fig, axes = vis.plot_image(image, image, dist_image, colorbar=True)
axes[0, 1].plot(prediction[:, 1], prediction[:, 0], 'rx')
axes[0, 2].plot(prediction[:, 1], prediction[:, 0], 'rx')
logger.info(f"prediction:\n{prediction}")
vis.show(join(self.figs_dir, 'prediction.pdf'))
if not self.show:
break
def show(fname=None, save=False):
"""Show the figure currently in matplotlib or save it, if not self.show.
If no fname provided, and self.show is False, then closes the figure. If save
is True, figure is saved regardless of show.
:param fname: name of the file to save to.
:param save: whether to save the file.
"""
if _show and not save:
logger.info("showing figure...")
plt.show()
elif fname is None:
logger.warning("Cannot save figure. Did you forget to set --show?")
plt.close()
else:
plt.savefig(fname)
logger.info(f"saved figure to {fname}.")
def train(self, art_data, initial_epoch=0, epochs=1, seconds=0, **kwargs):
"""Fits the model, saving it along the way, and reloads every epoch.
:param art_data: ArtificeData set
:param initial_epoch: epoch that training is starting from
:param epochs: epoch number to stop at. If -1, training continues forever.
:param seconds: seconds after which to stop reloading every epoch. If -1,
reload is never stopped. If 0, dataset is loaded only once, at beginning.
:returns: history dictionary
"""
if (initial_epoch > 0 and os.path.exists(self.history_path)
and not self.overwrite):
hist = utils.json_load(self.history_path)
else:
hist = {}
start_time = time()
epoch = initial_epoch
while epoch != epochs and time() - start_time > seconds > 0:
logger.info("reloading dataset (not cached)...")
hist = self.fit(art_data,
hist=hist,
initial_epoch=epoch,
epochs=(epoch + 1),
**kwargs)
epoch += 1
if epoch != epochs:
hist = self.fit(art_data,
hist=hist,
initial_epoch=epoch,
epochs=epochs,
**kwargs)
self.save()
return hist
def predict(self, art_data, multiscale=False):
"""Run prediction, reassembling tiles, with the Artifice data."""
if tf.executing_eagerly():
outputs = []
for i, batch in enumerate(art_data.prediction_input()):
if i % 100 == 0:
logger.info(f"batch {i} / {art_data.steps_per_epoch}")
outputs += _unbatch_outputs(
self.model.predict_on_batch(batch))
while len(outputs) >= art_data.num_tiles:
prediction = art_data.analyze_outputs(
outputs, multiscale=multiscale)
yield prediction
del outputs[:art_data.num_tiles]
else:
raise NotImplementedError(
"enable eager execution for eval (remove --patient)")
outputs = []
next_batch = (
art_data.prediction_input().make_one_shot_iterator().get_next())
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
for i in itertools.count():
try:
batch = sess.run(next_batch)
except tf.errors.OutOfRangeError:
return
if i % 100 == 0:
logger.info(f"batch {i} / {art_data.steps_per_epoch}")
outputs += _unbatch_outputs(
self.model.predict_on_batch(batch))
while len(outputs) >= art_data.num_tiles:
prediction = art_data.analyze_outputs(
outputs, multiscale=multiscale)
yield prediction
del outputs[:art_data.num_tiles]
def run(self, seconds=-1):
"""Run for at most `seconds`. If `seconds` is negative, run forever."""
start_time = time()
if tf.executing_eagerly():
dataset = self.data_set.enumerated_prediction_input().repeat(-1)
for indices, images in dataset:
logger.info(f"evaluating priorities for {indices}...")
priorities = list(self.prioritize(images))
self.info.push(list(zip(list(indices), priorities)))
logger.info(f"pushed {indices} with priorities {priorities}.")
if time() - start_time > seconds > 0:
logger.info(f"finished after {seconds}s.")
break
else:
raise NotImplementedError("patient execution")
def main():
_system_checks()
parser = argparse.ArgumentParser(description=docs.description)
parser.add_argument('commands', nargs='+', help=docs.commands)
# file settings
parser.add_argument('--data-root',
'--input',
'-i',
nargs=1,
default=['data/default'],
help=docs.data_root)
parser.add_argument('--model-root',
'--model-dir',
'-m',
nargs=1,
default=['models/tmp'],
help=docs.model_root)
parser.add_argument('--overwrite',
'-f',
action='store_true',
help=docs.overwrite)
parser.add_argument('--deep', action='store_true', help=docs.deep)
parser.add_argument('--figs-dir',
'--figures',
nargs=1,
default=['figs'],
help=docs.figs_dir)
# data settings
parser.add_argument('--convert-mode',
nargs='+',
default=[0, 4],
type=int,
help=docs.convert_mode)
parser.add_argument('--transformation',
'--augment',
'-a',
nargs='?',
default=None,
const=0,
type=int,
help=docs.transformation)
parser.add_argument('--identity-prob',
nargs=1,
default=[0.01],
type=float,
help=docs.identity_prob)
parser.add_argument('--priority-mode',
'--priority',
nargs=1,
default=['random'],
help=docs.priority_mode)
parser.add_argument('--labeled', action='store_true', help=docs.labeled)
# annotation settings
parser.add_argument('--annotation-mode',
'--annotate',
nargs=1,
default=['disks'],
help=docs.annotation_mode)
parser.add_argument('--record-size',
nargs=1,
default=[10],
type=int,
help=docs.record_size)
parser.add_argument('--annotation-delay',
nargs=1,
default=[60],
type=float,
help=docs.annotation_delay)
# sizes relating to data
parser.add_argument('--image-shape',
'--shape',
'-s',
nargs=3,
type=int,
default=[500, 500, 1],
help=docs.image_shape)
parser.add_argument('--data-size',
'-N',
nargs=1,
default=[10000],
type=int,
help=docs.data_size)
parser.add_argument('--test-size',
'-T',
nargs=1,
default=[1000],
type=int,
help=docs.test_size)
parser.add_argument('--batch-size',
'-b',
nargs=1,
default=[16],
type=int,
help=docs.batch_size)
parser.add_argument('--num-objects',
'-n',
nargs=1,
default=[40],
type=int,
help=docs.num_objects)
parser.add_argument('--pose-dim',
'-p',
nargs=1,
default=[2],
type=int,
help=docs.pose_dim)
parser.add_argument('--num-shuffle',
nargs=1,
default=[1000],
type=int,
help=docs.num_shuffle)
# model architecture
parser.add_argument('--base-shape',
nargs='+',
default=[28],
type=int,
help=docs.base_shape)
parser.add_argument('--level-filters',
nargs='+',
default=[128, 64, 32],
type=int,
help=docs.level_filters)
parser.add_argument('--level-depth',
nargs='+',
default=[2],
type=int,
help=docs.level_depth)
# sparse eval and other optimization settings
parser.add_argument('--model',
'-M',
nargs='?',
default='unet',
help=docs.model)
parser.add_argument('--multiscale',
action='store_true',
help=docs.multiscale)
parser.add_argument('--use-var', action='store_true', help=docs.use_var)
# model hyperparameters
parser.add_argument('--dropout',
nargs=1,
default=[0.5],
type=float,
help=docs.dropout)
parser.add_argument('--initial-epoch',
nargs=1,
default=[0],
type=int,
help=docs.initial_epoch) # todo: get from ckpt
parser.add_argument('--epochs',
'-e',
nargs=1,
default=[1],
type=int,
help=docs.epochs)
parser.add_argument('--learning-rate',
'-l',
nargs=1,
default=[0.1],
type=float,
help=docs.learning_rate)
parser.add_argument('--tol',
nargs=1,
default=[0.1],
type=float,
help=docs.tol)
# runtime settings
parser.add_argument('--num-parallel-calls',
'--cores',
nargs=1,
default=[-1],
type=int,
help=docs.num_parallel_calls)
parser.add_argument('--verbose',
'-v',
nargs='?',
const=1,
default=2,
type=int,
help=docs.verbose)
parser.add_argument('--keras-verbose',
nargs='?',
const=2,
default=1,
type=int,
help=docs.keras_verbose)
parser.add_argument('--patient', action='store_true', help=docs.patient)
parser.add_argument('--show', action='store_true', help=docs.show)
parser.add_argument('--cache', action='store_true', help=docs.cache)
parser.add_argument('--seconds',
'--time',
'--reload',
'-t',
'-r',
nargs='?',
default=0,
const=-1,
type=int,
help=docs.seconds)
args = parser.parse_args()
art = Artifice(commands=args.commands,
convert_mode=args.convert_mode,
transformation=args.transformation,
identity_prob=args.identity_prob[0],
priority_mode=args.priority_mode[0],
labeled=args.labeled,
annotation_mode=args.annotation_mode[0],
record_size=args.record_size[0],
annotation_delay=args.annotation_delay[0],
data_root=args.data_root[0],
model_root=args.model_root[0],
overwrite=args.overwrite,
deep=args.deep,
figs_dir=args.figs_dir[0],
image_shape=args.image_shape,
data_size=args.data_size[0],
test_size=args.test_size[0],
batch_size=args.batch_size[0],
num_objects=args.num_objects[0],
pose_dim=args.pose_dim[0],
num_shuffle=args.num_shuffle[0],
base_shape=args.base_shape,
level_filters=args.level_filters,
level_depth=args.level_depth[0],
model=args.model,
multiscale=args.multiscale,
use_var=args.use_var,
dropout=args.dropout[0],
initial_epoch=args.initial_epoch[0],
epochs=args.epochs[0],
learning_rate=args.learning_rate[0],
tol=args.tol[0],
num_parallel_calls=args.num_parallel_calls[0],
verbose=args.verbose,
keras_verbose=args.keras_verbose,
eager=(not args.patient),
show=args.show,
cache=args.cache,
seconds=args.seconds)
logger.info(art)
art()
def _ensure_dirs_exist(dirs):
for path in dirs:
if not exists(path):
logger.info(f"creating '{path}'")
os.makedirs(path)
def evaluate(self):
test_set = self._load_test()
model = self._load_model()
errors, num_failed = model.evaluate(test_set)
if not errors:
logger.warning(f"found ZERO objects, num_failed: {num_failed}")
return
avg_error = errors.mean(axis=0)
total_num_objects = self.test_size * self.num_objects
num_detected = total_num_objects - num_failed
logger.info(f"objects detected: {num_detected} / "
f"{total_num_objects}")
logger.info(f"avg (euclidean) detection error: {avg_error[0]}")
logger.info(f"avg (absolute) pose error: {avg_error[1:]}")
logger.info(
"note: some objects may be occluded, making detection impossible")
logger.info(f"avg: {errors.mean(axis=0)}")
logger.info(f"std: {errors.std(axis=0)}")
logger.info(f"min: {errors.min(axis=0)}")
logger.info(f"max: {errors.max(axis=0)}")
