def eval_main(dataset_dirs: List[pathlib.Path],
mode: str,
batch_size: int,
use_gt_rope: bool,
threshold: Optional[float] = None,
old_compat: bool = False,
take: Optional[int] = None,
checkpoint: Optional[pathlib.Path] = None,
trials_directory=pathlib.Path,
**kwargs):
###############
# Model
###############
trial_path = checkpoint.parent.absolute()
_, params = filepath_tools.create_or_load_trial(
trial_path=trial_path, trials_directory=trials_directory)
model_class = link_bot_classifiers.get_model(params['model_class'])
###############
# Dataset
###############
dataset = ClassifierDatasetLoader(dataset_dirs,
load_true_states=True,
use_gt_rope=use_gt_rope,
old_compat=old_compat,
threshold=threshold)
tf_dataset = dataset.get_datasets(mode=mode, take=take)
tf_dataset = balance(tf_dataset)
###############
# Evaluate
###############
tf_dataset = batch_tf_dataset(tf_dataset, batch_size, drop_remainder=True)
model = model_class(hparams=params,
batch_size=batch_size,
scenario=dataset.scenario)
# This call to model runner restores the model
runner = ModelRunner(model=model,
training=False,
params=params,
checkpoint=checkpoint,
trial_path=trial_path,
key_metric=AccuracyMetric,
batch_metadata=dataset.batch_metadata)
metrics = runner.val_epoch(tf_dataset)
for metric_name, metric_value in metrics.items():
print(f"{metric_name:30s}: {metric_value}")
return metrics
def viz_ensemble_main(dataset_dir: pathlib.Path,
checkpoints: List[pathlib.Path], mode: str,
batch_size: int, only_errors: bool, use_gt_rope: bool,
**kwargs):
dynamics_stdev_pub_ = rospy.Publisher("dynamics_stdev",
Float32,
queue_size=10)
classifier_stdev_pub_ = rospy.Publisher("classifier_stdev",
Float32,
queue_size=10)
accept_probability_pub_ = rospy.Publisher("accept_probability_viz",
Float32,
queue_size=10)
traj_idx_pub_ = rospy.Publisher("traj_idx_viz", Float32, queue_size=10)
###############
# Model
###############
model = load_generic_model(checkpoints)
###############
# Dataset
###############
test_dataset = ClassifierDatasetLoader([dataset_dir],
load_true_states=True,
use_gt_rope=use_gt_rope)
test_tf_dataset = test_dataset.get_datasets(mode=mode)
test_tf_dataset = batch_tf_dataset(test_tf_dataset,
batch_size,
drop_remainder=True)
scenario = test_dataset.scenario
###############
# Evaluate
###############
# Iterate over test set
all_accuracies_over_time = []
all_stdevs = []
all_labels = []
classifier_ensemble_stdevs = []
for batch_idx, test_batch in enumerate(test_tf_dataset):
test_batch.update(test_dataset.batch_metadata)
mean_predictions, stdev_predictions = model.check_constraint_from_example(
test_batch)
mean_probabilities = mean_predictions['probabilities']
stdev_probabilities = stdev_predictions['probabilities']
labels = tf.expand_dims(test_batch['is_close'][:, 1:], axis=2)
all_labels = tf.concat(
(all_labels, tf.reshape(test_batch['is_close'][:, 1:], [-1])),
axis=0)
all_stdevs = tf.concat(
(all_stdevs, tf.reshape(test_batch[add_predicted('stdev')], [-1])),
axis=0)
accuracy_over_time = tf.keras.metrics.binary_accuracy(
y_true=labels, y_pred=mean_probabilities)
all_accuracies_over_time.append(accuracy_over_time)
# Visualization
test_batch.pop("time")
test_batch.pop("batch_size")
decisions = mean_probabilities > 0.5
classifier_is_correct = tf.squeeze(tf.equal(decisions,
tf.cast(labels, tf.bool)),
axis=-1)
for b in range(batch_size):
example = index_dict_of_batched_tensors_tf(test_batch, b)
classifier_ensemble_stdev = stdev_probabilities[b].numpy().squeeze(
)
classifier_ensemble_stdevs.append(classifier_ensemble_stdev)
# if the classifier is correct at all time steps, ignore
if only_errors and tf.reduce_all(classifier_is_correct[b]):
continue
# if only_collision
predicted_rope_states = tf.reshape(
example[add_predicted('link_bot')][1], [-1, 3])
xs = predicted_rope_states[:, 0]
ys = predicted_rope_states[:, 1]
zs = predicted_rope_states[:, 2]
in_collision = bool(
batch_in_collision_tf_3d(environment=example,
xs=xs,
ys=ys,
zs=zs,
inflate_radius_m=0)[0].numpy())
label = bool(example['is_close'][1].numpy())
accept = decisions[b, 0, 0].numpy()
# if not (in_collision and accept):
# continue
scenario.plot_environment_rviz(example)
stdev_probabilities[b].numpy().squeeze()
classifier_stdev_msg = Float32()
classifier_stdev_msg.data = stdev_probabilities[b].numpy().squeeze(
)
classifier_stdev_pub_.publish(classifier_stdev_msg)
actual_0 = scenario.index_state_time(example, 0)
actual_1 = scenario.index_state_time(example, 1)
pred_0 = scenario.index_predicted_state_time(example, 0)
pred_1 = scenario.index_predicted_state_time(example, 1)
action = scenario.index_action_time(example, 0)
label = example['is_close'][1]
scenario.plot_state_rviz(actual_0,
label='actual',
color='#FF0000AA',
idx=0)
scenario.plot_state_rviz(actual_1,
label='actual',
color='#E00016AA',
idx=1)
scenario.plot_state_rviz(pred_0,
label='predicted',
color='#0000FFAA',
idx=0)
scenario.plot_state_rviz(pred_1,
label='predicted',
color='#0553FAAA',
idx=1)
scenario.plot_action_rviz(pred_0, action)
scenario.plot_is_close(label)
dynamics_stdev_t = example[add_predicted('stdev')][1, 0].numpy()
dynamics_stdev_msg = Float32()
dynamics_stdev_msg.data = dynamics_stdev_t
dynamics_stdev_pub_.publish(dynamics_stdev_msg)
accept_probability_t = mean_probabilities[b, 0, 0].numpy()
accept_probability_msg = Float32()
accept_probability_msg.data = accept_probability_t
accept_probability_pub_.publish(accept_probability_msg)
traj_idx_msg = Float32()
traj_idx_msg.data = batch_idx * batch_size + b
traj_idx_pub_.publish(traj_idx_msg)
# stepper = RvizSimpleStepper()
# stepper.step()
print(np.mean(classifier_ensemble_stdevs))
all_accuracies_over_time = tf.concat(all_accuracies_over_time, axis=0)
mean_accuracies_over_time = tf.reduce_mean(all_accuracies_over_time,
axis=0)
std_accuracies_over_time = tf.math.reduce_std(all_accuracies_over_time,
axis=0)
mean_classifier_ensemble_stdev = tf.reduce_mean(classifier_ensemble_stdevs)
print(mean_classifier_ensemble_stdev)
def viz_main(dataset_dirs: List[pathlib.Path],
checkpoint: pathlib.Path,
mode: str,
batch_size: int,
only_errors: bool,
use_gt_rope: bool,
old_compat: bool = False,
**kwargs):
stdev_pub_ = rospy.Publisher("stdev", Float32, queue_size=10)
traj_idx_pub_ = rospy.Publisher("traj_idx_viz", Float32, queue_size=10)
###############
# Model
###############
trials_directory = pathlib.Path('trials').absolute()
trial_path = checkpoint.parent.absolute()
_, params = filepath_tools.create_or_load_trial(
trial_path=trial_path, trials_directory=trials_directory)
model_class = link_bot_classifiers.get_model(params['model_class'])
###############
# Dataset
###############
dataset = ClassifierDatasetLoader(
dataset_dirs,
load_true_states=True,
use_gt_rope=use_gt_rope,
threshold=params['classifier_dataset_hparams']['labeling_params']
['threshold'],
old_compat=old_compat)
model = model_class(hparams=params,
batch_size=batch_size,
scenario=dataset.scenario)
tf_dataset = dataset.get_datasets(mode=mode)
scenario = dataset.scenario
###############
# Evaluate
###############
tf_dataset = batch_tf_dataset(tf_dataset, batch_size, drop_remainder=True)
model = classifier_utils.load_generic_model([checkpoint])
for batch_idx, example in enumerate(
progressbar(tf_dataset, widgets=base_dataset.widgets)):
example.update(dataset.batch_metadata)
predictions, _ = model.check_constraint_from_example(example,
training=False)
labels = tf.expand_dims(example['is_close'][:, 1:], axis=2)
probabilities = predictions['probabilities']
# Visualization
example.pop("time")
example.pop("batch_size")
decisions = probabilities > 0.5
classifier_is_correct = tf.squeeze(tf.equal(decisions,
tf.cast(labels, tf.bool)),
axis=-1)
for b in range(batch_size):
example_b = index_dict_of_batched_tensors_tf(example, b)
# if the classifier is correct at all time steps, ignore
if only_errors and tf.reduce_all(classifier_is_correct[b]):
continue
def _custom_viz_t(scenario: Base3DScenario, e: Dict, t: int):
if t > 0:
accept_probability_t = predictions['probabilities'][
b, t - 1, 0].numpy()
else:
accept_probability_t = -999
scenario.plot_accept_probability(accept_probability_t)
traj_idx_msg = Float32()
traj_idx_msg.data = batch_idx * batch_size + b
traj_idx_pub_.publish(traj_idx_msg)
anim = RvizAnimation(scenario=scenario,
n_time_steps=dataset.horizon,
init_funcs=[
init_viz_env,
dataset.init_viz_action(),
],
t_funcs=[
_custom_viz_t,
dataset.classifier_transition_viz_t(),
ExperimentScenario.plot_stdev_t,
])
with open("debugging.hjson", 'w') as f:
my_hdump(numpify(example_b), f)
anim.play(example_b)
def main():
parser = argparse.ArgumentParser(formatter_class=my_formatter)
parser.add_argument('dataset_dir',
type=pathlib.Path,
help='dataset directory',
nargs='+')
parser.add_argument('--mode',
choices=['train', 'test', 'val'],
default='train')
parser.add_argument('--n-repetitions', type=int, default=1)
parser.add_argument('--batch-size', type=int, default=64)
args = parser.parse_args()
dataset = ClassifierDatasetLoader(args.dataset_dir)
dataset.cache_negative = False
t0 = perf_counter()
tf_dataset = dataset.get_datasets(mode=args.mode)
scenario = get_scenario('link_bot')
tf_dataset = tf_dataset.batch(args.batch_size)
time_to_load = perf_counter() - t0
print("Time to Load (s): {:5.3f}".format(time_to_load))
n = 8192
batches = int(n / args.batch_size)
try:
# ram_usage = []
for _ in range(args.n_repetitions):
# None
t0 = perf_counter()
for e in progressbar.progressbar(tf_dataset.take(batches)):
pass
print('{:.5f}'.format(perf_counter() - t0))
# NEW
t0 = perf_counter()
for e in progressbar.progressbar(tf_dataset.take(batches)):
e = add_traj_image(scenario=scenario,
example=e,
local_env_w=100,
states_keys=['link_bot'],
local_env_h=100,
rope_image_k=10000,
batch_size=args.batch_size)
print('{:.5f}'.format(perf_counter() - t0))
# OLD
t0 = perf_counter()
for e in progressbar.progressbar(tf_dataset.take(batches)):
e = add_traj_image_to_example_old(scenario=scenario,
example=e,
local_env_w=100,
states_keys=['link_bot'],
local_env_h=100,
rope_image_k=10000,
batch_size=args.batch_size)
print('{:.5f}'.format(perf_counter() - t0))
# plt.plot(ram_usage)
# plt.xlabel("iteration")
# plt.ylabel("ram usage (bytes)")
# plt.show()
except KeyboardInterrupt:
pass