def eval_main(dataset_dirs: List[pathlib.Path],
mode: str,
batch_size: int,
use_gt_rope: bool,
threshold: Optional[float] = None,
old_compat: bool = False,
take: Optional[int] = None,
checkpoint: Optional[pathlib.Path] = None,
trials_directory=pathlib.Path,
**kwargs):
###############
# Model
###############
trial_path = checkpoint.parent.absolute()
_, params = filepath_tools.create_or_load_trial(
trial_path=trial_path, trials_directory=trials_directory)
model_class = link_bot_classifiers.get_model(params['model_class'])
###############
# Dataset
###############
dataset = ClassifierDatasetLoader(dataset_dirs,
load_true_states=True,
use_gt_rope=use_gt_rope,
old_compat=old_compat,
threshold=threshold)
tf_dataset = dataset.get_datasets(mode=mode, take=take)
tf_dataset = balance(tf_dataset)
###############
# Evaluate
###############
tf_dataset = batch_tf_dataset(tf_dataset, batch_size, drop_remainder=True)
model = model_class(hparams=params,
batch_size=batch_size,
scenario=dataset.scenario)
# This call to model runner restores the model
runner = ModelRunner(model=model,
training=False,
params=params,
checkpoint=checkpoint,
trial_path=trial_path,
key_metric=AccuracyMetric,
batch_metadata=dataset.batch_metadata)
metrics = runner.val_epoch(tf_dataset)
for metric_name, metric_value in metrics.items():
print(f"{metric_name:30s}: {metric_value}")
return metrics
def main():
colorama.init(autoreset=True)
parser = argparse.ArgumentParser(
description="adds file path to the example",
formatter_class=my_formatter)
parser.add_argument('dataset_dir',
type=pathlib.Path,
help='dataset directory')
parser.add_argument('dataset_type',
choices=['dy', 'cl', 'rcv'],
help='dataset type')
args = parser.parse_args()
rospy.init_node("add_paths")
outdir = args.dataset_dir.parent / f"{args.dataset_dir.name}+paths"
if args.dataset_type == 'dy':
dataset = DynamicsDatasetLoader([args.dataset_dir])
elif args.dataset_type == 'cl':
dataset = ClassifierDatasetLoader([args.dataset_dir],
load_true_states=True,
use_gt_rope=False)
elif args.dataset_type == 'rcv':
dataset = RecoveryDatasetLoader([args.dataset_dir])
else:
raise NotImplementedError(f"Invalid dataset type {args.dataset_type}")
# hparams
hparams_update = {'has_tfrecord_path': True}
modify_hparams(args.dataset_dir, outdir, hparams_update)
total_count = 0
for mode in ['train', 'test', 'val']:
tf_dataset = dataset.get_datasets(mode=mode,
shuffle_files=False,
do_not_process=True)
full_output_directory = outdir / mode
full_output_directory.mkdir(parents=True, exist_ok=True)
for example, tfrecord_path in zip(
progressbar(tf_dataset, widgets=base_dataset.widgets),
tf_dataset.records):
features = {
k: float_tensor_to_bytes_feature(v)
for k, v in example.items()
}
features['tfrecord_path'] = bytes_feature(
tf.io.serialize_tensor(
tf.convert_to_tensor(tfrecord_path,
dtype=tf.string)).numpy())
tf_write_features(total_count, features, full_output_directory)
total_count += 1
print(Fore.GREEN + f"Modified {total_count} examples")
def test_mem_usage(self):
no_cache_dataset = ClassifierDatasetLoader(
[pathlib.Path('classifier_data/rope_relaxed_mer')])
no_cache_dataset.cache_negative = False
max_ram_no_cache = iterate_and_record_max_ram_usage(no_cache_dataset)
del no_cache_dataset
cache_dataset = ClassifierDatasetLoader(
[pathlib.Path('classifier_data/rope_relaxed_mer')])
cache_dataset.cache_negative = True
max_ram_yes_cache = iterate_and_record_max_ram_usage(cache_dataset)
self.assertGreater(max_ram_yes_cache, max_ram_no_cache * 1.3)
def main():
colorama.init(autoreset=True)
parser = argparse.ArgumentParser(formatter_class=my_formatter)
parser.add_argument('dataset_dir',
type=pathlib.Path,
help='dataset directory')
parser.add_argument('suffix',
type=str,
help='string added to the new dataset name')
args = parser.parse_args()
rospy.init_node("modify_dynamics_dataset")
outdir = args.dataset_dir.parent / f"{args.dataset_dir.name}+{args.suffix}"
def _process_example(dataset: ClassifierDatasetLoader, example: Dict):
example['left_gripper'] = example.pop('gripper1')
example['right_gripper'] = example.pop('gripper2')
example['left_gripper_position'] = example.pop('gripper1_position')
example['right_gripper_position'] = example.pop('gripper2_position')
example['rope'] = example.pop('link_bot')
example[add_predicted('left_gripper')] = example.pop(
add_predicted('gripper1'))
example[add_predicted('right_gripper')] = example.pop(
add_predicted('gripper2'))
example[add_predicted('rope')] = example.pop(add_predicted('link_bot'))
yield example
hparams_update = {}
dataset = ClassifierDatasetLoader([args.dataset_dir],
use_gt_rope=False,
load_true_states=True)
modify_dataset(dataset_dir=args.dataset_dir,
dataset=dataset,
outdir=outdir,
process_example=_process_example,
hparams_update=hparams_update)
def main():
colorama.init(autoreset=True)
plt.style.use("slides")
np.set_printoptions(suppress=True, linewidth=200, precision=3)
parser = argparse.ArgumentParser()
parser.add_argument('dataset_dirs', type=pathlib.Path, nargs='+')
parser.add_argument('--display-type', choices=['just_count', '3d', 'stdev'], default='3d')
parser.add_argument('--mode', choices=['train', 'val', 'test', 'all'], default='train')
parser.add_argument('--shuffle', action='store_true')
parser.add_argument('--save', action='store_true')
parser.add_argument('--threshold', type=float, default=None)
parser.add_argument('--seed', type=int, default=1)
parser.add_argument('--take', type=int)
parser.add_argument('--use-gt-rope', action='store_true')
parser.add_argument('--old-compat', action='store_true')
parser.add_argument('--only-negative', action='store_true')
parser.add_argument('--only-positive', action='store_true')
parser.add_argument('--only-in-collision', action='store_true')
parser.add_argument('--only-reconverging', action='store_true')
parser.add_argument('--perf', action='store_true', help='print time per iteration')
parser.add_argument('--no-plot', action='store_true', help='only print statistics')
args = parser.parse_args()
args.batch_size = 1
np.random.seed(args.seed)
tf.random.set_seed(args.seed)
rospy.init_node("visualize_classifier_data")
classifier_dataset = ClassifierDatasetLoader(args.dataset_dirs,
load_true_states=True,
threshold=args.threshold,
use_gt_rope=args.use_gt_rope,
old_compat=args.old_compat)
visualize_dataset(args, classifier_dataset)
def train_main(dataset_dirs: List[pathlib.Path],
model_hparams: pathlib.Path,
log: str,
batch_size: int,
epochs: int,
seed: int,
use_gt_rope: bool,
checkpoint: Optional[pathlib.Path] = None,
threshold: Optional[float] = None,
ensemble_idx: Optional[int] = None,
old_compat: bool = False,
take: Optional[int] = None,
validate: bool = True,
trials_directory: Optional[pathlib.Path] = None,
**kwargs):
model_hparams = hjson.load(model_hparams.open('r'))
model_class = link_bot_classifiers.get_model(model_hparams['model_class'])
# set load_true_states=True when debugging
train_dataset = ClassifierDatasetLoader(
dataset_dirs=dataset_dirs,
load_true_states=True,
use_gt_rope=use_gt_rope,
threshold=threshold,
old_compat=old_compat,
)
val_dataset = ClassifierDatasetLoader(
dataset_dirs=dataset_dirs,
load_true_states=True,
use_gt_rope=use_gt_rope,
threshold=threshold,
old_compat=old_compat,
)
model_hparams.update(
setup_hparams(batch_size, dataset_dirs, seed, train_dataset,
use_gt_rope))
model = model_class(hparams=model_hparams,
batch_size=batch_size,
scenario=train_dataset.scenario)
checkpoint_name, trial_path = setup_training_paths(checkpoint,
ensemble_idx, log,
model_hparams,
trials_directory)
if validate:
mid_epoch_val_batches = 100
val_every_n_batches = 100
save_every_n_minutes = 20
validate_first = True
else:
mid_epoch_val_batches = None
val_every_n_batches = None
save_every_n_minutes = None
validate_first = False
runner = ModelRunner(model=model,
training=True,
params=model_hparams,
trial_path=trial_path,
key_metric=AccuracyMetric,
checkpoint=checkpoint,
mid_epoch_val_batches=mid_epoch_val_batches,
val_every_n_batches=val_every_n_batches,
save_every_n_minutes=save_every_n_minutes,
validate_first=validate_first,
batch_metadata=train_dataset.batch_metadata)
train_tf_dataset, val_tf_dataset = setup_datasets(model_hparams,
batch_size, seed,
train_dataset,
val_dataset, take)
final_val_metrics = runner.train(train_tf_dataset,
val_tf_dataset,
num_epochs=epochs)
return trial_path, final_val_metrics
def viz_ensemble_main(dataset_dir: pathlib.Path,
checkpoints: List[pathlib.Path], mode: str,
batch_size: int, only_errors: bool, use_gt_rope: bool,
**kwargs):
dynamics_stdev_pub_ = rospy.Publisher("dynamics_stdev",
Float32,
queue_size=10)
classifier_stdev_pub_ = rospy.Publisher("classifier_stdev",
Float32,
queue_size=10)
accept_probability_pub_ = rospy.Publisher("accept_probability_viz",
Float32,
queue_size=10)
traj_idx_pub_ = rospy.Publisher("traj_idx_viz", Float32, queue_size=10)
###############
# Model
###############
model = load_generic_model(checkpoints)
###############
# Dataset
###############
test_dataset = ClassifierDatasetLoader([dataset_dir],
load_true_states=True,
use_gt_rope=use_gt_rope)
test_tf_dataset = test_dataset.get_datasets(mode=mode)
test_tf_dataset = batch_tf_dataset(test_tf_dataset,
batch_size,
drop_remainder=True)
scenario = test_dataset.scenario
###############
# Evaluate
###############
# Iterate over test set
all_accuracies_over_time = []
all_stdevs = []
all_labels = []
classifier_ensemble_stdevs = []
for batch_idx, test_batch in enumerate(test_tf_dataset):
test_batch.update(test_dataset.batch_metadata)
mean_predictions, stdev_predictions = model.check_constraint_from_example(
test_batch)
mean_probabilities = mean_predictions['probabilities']
stdev_probabilities = stdev_predictions['probabilities']
labels = tf.expand_dims(test_batch['is_close'][:, 1:], axis=2)
all_labels = tf.concat(
(all_labels, tf.reshape(test_batch['is_close'][:, 1:], [-1])),
axis=0)
all_stdevs = tf.concat(
(all_stdevs, tf.reshape(test_batch[add_predicted('stdev')], [-1])),
axis=0)
accuracy_over_time = tf.keras.metrics.binary_accuracy(
y_true=labels, y_pred=mean_probabilities)
all_accuracies_over_time.append(accuracy_over_time)
# Visualization
test_batch.pop("time")
test_batch.pop("batch_size")
decisions = mean_probabilities > 0.5
classifier_is_correct = tf.squeeze(tf.equal(decisions,
tf.cast(labels, tf.bool)),
axis=-1)
for b in range(batch_size):
example = index_dict_of_batched_tensors_tf(test_batch, b)
classifier_ensemble_stdev = stdev_probabilities[b].numpy().squeeze(
)
classifier_ensemble_stdevs.append(classifier_ensemble_stdev)
# if the classifier is correct at all time steps, ignore
if only_errors and tf.reduce_all(classifier_is_correct[b]):
continue
# if only_collision
predicted_rope_states = tf.reshape(
example[add_predicted('link_bot')][1], [-1, 3])
xs = predicted_rope_states[:, 0]
ys = predicted_rope_states[:, 1]
zs = predicted_rope_states[:, 2]
in_collision = bool(
batch_in_collision_tf_3d(environment=example,
xs=xs,
ys=ys,
zs=zs,
inflate_radius_m=0)[0].numpy())
label = bool(example['is_close'][1].numpy())
accept = decisions[b, 0, 0].numpy()
# if not (in_collision and accept):
# continue
scenario.plot_environment_rviz(example)
stdev_probabilities[b].numpy().squeeze()
classifier_stdev_msg = Float32()
classifier_stdev_msg.data = stdev_probabilities[b].numpy().squeeze(
)
classifier_stdev_pub_.publish(classifier_stdev_msg)
actual_0 = scenario.index_state_time(example, 0)
actual_1 = scenario.index_state_time(example, 1)
pred_0 = scenario.index_predicted_state_time(example, 0)
pred_1 = scenario.index_predicted_state_time(example, 1)
action = scenario.index_action_time(example, 0)
label = example['is_close'][1]
scenario.plot_state_rviz(actual_0,
label='actual',
color='#FF0000AA',
idx=0)
scenario.plot_state_rviz(actual_1,
label='actual',
color='#E00016AA',
idx=1)
scenario.plot_state_rviz(pred_0,
label='predicted',
color='#0000FFAA',
idx=0)
scenario.plot_state_rviz(pred_1,
label='predicted',
color='#0553FAAA',
idx=1)
scenario.plot_action_rviz(pred_0, action)
scenario.plot_is_close(label)
dynamics_stdev_t = example[add_predicted('stdev')][1, 0].numpy()
dynamics_stdev_msg = Float32()
dynamics_stdev_msg.data = dynamics_stdev_t
dynamics_stdev_pub_.publish(dynamics_stdev_msg)
accept_probability_t = mean_probabilities[b, 0, 0].numpy()
accept_probability_msg = Float32()
accept_probability_msg.data = accept_probability_t
accept_probability_pub_.publish(accept_probability_msg)
traj_idx_msg = Float32()
traj_idx_msg.data = batch_idx * batch_size + b
traj_idx_pub_.publish(traj_idx_msg)
# stepper = RvizSimpleStepper()
# stepper.step()
print(np.mean(classifier_ensemble_stdevs))
all_accuracies_over_time = tf.concat(all_accuracies_over_time, axis=0)
mean_accuracies_over_time = tf.reduce_mean(all_accuracies_over_time,
axis=0)
std_accuracies_over_time = tf.math.reduce_std(all_accuracies_over_time,
axis=0)
mean_classifier_ensemble_stdev = tf.reduce_mean(classifier_ensemble_stdevs)
print(mean_classifier_ensemble_stdev)
def viz_main(dataset_dirs: List[pathlib.Path],
checkpoint: pathlib.Path,
mode: str,
batch_size: int,
only_errors: bool,
use_gt_rope: bool,
old_compat: bool = False,
**kwargs):
stdev_pub_ = rospy.Publisher("stdev", Float32, queue_size=10)
traj_idx_pub_ = rospy.Publisher("traj_idx_viz", Float32, queue_size=10)
###############
# Model
###############
trials_directory = pathlib.Path('trials').absolute()
trial_path = checkpoint.parent.absolute()
_, params = filepath_tools.create_or_load_trial(
trial_path=trial_path, trials_directory=trials_directory)
model_class = link_bot_classifiers.get_model(params['model_class'])
###############
# Dataset
###############
dataset = ClassifierDatasetLoader(
dataset_dirs,
load_true_states=True,
use_gt_rope=use_gt_rope,
threshold=params['classifier_dataset_hparams']['labeling_params']
['threshold'],
old_compat=old_compat)
model = model_class(hparams=params,
batch_size=batch_size,
scenario=dataset.scenario)
tf_dataset = dataset.get_datasets(mode=mode)
scenario = dataset.scenario
###############
# Evaluate
###############
tf_dataset = batch_tf_dataset(tf_dataset, batch_size, drop_remainder=True)
model = classifier_utils.load_generic_model([checkpoint])
for batch_idx, example in enumerate(
progressbar(tf_dataset, widgets=base_dataset.widgets)):
example.update(dataset.batch_metadata)
predictions, _ = model.check_constraint_from_example(example,
training=False)
labels = tf.expand_dims(example['is_close'][:, 1:], axis=2)
probabilities = predictions['probabilities']
# Visualization
example.pop("time")
example.pop("batch_size")
decisions = probabilities > 0.5
classifier_is_correct = tf.squeeze(tf.equal(decisions,
tf.cast(labels, tf.bool)),
axis=-1)
for b in range(batch_size):
example_b = index_dict_of_batched_tensors_tf(example, b)
# if the classifier is correct at all time steps, ignore
if only_errors and tf.reduce_all(classifier_is_correct[b]):
continue
def _custom_viz_t(scenario: Base3DScenario, e: Dict, t: int):
if t > 0:
accept_probability_t = predictions['probabilities'][
b, t - 1, 0].numpy()
else:
accept_probability_t = -999
scenario.plot_accept_probability(accept_probability_t)
traj_idx_msg = Float32()
traj_idx_msg.data = batch_idx * batch_size + b
traj_idx_pub_.publish(traj_idx_msg)
anim = RvizAnimation(scenario=scenario,
n_time_steps=dataset.horizon,
init_funcs=[
init_viz_env,
dataset.init_viz_action(),
],
t_funcs=[
_custom_viz_t,
dataset.classifier_transition_viz_t(),
ExperimentScenario.plot_stdev_t,
])
with open("debugging.hjson", 'w') as f:
my_hdump(numpify(example_b), f)
anim.play(example_b)
def make_classifier_dataset_from_params_dict(dataset_dir: pathlib.Path,
fwd_model_dir: List[pathlib.Path],
labeling_params: Dict,
outdir: pathlib.Path,
use_gt_rope: bool,
visualize: bool,
take: Optional[int] = None,
batch_size: Optional[int] = None,
start_at: Optional[int] = None,
stop_at: Optional[int] = None):
# append "best_checkpoint" before loading
if not isinstance(fwd_model_dir, List):
fwd_model_dir = [fwd_model_dir]
fwd_model_dir = [p / 'best_checkpoint' for p in fwd_model_dir]
dynamics_hparams = hjson.load((dataset_dir / 'hparams.hjson').open('r'))
fwd_models, _ = dynamics_utils.load_generic_model(fwd_model_dir)
dataset = DynamicsDatasetLoader([dataset_dir], use_gt_rope=use_gt_rope)
new_hparams_filename = outdir / 'hparams.hjson'
classifier_dataset_hparams = dynamics_hparams
classifier_dataset_hparams['dataset_dir'] = dataset_dir.as_posix()
classifier_dataset_hparams['fwd_model_hparams'] = fwd_models.hparams
classifier_dataset_hparams['labeling_params'] = labeling_params
classifier_dataset_hparams['true_state_keys'] = dataset.state_keys
classifier_dataset_hparams['predicted_state_keys'] = fwd_models.state_keys
classifier_dataset_hparams['action_keys'] = dataset.action_keys
classifier_dataset_hparams['scenario_metadata'] = dataset.hparams[
'scenario_metadata']
classifier_dataset_hparams['start-at'] = start_at
classifier_dataset_hparams['stop-at'] = stop_at
my_hdump(classifier_dataset_hparams,
new_hparams_filename.open("w"),
indent=2)
# because we're currently making this dataset, we can't call "get_dataset" but we can still use it to visualize
classifier_dataset_for_viz = ClassifierDatasetLoader(
[outdir], use_gt_rope=use_gt_rope)
t0 = perf_counter()
total_example_idx = 0
for mode in ['train', 'val', 'test']:
tf_dataset = dataset.get_datasets(mode=mode, take=take)
full_output_directory = outdir / mode
full_output_directory.mkdir(parents=True, exist_ok=True)
out_examples_gen = generate_classifier_examples(
fwd_models, tf_dataset, dataset, labeling_params, batch_size)
for out_examples in out_examples_gen:
for out_examples_for_start_t in out_examples:
actual_batch_size = out_examples_for_start_t['traj_idx'].shape[
0]
for batch_idx in range(actual_batch_size):
out_example_b = index_dict_of_batched_tensors_tf(
out_examples_for_start_t, batch_idx)
if out_example_b['time_idx'].ndim == 0:
continue
if visualize:
add_label(out_example_b, labeling_params['threshold'])
classifier_dataset_for_viz.anim_transition_rviz(
out_example_b)
tf_write_example(full_output_directory, out_example_b,
total_example_idx)
rospy.loginfo_throttle(
10,
f"Examples: {total_example_idx:10d}, Time: {perf_counter() - t0:.3f}"
)
total_example_idx += 1
return outdir
def main():
parser = argparse.ArgumentParser(formatter_class=my_formatter)
parser.add_argument('dataset_dir',
type=pathlib.Path,
help='dataset directory',
nargs='+')
parser.add_argument('--mode',
choices=['train', 'test', 'val'],
default='train')
parser.add_argument('--n-repetitions', type=int, default=1)
parser.add_argument('--batch-size', type=int, default=64)
args = parser.parse_args()
dataset = ClassifierDatasetLoader(args.dataset_dir)
dataset.cache_negative = False
t0 = perf_counter()
tf_dataset = dataset.get_datasets(mode=args.mode)
scenario = get_scenario('link_bot')
tf_dataset = tf_dataset.batch(args.batch_size)
time_to_load = perf_counter() - t0
print("Time to Load (s): {:5.3f}".format(time_to_load))
n = 8192
batches = int(n / args.batch_size)
try:
# ram_usage = []
for _ in range(args.n_repetitions):
# None
t0 = perf_counter()
for e in progressbar.progressbar(tf_dataset.take(batches)):
pass
print('{:.5f}'.format(perf_counter() - t0))
# NEW
t0 = perf_counter()
for e in progressbar.progressbar(tf_dataset.take(batches)):
e = add_traj_image(scenario=scenario,
example=e,
local_env_w=100,
states_keys=['link_bot'],
local_env_h=100,
rope_image_k=10000,
batch_size=args.batch_size)
print('{:.5f}'.format(perf_counter() - t0))
# OLD
t0 = perf_counter()
for e in progressbar.progressbar(tf_dataset.take(batches)):
e = add_traj_image_to_example_old(scenario=scenario,
example=e,
local_env_w=100,
states_keys=['link_bot'],
local_env_h=100,
rope_image_k=10000,
batch_size=args.batch_size)
print('{:.5f}'.format(perf_counter() - t0))
# plt.plot(ram_usage)
# plt.xlabel("iteration")
# plt.ylabel("ram usage (bytes)")
# plt.show()
except KeyboardInterrupt:
pass