def test_mem_usage(self):
no_cache_dataset = ClassifierDatasetLoader(
[pathlib.Path('classifier_data/rope_relaxed_mer')])
no_cache_dataset.cache_negative = False
max_ram_no_cache = iterate_and_record_max_ram_usage(no_cache_dataset)
del no_cache_dataset
cache_dataset = ClassifierDatasetLoader(
[pathlib.Path('classifier_data/rope_relaxed_mer')])
cache_dataset.cache_negative = True
max_ram_yes_cache = iterate_and_record_max_ram_usage(cache_dataset)
self.assertGreater(max_ram_yes_cache, max_ram_no_cache * 1.3)
def main():
parser = argparse.ArgumentParser(formatter_class=my_formatter)
parser.add_argument('dataset_dir',
type=pathlib.Path,
help='dataset directory',
nargs='+')
parser.add_argument('--mode',
choices=['train', 'test', 'val'],
default='train')
parser.add_argument('--n-repetitions', type=int, default=1)
parser.add_argument('--batch-size', type=int, default=64)
args = parser.parse_args()
dataset = ClassifierDatasetLoader(args.dataset_dir)
dataset.cache_negative = False
t0 = perf_counter()
tf_dataset = dataset.get_datasets(mode=args.mode)
scenario = get_scenario('link_bot')
tf_dataset = tf_dataset.batch(args.batch_size)
time_to_load = perf_counter() - t0
print("Time to Load (s): {:5.3f}".format(time_to_load))
n = 8192
batches = int(n / args.batch_size)
try:
# ram_usage = []
for _ in range(args.n_repetitions):
# None
t0 = perf_counter()
for e in progressbar.progressbar(tf_dataset.take(batches)):
pass
print('{:.5f}'.format(perf_counter() - t0))
# NEW
t0 = perf_counter()
for e in progressbar.progressbar(tf_dataset.take(batches)):
e = add_traj_image(scenario=scenario,
example=e,
local_env_w=100,
states_keys=['link_bot'],
local_env_h=100,
rope_image_k=10000,
batch_size=args.batch_size)
print('{:.5f}'.format(perf_counter() - t0))
# OLD
t0 = perf_counter()
for e in progressbar.progressbar(tf_dataset.take(batches)):
e = add_traj_image_to_example_old(scenario=scenario,
example=e,
local_env_w=100,
states_keys=['link_bot'],
local_env_h=100,
rope_image_k=10000,
batch_size=args.batch_size)
print('{:.5f}'.format(perf_counter() - t0))
# plt.plot(ram_usage)
# plt.xlabel("iteration")
# plt.ylabel("ram usage (bytes)")
# plt.show()
except KeyboardInterrupt:
pass