def vis_example2d(
example,
num_frames: int = 20,
fps: int = 4,
reverse_xy: bool = False,
flip_up_down: bool = False,
class_names: Optional[Sequence[str]] = None,
):
"""
Visualize 2d example as an animated sequence of frames.
Args:
example: element from unbatched image stream, tuple of (features, label),
where features is a dict with "coords", "time" and "polarity" keys.
num_frames: total number of frames in resulting animation.
fps: frame-rate
reverse_xy: if True, transposes coordinates.
flip_up_down: if true, flips up/down (after possible transposition from
`reverse_xy`).
class_names: if provided, prints `class_names[label]` rather than `label`.
"""
features, label = example
coords = features["coords"]
time = features["time"]
time = (time - tf.reduce_min(time)).numpy()
polarity = features["polarity"].numpy()
if class_names is not None:
print(class_names[label.numpy()])
else:
print(label.numpy())
coords = (coords - tf.reduce_min(coords, axis=0)).numpy()
print(f"{time.shape[0]} events over {time[-1] - time[0]} dt")
if reverse_xy:
coords = coords[:, -1::-1]
if num_frames == 1:
frame = as_frame(coords, polarity)
plt.imshow(frame)
plt.show()
else:
frames = as_frames(
coords, time, polarity, num_frames=num_frames, flip_up_down=flip_up_down
)
anim.animate_frames(frames, fps=fps, tight_layout=True)
import os
from events_tfds.utils import make_monotonic
from events_tfds.vis import image
from events_tfds.vis import anim
folder = "/home/rslsync/Resilio Sync/DHP19/DVS_movies/S1/session1"
filename = "mov8.aedat"
path = os.path.join(folder, filename)
with tf.io.gfile.GFile(path, "rb") as fp:
timestamp, x, y, pol, camera = load_dhp19_events(fp)
timestamp = make_monotonic(timestamp, dtype=np.uint64)
assert np.all(timestamp[1:] >= timestamp[:-1])
print("camera", np.min(camera), np.max(camera))
print("polarity", np.min(pol), np.max(pol))
print("x", np.min(x), np.max(x))
print("y", np.min(y), np.max(y))
cam_mask = camera == 1
pol = pol[cam_mask]
x = x[cam_mask]
y = y[cam_mask]
timestamp = timestamp[cam_mask]
print("num_events: {}".format(timestamp.size))
coords = np.stack((x, np.max(y) - y), axis=-1)
print("Creating animation...")
frames = image.as_frames(coords, timestamp, pol, num_frames=200)
anim.animate_frames(frames, fps=20)
if __name__ == "__main__":
save_path = "/tmp/asl-dvs.gif"
# from scipy.io import loadmat
# path = '/home/jackd/Downloads/y_4200.mat'
# loadmat(path)
download_config = None
# download_config = tfds.core.download.DownloadConfig(register_checksums=True)
builder = AslDvs()
builder.download_and_prepare(download_config=download_config)
from events_tfds.vis.image import as_frames
from events_tfds.vis.anim import animate_frames
for events, labels in builder.as_dataset(split="train",
as_supervised=True):
frames = as_frames(
**{k: v.numpy()
for k, v in events.items()},
num_frames=24,
flip_up_down=True,
)
print(tf.reduce_max(events["coords"], axis=0).numpy() + 1)
t = events["time"].numpy()
print(f"{t.size} events over dt = {t[-1] - t[0]}")
print(f"class = {CLASSES[labels.numpy()]}")
animate_frames(frames, fps=8, save_path=save_path)
if save_path is not None:
print(f"Animation saved to {save_path}")
break
continue
_, label, filename = path.split("/")
example_id = int(filename[:-4])
time, coords, polarity = load_neuro_events(fobj)
features = dict(
events=dict(time=time, coords=coords, polarity=polarity),
label=int(label),
example_id=example_id,
)
yield example_id, features
if __name__ == "__main__":
download_config = None
# download_config = tfds.core.download.DownloadConfig(
# register_checksums=True)
builder = NMNIST()
builder.download_and_prepare(download_config=download_config)
from events_tfds.vis.image import as_frames
from events_tfds.vis.anim import animate_frames
for events, labels in builder.as_dataset(split="train",
as_supervised=True):
frames = as_frames(**{k: v.numpy()
for k, v in events.items()},
num_frames=20)
print(labels.numpy())
t = events["time"].numpy()
print(t[-1] - t[0], len(t))
animate_frames(frames, fps=4)
import tensorflow_datasets as tfds
import events_tfds.events.cifar10_dvs # pylint:disable=unused-import
from events_tfds.vis.anim import animate_frames
from events_tfds.vis.image import as_frames
train_ds = tfds.load("cifar10_dvs", split="train", as_supervised=True)
for events, labels in train_ds:
coords = events["coords"].numpy()
time = events["time"].numpy()
polarity = events["polarity"].numpy()
coords = coords[:, -1::-1] # x-y flipped
frames = as_frames(coords=coords,
time=time,
polarity=polarity,
num_frames=20)
animate_frames(frames, fps=4)
timestamp, addr = load_raw_events(
fp,
filter_dvs=filter_dvs,
# bytes_skip=bytes_skip,
# bytes_trim=bytes_trim,
# times_first=times_first
)
x, y, polarity = parse_raw_address(addr, **kwargs)
return timestamp, x, y, polarity
if __name__ == "__main__":
from events_tfds.utils import make_monotonic
folder = "/home/rslsync/Resilio Sync/RoShamBoNPP/recordings/aedat/"
# filename = 'background_10.aedat'
filename = "paper_tobi_front.aedat"
path = os.path.join(folder, filename)
with open(path, "rb") as fp:
time, x, y, pol = load_events(fp)
time = make_monotonic(time)
time -= np.min(time)
x -= np.min(x)
y -= np.min(y)
from events_tfds.vis import image
from events_tfds.vis import anim
coords = np.stack((x, y), axis=-1)
frames = image.as_frames(coords, time, pol, num_frames=100)
anim.animate_frames(frames, 20)
def process_events(times, coords, polarity):
print("{} events over {} ms".format(times.shape[0],
(times[-1] - times[0]).numpy()))
decay_time = DECAY_TIME
spatial_buffer_size = SPATIAL_BUFFER
event_sizes = [times.shape[0]]
img_data = [
as_frames(
coords.numpy(),
[0, 1] if times.shape[0] == 0 else times.numpy(),
polarity.numpy(),
num_frames=NUM_FRAMES,
shape=GRID_SHAPE[-1::-1],
)
]
neigh_sizes = []
mean_neigh_sizes = []
shape = tf.constant(GRID_SHAPE, dtype=tf.int64)
shaped_in_coords = coords
coords = grid.ravel_multi_index(coords, shape)
for spec, thresh in zip(SPECS, THRESHOLDS):
partitions, indices, splits, out_shape = grid.sparse_neighborhood(
shape, *spec)
# resample
indices_T, splits_T, _ = ragged.transpose_csr(indices, splits,
partitions)
out_times, out_coords = lif.spatial_leaky_integrate_and_fire(
times,
coords,
grid_indices=indices_T,
grid_splits=splits_T,
decay_time=decay_time,
threshold=thresh,
out_size=tf.math.reduce_prod(out_shape).numpy(),
**lif_kwargs)
event_sizes.append(out_times.shape[0])
shaped_out_coords = grid.unravel_index_transpose(out_coords, out_shape)
img_data.append(
as_frames(
shaped_out_coords.numpy(),
[0, 1] if times.shape[0] == 0 else out_times.numpy(),
num_frames=NUM_FRAMES,
shape=out_shape.numpy()[-1::-1],
))
# resample conv
neigh_part, neigh_indices, neigh_splits = neigh.compute_neighbors(
in_times=times,
in_coords=coords,
out_times=out_times,
out_coords=out_coords,
grid_indices=indices,
grid_partitions=partitions,
grid_splits=splits,
event_duration=decay_time * 4,
spatial_buffer_size=spatial_buffer_size,
)
del neigh_part
# neigh_indices, neigh_splits = ragged.mask_rows(neigh_indices,
# neigh_splits,
# neigh_part == 1)
neigh_sizes.append(neigh_indices.shape[0])
mean_neigh_sizes.append(neigh_indices.shape[0] / out_times.shape[0])
# vis_adjacency(neigh_indices.numpy(), neigh_splits.numpy(),
# times.numpy(), out_times.numpy(), decay_time)
vis_graph(
shaped_in_coords.numpy(),
times.numpy(),
grid.shift_grid_coords(shaped_out_coords, *spec).numpy(),
out_times.numpy(),
neigh_indices.numpy(),
neigh_splits.numpy(),
n=10,
)
# in-place conv
partitions, indices, splits = grid.sparse_neighborhood_in_place(
out_shape, (3, 3))
neigh_part, neigh_indices, neigh_splits = neigh.compute_neighbors(
out_times,
out_coords,
out_times,
out_coords,
grid_partitions=partitions,
grid_indices=indices,
grid_splits=splits,
event_duration=decay_time * 8,
spatial_buffer_size=spatial_buffer_size,
)
neigh_sizes.append(neigh_indices.shape[0])
mean_neigh_sizes.append(neigh_indices.shape[0] / out_times.shape[0])
# vis_adjacency(neigh_indices.numpy(), neigh_splits.numpy(),
# times.numpy(), out_times.numpy(), decay_time * 2)
vis_graph(
shaped_out_coords.numpy(),
out_times.numpy(),
shaped_out_coords.numpy(),
# grid.shift_grid_coords(shaped_out_coords, (3, 3), (1, 1),
# (1, 1)).numpy(),
out_times.numpy(),
neigh_indices.numpy(),
neigh_splits.numpy(),
n=10,
)
plt.show()
decay_time *= 2
shape = out_shape
coords = out_coords
shaped_in_coords = shaped_out_coords
times = out_times
print("event sizes: ", event_sizes)
print("neigh sizes: ", neigh_sizes)
print("mean degree: ",
" ".join(["{:.2f}".format(s) for s in mean_neigh_sizes]))
anim.animate_frames_multi(*img_data, fps=FPS)
save_path = "/tmp/lif.gif"
dataset = NMNIST().as_dataset(split="train", as_supervised=True)
dims = np.array((34, 34), dtype=np.int64)
kernel_size = 3
stride = 2
padding = 1
frame_kwargs = dict(num_frames=60)
for events, label in dataset:
coords = events["coords"].numpy()
time = events["time"].numpy()
polarity = events["polarity"].numpy()
img_data = [as_frames(coords, time, polarity, **frame_kwargs)]
coords = grid.ravel_multi_index_transpose(coords, dims)
sizes = [time.size]
curr_shape = dims
for _ in range(3):
partitions, indices, splits, curr_shape = grid.sparse_neighborhood(
in_shape=curr_shape,
kernel_shape=np.array((kernel_size, kernel_size), dtype=np.int64),
strides=np.array((stride, stride), dtype=np.int64),
padding=np.array((padding, padding), dtype=np.int64),
)
# spatial_leaky_integrate_and_fire needs transposed grid
indices, splits, partitions = transpose_csr(indices, splits,
partitions)
def vis_streams2d(
meta_model_func: Callable,
dataset: tf.data.Dataset,
augment_func: Optional[Callable] = None,
num_frames: int = 20,
fps: int = 4,
group_size: int = 1,
skip_vis: int = False,
flip_up_down: bool = False,
):
"""
Visualize 2D event streams.
Args:
meta_model_func: meta-model building function
dataset: base dataset to visualize.
augment_func: optional map function to apply to dataset.
num_frames: number of frames in the final animation.
fps: frame-rate
group_size: if greater than 1, mean / std stats are printed for the size of each
stream over this number of examples.
skip_vis: if True, the visualization will be skipped and only example stats will
be printed to screen.
flip_up_down: if True, flips images in the final animation up/down.
"""
if augment_func is not None:
dataset = dataset.map(augment_func)
dataset, static_shapes = _get_cached_stream_dataset(
meta_model_func, dataset, num_dims=2
)
sizes = np.zeros((group_size, len(static_shapes)), dtype=np.int64)
for i, example in enumerate(dataset):
example = tf.nest.map_structure(lambda x: x.numpy(), example)
if len(example) == 3:
times, coords, polarity = example
else:
times, coords = example
polarity = None
assert len(times) == len(coords) == len(static_shapes)
img_data = []
ii = i % group_size
sizes[ii] = [t.shape[0] for t in times]
if ii == group_size - 1:
print(f"{i}: sizes: {np.mean(sizes, axis=0).astype(np.int64)}")
if group_size > 1:
print(f"{i}: stds : {np.std(sizes, axis=0).astype(np.int64)}")
if skip_vis:
continue
for i, (t, c, shape) in enumerate(zip(times, coords, static_shapes)):
if t.shape[0] > 0:
imgs = as_frames(
c,
t,
num_frames=num_frames,
polarity=polarity[: c.shape[0]] if i == 0 else None,
shape=shape,
flip_up_down=flip_up_down,
)
img_data.append(imgs)
anim.animate_frames_multi(
*img_data, fps=fps, figsize=(10, 2), tight_layout=True
)