def vis_example2d( example, num_frames: int = 20, fps: int = 4, reverse_xy: bool = False, flip_up_down: bool = False, class_names: Optional[Sequence[str]] = None, ): """ Visualize 2d example as an animated sequence of frames. Args: example: element from unbatched image stream, tuple of (features, label), where features is a dict with "coords", "time" and "polarity" keys. num_frames: total number of frames in resulting animation. fps: frame-rate reverse_xy: if True, transposes coordinates. flip_up_down: if true, flips up/down (after possible transposition from `reverse_xy`). class_names: if provided, prints `class_names[label]` rather than `label`. """ features, label = example coords = features["coords"] time = features["time"] time = (time - tf.reduce_min(time)).numpy() polarity = features["polarity"].numpy() if class_names is not None: print(class_names[label.numpy()]) else: print(label.numpy()) coords = (coords - tf.reduce_min(coords, axis=0)).numpy() print(f"{time.shape[0]} events over {time[-1] - time[0]} dt") if reverse_xy: coords = coords[:, -1::-1] if num_frames == 1: frame = as_frame(coords, polarity) plt.imshow(frame) plt.show() else: frames = as_frames( coords, time, polarity, num_frames=num_frames, flip_up_down=flip_up_down ) anim.animate_frames(frames, fps=fps, tight_layout=True)
import os from events_tfds.utils import make_monotonic from events_tfds.vis import image from events_tfds.vis import anim folder = "/home/rslsync/Resilio Sync/DHP19/DVS_movies/S1/session1" filename = "mov8.aedat" path = os.path.join(folder, filename) with tf.io.gfile.GFile(path, "rb") as fp: timestamp, x, y, pol, camera = load_dhp19_events(fp) timestamp = make_monotonic(timestamp, dtype=np.uint64) assert np.all(timestamp[1:] >= timestamp[:-1]) print("camera", np.min(camera), np.max(camera)) print("polarity", np.min(pol), np.max(pol)) print("x", np.min(x), np.max(x)) print("y", np.min(y), np.max(y)) cam_mask = camera == 1 pol = pol[cam_mask] x = x[cam_mask] y = y[cam_mask] timestamp = timestamp[cam_mask] print("num_events: {}".format(timestamp.size)) coords = np.stack((x, np.max(y) - y), axis=-1) print("Creating animation...") frames = image.as_frames(coords, timestamp, pol, num_frames=200) anim.animate_frames(frames, fps=20)
if __name__ == "__main__": save_path = "/tmp/asl-dvs.gif" # from scipy.io import loadmat # path = '/home/jackd/Downloads/y_4200.mat' # loadmat(path) download_config = None # download_config = tfds.core.download.DownloadConfig(register_checksums=True) builder = AslDvs() builder.download_and_prepare(download_config=download_config) from events_tfds.vis.image import as_frames from events_tfds.vis.anim import animate_frames for events, labels in builder.as_dataset(split="train", as_supervised=True): frames = as_frames( **{k: v.numpy() for k, v in events.items()}, num_frames=24, flip_up_down=True, ) print(tf.reduce_max(events["coords"], axis=0).numpy() + 1) t = events["time"].numpy() print(f"{t.size} events over dt = {t[-1] - t[0]}") print(f"class = {CLASSES[labels.numpy()]}") animate_frames(frames, fps=8, save_path=save_path) if save_path is not None: print(f"Animation saved to {save_path}") break
continue _, label, filename = path.split("/") example_id = int(filename[:-4]) time, coords, polarity = load_neuro_events(fobj) features = dict( events=dict(time=time, coords=coords, polarity=polarity), label=int(label), example_id=example_id, ) yield example_id, features if __name__ == "__main__": download_config = None # download_config = tfds.core.download.DownloadConfig( # register_checksums=True) builder = NMNIST() builder.download_and_prepare(download_config=download_config) from events_tfds.vis.image import as_frames from events_tfds.vis.anim import animate_frames for events, labels in builder.as_dataset(split="train", as_supervised=True): frames = as_frames(**{k: v.numpy() for k, v in events.items()}, num_frames=20) print(labels.numpy()) t = events["time"].numpy() print(t[-1] - t[0], len(t)) animate_frames(frames, fps=4)
import tensorflow_datasets as tfds import events_tfds.events.cifar10_dvs # pylint:disable=unused-import from events_tfds.vis.anim import animate_frames from events_tfds.vis.image import as_frames train_ds = tfds.load("cifar10_dvs", split="train", as_supervised=True) for events, labels in train_ds: coords = events["coords"].numpy() time = events["time"].numpy() polarity = events["polarity"].numpy() coords = coords[:, -1::-1] # x-y flipped frames = as_frames(coords=coords, time=time, polarity=polarity, num_frames=20) animate_frames(frames, fps=4)
timestamp, addr = load_raw_events( fp, filter_dvs=filter_dvs, # bytes_skip=bytes_skip, # bytes_trim=bytes_trim, # times_first=times_first ) x, y, polarity = parse_raw_address(addr, **kwargs) return timestamp, x, y, polarity if __name__ == "__main__": from events_tfds.utils import make_monotonic folder = "/home/rslsync/Resilio Sync/RoShamBoNPP/recordings/aedat/" # filename = 'background_10.aedat' filename = "paper_tobi_front.aedat" path = os.path.join(folder, filename) with open(path, "rb") as fp: time, x, y, pol = load_events(fp) time = make_monotonic(time) time -= np.min(time) x -= np.min(x) y -= np.min(y) from events_tfds.vis import image from events_tfds.vis import anim coords = np.stack((x, y), axis=-1) frames = image.as_frames(coords, time, pol, num_frames=100) anim.animate_frames(frames, 20)
def process_events(times, coords, polarity): print("{} events over {} ms".format(times.shape[0], (times[-1] - times[0]).numpy())) decay_time = DECAY_TIME spatial_buffer_size = SPATIAL_BUFFER event_sizes = [times.shape[0]] img_data = [ as_frames( coords.numpy(), [0, 1] if times.shape[0] == 0 else times.numpy(), polarity.numpy(), num_frames=NUM_FRAMES, shape=GRID_SHAPE[-1::-1], ) ] neigh_sizes = [] mean_neigh_sizes = [] shape = tf.constant(GRID_SHAPE, dtype=tf.int64) shaped_in_coords = coords coords = grid.ravel_multi_index(coords, shape) for spec, thresh in zip(SPECS, THRESHOLDS): partitions, indices, splits, out_shape = grid.sparse_neighborhood( shape, *spec) # resample indices_T, splits_T, _ = ragged.transpose_csr(indices, splits, partitions) out_times, out_coords = lif.spatial_leaky_integrate_and_fire( times, coords, grid_indices=indices_T, grid_splits=splits_T, decay_time=decay_time, threshold=thresh, out_size=tf.math.reduce_prod(out_shape).numpy(), **lif_kwargs) event_sizes.append(out_times.shape[0]) shaped_out_coords = grid.unravel_index_transpose(out_coords, out_shape) img_data.append( as_frames( shaped_out_coords.numpy(), [0, 1] if times.shape[0] == 0 else out_times.numpy(), num_frames=NUM_FRAMES, shape=out_shape.numpy()[-1::-1], )) # resample conv neigh_part, neigh_indices, neigh_splits = neigh.compute_neighbors( in_times=times, in_coords=coords, out_times=out_times, out_coords=out_coords, grid_indices=indices, grid_partitions=partitions, grid_splits=splits, event_duration=decay_time * 4, spatial_buffer_size=spatial_buffer_size, ) del neigh_part # neigh_indices, neigh_splits = ragged.mask_rows(neigh_indices, # neigh_splits, # neigh_part == 1) neigh_sizes.append(neigh_indices.shape[0]) mean_neigh_sizes.append(neigh_indices.shape[0] / out_times.shape[0]) # vis_adjacency(neigh_indices.numpy(), neigh_splits.numpy(), # times.numpy(), out_times.numpy(), decay_time) vis_graph( shaped_in_coords.numpy(), times.numpy(), grid.shift_grid_coords(shaped_out_coords, *spec).numpy(), out_times.numpy(), neigh_indices.numpy(), neigh_splits.numpy(), n=10, ) # in-place conv partitions, indices, splits = grid.sparse_neighborhood_in_place( out_shape, (3, 3)) neigh_part, neigh_indices, neigh_splits = neigh.compute_neighbors( out_times, out_coords, out_times, out_coords, grid_partitions=partitions, grid_indices=indices, grid_splits=splits, event_duration=decay_time * 8, spatial_buffer_size=spatial_buffer_size, ) neigh_sizes.append(neigh_indices.shape[0]) mean_neigh_sizes.append(neigh_indices.shape[0] / out_times.shape[0]) # vis_adjacency(neigh_indices.numpy(), neigh_splits.numpy(), # times.numpy(), out_times.numpy(), decay_time * 2) vis_graph( shaped_out_coords.numpy(), out_times.numpy(), shaped_out_coords.numpy(), # grid.shift_grid_coords(shaped_out_coords, (3, 3), (1, 1), # (1, 1)).numpy(), out_times.numpy(), neigh_indices.numpy(), neigh_splits.numpy(), n=10, ) plt.show() decay_time *= 2 shape = out_shape coords = out_coords shaped_in_coords = shaped_out_coords times = out_times print("event sizes: ", event_sizes) print("neigh sizes: ", neigh_sizes) print("mean degree: ", " ".join(["{:.2f}".format(s) for s in mean_neigh_sizes])) anim.animate_frames_multi(*img_data, fps=FPS)
save_path = "/tmp/lif.gif" dataset = NMNIST().as_dataset(split="train", as_supervised=True) dims = np.array((34, 34), dtype=np.int64) kernel_size = 3 stride = 2 padding = 1 frame_kwargs = dict(num_frames=60) for events, label in dataset: coords = events["coords"].numpy() time = events["time"].numpy() polarity = events["polarity"].numpy() img_data = [as_frames(coords, time, polarity, **frame_kwargs)] coords = grid.ravel_multi_index_transpose(coords, dims) sizes = [time.size] curr_shape = dims for _ in range(3): partitions, indices, splits, curr_shape = grid.sparse_neighborhood( in_shape=curr_shape, kernel_shape=np.array((kernel_size, kernel_size), dtype=np.int64), strides=np.array((stride, stride), dtype=np.int64), padding=np.array((padding, padding), dtype=np.int64), ) # spatial_leaky_integrate_and_fire needs transposed grid indices, splits, partitions = transpose_csr(indices, splits, partitions)
def vis_streams2d( meta_model_func: Callable, dataset: tf.data.Dataset, augment_func: Optional[Callable] = None, num_frames: int = 20, fps: int = 4, group_size: int = 1, skip_vis: int = False, flip_up_down: bool = False, ): """ Visualize 2D event streams. Args: meta_model_func: meta-model building function dataset: base dataset to visualize. augment_func: optional map function to apply to dataset. num_frames: number of frames in the final animation. fps: frame-rate group_size: if greater than 1, mean / std stats are printed for the size of each stream over this number of examples. skip_vis: if True, the visualization will be skipped and only example stats will be printed to screen. flip_up_down: if True, flips images in the final animation up/down. """ if augment_func is not None: dataset = dataset.map(augment_func) dataset, static_shapes = _get_cached_stream_dataset( meta_model_func, dataset, num_dims=2 ) sizes = np.zeros((group_size, len(static_shapes)), dtype=np.int64) for i, example in enumerate(dataset): example = tf.nest.map_structure(lambda x: x.numpy(), example) if len(example) == 3: times, coords, polarity = example else: times, coords = example polarity = None assert len(times) == len(coords) == len(static_shapes) img_data = [] ii = i % group_size sizes[ii] = [t.shape[0] for t in times] if ii == group_size - 1: print(f"{i}: sizes: {np.mean(sizes, axis=0).astype(np.int64)}") if group_size > 1: print(f"{i}: stds : {np.std(sizes, axis=0).astype(np.int64)}") if skip_vis: continue for i, (t, c, shape) in enumerate(zip(times, coords, static_shapes)): if t.shape[0] > 0: imgs = as_frames( c, t, num_frames=num_frames, polarity=polarity[: c.shape[0]] if i == 0 else None, shape=shape, flip_up_down=flip_up_down, ) img_data.append(imgs) anim.animate_frames_multi( *img_data, fps=fps, figsize=(10, 2), tight_layout=True )