def _add_3d_torch(self,
tag,
data,
step,
logdir=None,
max_outputs=1,
label_to_names=None,
description=None):
walltime = None
if step is None:
raise ValueError("Step is not provided or set.")
mdata = {} if label_to_names is None else {'label_to_names': label_to_names}
summary_metadata = metadata.create_summary_metadata(description=description,
metadata=mdata)
writer = self._get_file_writer()
if logdir is None:
logdir = writer.get_logdir()
write_dir = PluginDirectory(logdir, metadata.PLUGIN_NAME)
geometry_metadata_string = _write_geometry_data(write_dir, tag, step, data,
max_outputs)
tensor_proto = TensorProto(dtype='DT_STRING',
string_val=[geometry_metadata_string],
tensor_shape=TensorShapeProto())
writer.add_summary(
Summary(value=[
Summary.Value(
tag=tag, tensor=tensor_proto, metadata=summary_metadata)
]), step, walltime)
def add_3d(name,
data,
step,
logdir=None,
max_outputs=1,
label_to_names=None,
description=None):
"""Write 3D geometry data as summary.
Args:
name (str): A name or tag for this summary. The summary tag used for
TensorBoard will be this name prefixed by any active name scopes.
data (dict): A dictionary of tensors representing 3D data. Tensorflow,
PyTorch, Numpy and Open3D tensors are supported. The following keys
are supported:
- ``vertex_positions``: shape `(B, N, 3)` where B is the number of
point clouds and must be same for each key. N is the number of
3D points. Will be cast to ``float32``.
- ``vertex_colors``: shape `(B, N, 3)` Will be converted to ``uint8``.
- ``vertex_normals``: shape `(B, N, 3)` Will be cast to ``float32``.
- ``vertex_texture_uvs``: shape `(B, N, 2)` Per vertex UV coordinates
for applying material texture maps. Will be cast to ``float32``.
Only one of ``[vertex|triangle]_texture_uvs`` should be provided.
- ``vertex_*FEATURE*``: shape `(B, N, _)`. Store arbitrary vertex
features. Floats will be cast to ``float32`` and integers to
``int32``.
- ``triangle_indices``: shape `(B, Nf, 3)`. Will be cast to ``uint32``.
- ``triangle_texture_uvs``: shape `(B, Nf, 3, 2)` Per triangle UV
coordinates for applying material texture maps. Will be cast to
``float32``. Only one of ``[vertex|triangle]_texture_uvs`` should be
provided.
- ``line_indices``: shape `(B, Nl, 2)`. Will be cast to ``uint32``.
- ``bboxes``: shape `(B, Nbb)`. The tensor dtype should be
`open3d.ml.vis.BoundingBox3D``. Property references not supported.
Use separate from other 3D properties.
- ``material_name``: shape `(B,)` and dtype ``str``. Base PBR material
name is required to specify any material properties. Open3D
built-in materials: ``defaultLit``, ``defaultUnlit``, ``unlitLine``,
``unlitGradient``, ``unlitSolidColor``.
- ``material_scalar_*PROPERTY*``: Any material scalar property with
float values of shape `(B,)`. e.g. To specify the property
`metallic`, use the key `material_scalar_metallic`.
- ``material_vector_*PROPERTY*``: Any material 4-vector property with
float values of shape `(B, 4)` e.g. To specify the property
`baseColor`, use the key `material_vector_base_color`.
- ``material_texture_map_*TEXTURE_MAP*``: PBR Material texture maps.
e.g. ``material_texture_map_metallic`` represents a texture map
describing the metallic property for rendering.
Values are Tensors with shape `(B, Nr, Nc, C)`, corresponding to a
batch of texture maps with `C` channels and shape `(Nr, Nc)`. The
geometry must have ``[vertex|triangle]_texture_uvs`` coordinates to
use any texture map.
For batch_size B=1, the tensors may drop a rank (e.g. (N,3)
``vertex_positions``, (4,) material vector properties or float scalar ()
material scalar properties.). Variable sized elements in a batch are
also supported. In this case, use a sequence of tensors. For example, to
save a batch of 2 point clouds with 8 and 16 points each, data should
contain `{'vertex_positions': (pcd1, pcd2)}` where `pcd1.shape = (8, 3)` and
`pcd2.shape = (16, 3)`.
Floating point color and texture map data will be clipped to the range
[0,1] and converted to ``uint8`` range [0,255]. ``uint16`` data will be
compressed to the range [0,255].
Any data tensor (with ndim>=2 including batch_size), may be replaced by
an ``int`` scalar referring to a previous step. This allows reusing a
previously written property in case that it does not change at different
steps. This is not supported for ``material_name``,
``material_scalar_*PROPERTY*`` and custom vertex features.
Please see the `Filament Materials Guide
<https://google.github.io/filament/Materials.html#materialmodels>`__ for
a complete explanation of material properties.
step (int): Explicit ``int64``-castable monotonic step value for this
summary. [`TensorFlow`: If ``None``, this defaults to
`tf.summary.experimental.get_step()`, which must not be ``None``.]
logdir (str): The logging directory used to create the SummaryWriter.
[`PyTorch`: This will be automatically inferred if not provided or
``None``.]
max_outputs (int): Optional integer. At most this many 3D elements will be
emitted at each step. When more than `max_outputs` 3D elements are
provided, the first ``max_outputs`` 3D elements will be used and the
rest silently discarded. Use ``0`` to save everything.
label_to_names (dict): Optional mapping from labels (e.g. int used in
labels for bboxes or vertices) to category names. Only data from the
first step is saved for any tag during a run.
description (str): Optional long-form description for this summary, as a
constant ``str``. Markdown is supported. Defaults to empty. Currently
unused.
Returns:
[TensorFlow] True on success, or false if no summary was emitted because
no default summary writer was available.
Raises:
ValueError: if a default writer exists, but no step was provided and
`tf.summary.experimental.get_step()` is None. Also raised when used with
Tensorflow and ``logdir`` is not provided or ``None``.
RuntimeError: Module level function is used without a TensorFlow
installation. Use the PyTorch `SummaryWriter.add_3d()` bound method
instead.
Examples:
With Tensorflow:
.. code::
import tensorflow as tf
import open3d as o3d
from open3d.visualization.tensorboard_plugin import summary
from open3d.visualization.tensorboard_plugin.util import to_dict_batch
logdir = "demo_logs/"
writer = tf.summary.create_file_writer(logdir)
cube = o3d.geometry.TriangleMesh.create_box(1, 2, 4)
cube.compute_vertex_normals()
colors = [(1.0, 0.0, 0.0), (0.0, 1.0, 0.0), (0.0, 0.0, 1.0)]
with writer.as_default():
for step in range(3):
cube.paint_uniform_color(colors[step])
summary.add_3d('cube',
to_dict_batch([cube]),
step=step,
logdir=logdir)
With PyTorch:
.. code::
from torch.utils.tensorboard import SummaryWriter
import open3d as o3d
from open3d.visualization.tensorboard_plugin import summary
from open3d.visualization.tensorboard_plugin.util import to_dict_batch
writer = SummaryWriter("demo_logs/")
cube = o3d.geometry.TriangleMesh.create_box(1, 2, 4)
cube.compute_vertex_normals()
colors = [(1.0, 0.0, 0.0), (0.0, 1.0, 0.0), (0.0, 0.0, 1.0)]
for step in range(3):
cube.paint_uniform_color(colors[step])
writer.add_3d('cube', to_dict_batch([cube]), step=step)
Now use ``tensorboard --logdir demo_logs`` to visualize the 3D data.
"""
if tf is None:
raise RuntimeError(
"TensorFlow not found. Please use module level ``add_3d`` only "
"with TensorFlow. Use the bound method ``SummaryWriter.add_3d`` "
"with PyTorch.")
if step is None:
step = tf.summary.experimental.get_step()
if step is None:
raise ValueError("Step is not provided or set.")
if logdir is None:
raise ValueError("logdir must be provided with TensorFlow.")
mdata = {} if label_to_names is None else {'label_to_names': label_to_names}
summary_metadata = metadata.create_summary_metadata(description=description,
metadata=mdata)
# TODO(https://github.com/tensorflow/tensorboard/issues/2109): remove fallback
summary_scope = (getattr(tf.summary.experimental, "summary_scope", None) or
tf.summary.summary_scope)
with summary_scope(name, "open3d_summary",
values=[data, max_outputs, step]) as (tag, unused_scope):
# Defer preprocessing by passing it as a callable to write(),
# wrapped in a LazyTensorCreator for backwards compatibility, so that we
# only do this work when summaries are actually written, i.e. if
# record_if() returns True.
@lazy_tensor_creator.LazyTensorCreator
def lazy_tensor():
write_dir = PluginDirectory(logdir, metadata.PLUGIN_NAME)
geometry_metadata_string = _write_geometry_data(
write_dir, tag, step, data, max_outputs)
return tf.convert_to_tensor(geometry_metadata_string)
return tf.summary.write(tag=tag,
tensor=lazy_tensor,
step=step,
metadata=summary_metadata)
def add_3d(name, data, step, logdir=None, max_outputs=1, description=None):
"""Write 3D geometry data as summary.
Args:
name (str): A name or tag for this summary. The summary tag used for TensorBoard
will be this name prefixed by any active name scopes.
data (dict): A dictionary of tensors representing 3D data. Tensorflow,
PyTorch, Numpy and Open3D tensors are supported. The following keys
are supported:
- ``vertex_positions``: shape `(B, N, 3)` where B is the number of
point clouds and must be same for each key. N is the number of
3D points. Will be cast to ``float32``.
- ``vertex_colors``: shape `(B, N, 3)` Will be converted to ``uint8``.
- ``vertex_normals``: shape `(B, N, 3)` Will be cast to ``float32``.
- ``triangle_indices``: shape `(B, Nf, 3)`. Will be cast to ``uint32``.
- ``line_indices``: shape `(B, Nl, 2)`. Will be cast to ``uint32``.
For `batch_size` B=1, the tensors may have rank 2 instead of rank 3.
Floating point color data will be clipped to the range [0,1] and
converted to `uint8` range [0,255]. Other data types will be clipped into
an allowed range for safe casting to uint8.
Any data tensor, may be replaced by an integer scalar referring to a
previous step. This allows reusing a previously written property tensor
in the case that it does not change at different steps.
step (int): Explicit ``int64``-castable monotonic step value for this summary.
[`TensorFlow`: If ``None``, this defaults to
`tf.summary.experimental.get_step()`, which must not be ``None``.]
logdir (str): The logging directory used to create the SummaryWriter.
[`PyTorch`: This will be automatically inferred if not provided or
``None``.]
max_outputs (int): Optional integer. At most this many 3D elements will be
emitted at each step. When more than `max_outputs` 3D elements are
provided, the first ``max_outputs`` 3D elements will be used and the
rest silently discarded.
description (str): Optional long-form description for this summary, as a
constant ``str``. Markdown is supported. Defaults to empty. Currently
unused.
Returns:
True on success, or false if no summary was emitted because no default
summary writer was available.
Raises:
ValueError: if a default writer exists, but no step was provided and
`tf.summary.experimental.get_step()` is None. Also raised when used with
Tensorflow and ``logdir`` is not provided or ``None``.
RuntimeError: Module level function is used without a TensorFlow
installation. Use the PyTorch `SummaryWriter.add_3d()` bound method
instead.
Examples:
With Tensorflow:
.. code::
import tensorflow as tf
import open3d as o3d
from open3d.visualization.tensorboard_plugin import summary
from open3d.visualization.tensorboard_plugin.util import to_dict_batch
logdir = "demo_logs/"
writer = tf.summary.create_file_writer(logdir)
cube = o3d.geometry.TriangleMesh.create_box(1, 2, 4)
cube.compute_vertex_normals()
colors = [(1.0, 0.0, 0.0), (0.0, 1.0, 0.0), (0.0, 0.0, 1.0)]
with writer.as_default():
for step in range(3):
cube.paint_uniform_color(colors[step])
summary.add_3d('cube',
to_dict_batch([cube]),
step=step,
logdir=logdir)
With PyTorch:
.. code::
from torch.utils.tensorboard import SummaryWriter
import open3d as o3d
from open3d.visualization.tensorboard_plugin import summary
from open3d.visualization.tensorboard_plugin.util import to_dict_batch
writer = SummaryWriter("demo_logs/")
cube = o3d.geometry.TriangleMesh.create_box(1, 2, 4)
cube.compute_vertex_normals()
colors = [(1.0, 0.0, 0.0), (0.0, 1.0, 0.0), (0.0, 0.0, 1.0)]
for step in range(3):
cube.paint_uniform_color(colors[step])
writer.add_3d('cube', to_dict_batch([cube]), step=step)
Now use ``tensorboard --logdir demo_logs`` to visualize the 3D data.
"""
if tf is None:
raise RuntimeError(
"TensorFlow not found. Please use module level ``add_3d`` only "
"with TensorFlow. Use the bound method ``SummaryWriter.add_3d`` "
"with PyTorch.")
if step is None:
step = tf.summary.experimental.get_step()
if step is None:
raise ValueError("Step is not provided or set.")
if logdir is None:
raise ValueError("logdir must be provided with TensorFlow.")
summary_metadata = metadata.create_summary_metadata(
description=description)
# TODO(https://github.com/tensorflow/tensorboard/issues/2109): remove fallback
summary_scope = (getattr(tf.summary.experimental, "summary_scope", None)
or tf.summary.summary_scope)
with summary_scope(name,
"open3d_summary",
values=[data, max_outputs,
step]) as (tag, unused_scope):
# Defer preprocessing by passing it as a callable to write(),
# wrapped in a LazyTensorCreator for backwards compatibility, so that we
# only do this work when summaries are actually written, i.e. if
# record_if() returns True.
@lazy_tensor_creator.LazyTensorCreator
def lazy_tensor():
write_dir = PluginDirectory(logdir, metadata.PLUGIN_NAME)
geometry_metadata_string = _write_geometry_data(
write_dir, tag, step, data, max_outputs)
return tf.convert_to_tensor(geometry_metadata_string)
return tf.summary.write(tag=tag,
tensor=lazy_tensor,
step=step,
metadata=summary_metadata)
