def visualize_nerf_outputs(nerf_out: dict, output_cache: List, viz: Visdom,
visdom_env: str):
"""
Visualizes the outputs of the `RadianceFieldRenderer`.
Args:
nerf_out: An output of the validation rendering pass.
output_cache: A list with outputs of several training render passes.
viz: A visdom connection object.
visdom_env: The name of visdom environment for visualization.
"""
# Show the training images.
ims = torch.stack([o["image"] for o in output_cache])
ims = torch.cat(list(ims), dim=1)
viz.image(
ims.permute(2, 0, 1),
env=visdom_env,
win="images",
opts={"title": "train_images"},
)
# Show the coarse and fine renders together with the ground truth images.
ims_full = torch.cat(
[
nerf_out[imvar][0].permute(2, 0, 1).detach().cpu().clamp(0.0, 1.0)
for imvar in ("rgb_coarse", "rgb_fine", "rgb_gt")
],
dim=2,
)
viz.image(
ims_full,
env=visdom_env,
win="images_full",
opts={"title": "coarse | fine | target"},
)
# Make a 3D plot of training cameras and their emitted rays.
camera_trace = {
f"camera_{ci:03d}": o["camera"].cpu()
for ci, o in enumerate(output_cache)
}
ray_pts_trace = {
f"ray_pts_{ci:03d}": Pointclouds(
ray_bundle_to_ray_points(
o["coarse_ray_bundle"]).detach().cpu().view(1, -1, 3))
for ci, o in enumerate(output_cache)
}
plotly_plot = plot_scene(
{
"training_scene": {
**camera_trace,
**ray_pts_trace,
},
},
pointcloud_max_points=5000,
pointcloud_marker_size=1,
camera_scale=0.3,
)
viz.plotlyplot(plotly_plot, env=visdom_env, win="scenes")
def _visdom_plot_scene(
train_cameras,
test_cameras,
) -> None:
from pytorch3d.vis.plotly_vis import plot_scene
p = plot_scene(
{"scene": {
"train_cams": train_cameras,
"test_cams": test_cameras,
}})
from visdom import Visdom
viz = Visdom()
viz.plotlyplot(p, env="cam_traj_dbg", win="cam_trajs")
import pdb
pdb.set_trace()
class Visualizer:
def __init__(self, cfg: DictConfig):
self.cfg = cfg.visdom
visdom_command = 'screen -S visdom_{} -d -m bash -c "python -m visdom.server -port {}"'.format(
self.cfg.port, self.cfg.port
)
os.mkdir("visdom")
os.system(visdom_command)
time.sleep(2)
# self.env = self.cfg.default_env_name # TODO: What is this
self.vis = Visdom(
port=self.cfg.port,
log_to_filename=os.path.join("visdom", self.cfg.log_to_filename),
offline=self.cfg.offline,
)
(self.x_min, self.x_max), (self.y_min, self.y_max) = (
(self.cfg.x_min, self.cfg.x_max),
(self.cfg.y_min, self.cfg.y_max),
)
self.counter = 0
self.plots = {}
def img_result(
self,
img_list: List[np.ndarray],
nrow: int,
caption: str = "view",
title: str = "title",
win: int = 1,
env: str = None,
):
self.vis.images(
img_list,
nrow=nrow,
win=win,
opts={"caption": caption, "title": title},
env=env,
)
def plot_img_255(
self, img_key: str, img: np.ndarray, env: str = None, opts: Dict = None,
):
"""Visdom plot a single image (channels-first CxHxW)"""
self.vis.image(img, win=img_key, opts=opts, env=env)
def plot_matplotlib(self, fig, caption="view", title="title", win=1, env=None):
self.vis.matplot(
fig,
win=win,
opts={"caption": caption, "title": title, "resizable": True},
env=env,
)
def plot_plotly(self, fig, caption="view", title="title", win=1, env=None):
self.vis.plotlyplot(fig, win=win, env=env)
def plot(
self,
plot_key: str,
split_name: str,
x: int,
y: int,
env: str = None,
opts: Dict = None,
):
"""Visdom plot line data"""
if plot_key not in self.plots:
self.plots[plot_key] = self.vis.line(
X=np.array([x, x]),
Y=np.array([y, y]),
env=env,
opts={**opts, "legend": [split_name]},
)
else:
self.vis.line(
X=np.array([x]),
Y=np.array([y]),
env=env,
win=self.plots[plot_key],
name=split_name,
update="append",
)
class TestDatasetVisualize(unittest.TestCase):
def setUp(self):
if os.environ.get("INSIDE_RE_WORKER") is not None:
raise unittest.SkipTest("Visdom not available")
category = "skateboard"
stack = contextlib.ExitStack()
dataset_root, path_manager = stack.enter_context(get_skateboard_data())
self.addCleanup(stack.close)
frame_file = os.path.join(dataset_root, category,
"frame_annotations.jgz")
sequence_file = os.path.join(dataset_root, category,
"sequence_annotations.jgz")
self.image_size = 256
self.datasets = {
"simple":
ImplicitronDataset(
frame_annotations_file=frame_file,
sequence_annotations_file=sequence_file,
dataset_root=dataset_root,
image_height=self.image_size,
image_width=self.image_size,
box_crop=True,
load_point_clouds=True,
path_manager=path_manager,
),
"nonsquare":
ImplicitronDataset(
frame_annotations_file=frame_file,
sequence_annotations_file=sequence_file,
dataset_root=dataset_root,
image_height=self.image_size,
image_width=self.image_size // 2,
box_crop=True,
load_point_clouds=True,
path_manager=path_manager,
),
"nocrop":
ImplicitronDataset(
frame_annotations_file=frame_file,
sequence_annotations_file=sequence_file,
dataset_root=dataset_root,
image_height=self.image_size,
image_width=self.image_size // 2,
box_crop=False,
load_point_clouds=True,
path_manager=path_manager,
),
}
self.datasets.update({
k + "_newndc": _change_annotations_to_new_ndc(dataset)
for k, dataset in self.datasets.items()
})
self.visdom = Visdom()
if not self.visdom.check_connection():
print(
"Visdom server not running! Disabling visdom visualizations.")
self.visdom = None
def _render_one_pointcloud(self, point_cloud, cameras, render_size):
(_image_render, _, _) = render_point_cloud_pytorch3d(
cameras,
point_cloud,
render_size=render_size,
point_radius=1e-2,
topk=10,
bg_color=0.0,
)
return _image_render.clamp(0.0, 1.0)
def test_one(self):
"""Test dataset visualization."""
if os.environ.get("INSIDE_RE_WORKER") is not None:
raise unittest.SkipTest("Visdom not available")
for max_frames in (16, -1):
for load_dataset_point_cloud in (True, False):
for dataset_key in self.datasets:
self._gen_and_render_pointcloud(max_frames,
load_dataset_point_cloud,
dataset_key)
def _gen_and_render_pointcloud(self, max_frames, load_dataset_point_cloud,
dataset_key):
dataset = self.datasets[dataset_key]
# load the point cloud of the first sequence
sequence_show = list(dataset.seq_annots.keys())[0]
device = torch.device("cuda:0")
point_cloud, sequence_frame_data = get_implicitron_sequence_pointcloud(
dataset,
sequence_name=sequence_show,
mask_points=True,
max_frames=max_frames,
num_workers=10,
load_dataset_point_cloud=load_dataset_point_cloud,
)
# render on gpu
point_cloud = point_cloud.to(device)
cameras = sequence_frame_data.camera.to(device)
# render the point_cloud from the viewpoint of loaded cameras
images_render = torch.cat([
self._render_one_pointcloud(
point_cloud,
cameras[frame_i],
(
dataset.image_height,
dataset.image_width,
),
) for frame_i in range(len(cameras))
]).cpu()
images_gt_and_render = torch.cat(
[sequence_frame_data.image_rgb, images_render], dim=3)
imfile = os.path.join(
os.path.split(os.path.abspath(__file__))[0],
"test_dataset_visualize" + f"_max_frames={max_frames}" +
f"_load_pcl={load_dataset_point_cloud}.png",
)
print(f"Exporting image {imfile}.")
torchvision.utils.save_image(images_gt_and_render, imfile, nrow=2)
if self.visdom is not None:
test_name = f"{max_frames}_{load_dataset_point_cloud}_{dataset_key}"
self.visdom.images(
images_gt_and_render,
env="test_dataset_visualize",
win=f"pcl_renders_{test_name}",
opts={"title": f"pcl_renders_{test_name}"},
)
plotlyplot = plot_scene(
{
"scene_batch": {
"cameras": cameras,
"point_cloud": point_cloud,
}
},
camera_scale=1.0,
pointcloud_max_points=10000,
pointcloud_marker_size=1.0,
)
self.visdom.plotlyplot(
plotlyplot,
env="test_dataset_visualize",
win=f"pcl_{test_name}",
)
class Custom_Vis:
def __init__(self):
self.vis = Visdom(username='', password='')
# self.vis.env = env_name
# self.vis.close()
# self.pred_error = {}
def _clear_env(self, clear: bool, env: str):
if clear:
self.vis.close(env=env)
def loss_plot(self, env: str, checkpoint: Checkpoint):
"""
The function to monitoring training.
"""
opts = dict(title='Loss curve',
legend=['train loss', 'valid_loss'],
showlegend=True,
width=400,
height=400,
linecolor=np.array([[0, 0, 255], [255, 0, 0]]))
win = np.inf
epoch_list = checkpoint.epoch_list
train_loss_list_per_epoch = checkpoint.train_loss_list_per_epoch
valid_loss_list = checkpoint.valid_loss_list
try:
if len(epoch_list) < win:
self._update_loss_plot(
epoch_list=epoch_list,
train_loss_list_per_epoch=train_loss_list_per_epoch,
valid_loss_list=valid_loss_list,
opts=opts,
win=self.loss_plt,
env=env)
else:
self._update_loss_plot(
epoch_list=epoch_list[-win:],
train_loss_list_per_epoch=train_loss_list_per_epoch[-win:],
valid_loss_list=valid_loss_list[-win:],
opts=opts,
win=self.loss_plt,
env=env)
except AttributeError:
self.loss_plt = self.vis.line(
X=np.array([epoch_list, epoch_list]).T,
Y=np.array([train_loss_list_per_epoch, valid_loss_list]).T,
opts=opts,
env=env)
def print_params(self,
env: str,
params: dict,
title: str,
clear: bool = False):
"""
Print function to display parameters
"""
self._clear_env(clear, env)
opts = dict(width=400, height=400)
text = f'<h2> {title} </h2><br>'
for keys, values in params.items():
text += f'<h4>{keys}: {values} </h4>'
self.vis.text(text, opts=opts, env=env)
def data_plot(self,
env: str,
data: np.ndarray,
ano_set: np.ndarray,
ano_features: np.ndarray,
clear: bool = True):
"""
The function for plot of anomaly subsequence.
"""
self._clear_env(clear, env)
if len(ano_set) > 10:
size = 10
else:
size = len(ano_set)
opts = dict(width=1200, height=400)
selected = np.random.choice(a=np.arange(len(ano_set)),
size=size,
replace=False)
for sel in selected:
setattr(self, f'v_{sel}',
self.vis.line(X=[0, 1], Y=[0, 1], opts=opts, env=env))
fig = go.Figure()
ano_idx, feature = ano_set[sel], ano_features[sel]
s, e = ano_idx[0], ano_idx[-1]
margin = e - s
for f in feature:
fig.add_trace(
go.Scatter(x=np.arange(s - margin, e + margin),
y=data[s - margin:e + margin, f],
mode='lines',
name=str(f)))
fig.update_layout(shapes=[
dict(type='rect',
xref='x',
yref='paper',
x0=s,
y0=0,
x1=e,
y1=1,
fillcolor='LightSalmon',
opacity=0.5,
layer='below',
line_width=0)
],
title='Comparision between normal and anomaly',
title_x=0.5,
autosize=False,
width=900,
height=400,
xaxis=go.layout.XAxis(title='timestamp',
showticklabels=False))
fig.update_xaxes(showgrid=False, zeroline=False)
fig.update_yaxes(showgrid=False, zeroline=False)
self.vis.plotlyplot(fig, win=getattr(self, f'v_{sel}'), env=env)
def print_training(self, env: str, EPOCH: str, epoch: str,
training_time: float, iter_time: float,
avg_train_loss: float, valid_loss: float, patience: int,
counter: int):
"""
Print training status and loss.
p: patience
c: counter
"""
t_m = f'{int(training_time//60):2d}'
t_s = f'{training_time%60:5.2f}'
i_m = f'{int(iter_time//60):2d}'
i_s = f'{iter_time%60:5.2f}'
state = patience - counter
opts = dict(width=400, height=400)
if patience != np.inf:
text = '<h2> Training status </h2><br>'\
f'\r <h4>Epoch: {epoch:3d} / {str(EPOCH):3s}</h4>'\
f'<h4>train time: {t_m}m {t_s}s</h4>'\
f'<h4>iteration time: {i_m}m {i_s}s</h4>'\
f'<h4>avg train loss: {avg_train_loss}</h4>'\
f'<h4>valid loss: {valid_loss}</h4>'\
f'\r <h4>EarlyStopping: {">"*counter+"-"*(state)} |</h4>'
else:
text = '<h2> Training status </h2><br>'\
f'\r <h4>Epoch: {epoch:3d} / {str(EPOCH):3s}</h4>'\
f'<h4>train time: {t_m}m {t_s}s</h4>'\
f'<h4>iteration time: {i_m}m {i_s}s</h4>'\
f'<h4>avg train loss: {avg_train_loss}</h4>'\
f'<h4>valid loss: {valid_loss}</h4>'
try:
self.vis.text(text,
win=self.training,
append=False,
opts=opts,
env=env)
except AttributeError:
self.training = self.vis.text(text, opts=opts, env=env)
def score_distribution(self, env: str, anomaly_label: np.ndarray,
anomaly_score: np.ndarray,
filtered_score: np.ndarray):
fig_score = go.Figure()
fig_score.add_trace(
go.Histogram(x=anomaly_score[anomaly_label == 0],
histnorm='probability density',
name='normal'))
fig_score.add_trace(
go.Histogram(x=anomaly_score[anomaly_label == 1],
histnorm='probability density',
name='anomaly'))
if not isinstance(filtered_score, type(None)):
fig_score.add_trace(
go.Histogram(x=filtered_score,
histnorm='probability density',
name='filtered'))
fig_score.update_layout(title_text='Anomaly Score Distribution',
title_x=0.5,
barmode='overlay',
xaxis=go.layout.XAxis(title='anomaly score'),
autosize=False,
width=600,
height=300)
fig_score.update_traces(opacity=0.5)
try:
self.vis.plotlyplot(fig_score, win=self.vis_score, env=env)
except AttributeError:
self.vis_score = self.vis.plotlyplot(fig_score, env=env)
def ROC_curve(self, env: str, anomaly_label: np.ndarray,
anomaly_score: np.ndarray):
label = anomaly_label
score = anomaly_score
if np.unique(label).shape[0] == 1:
return
fpr, tpr, thresholds = roc_curve(label, score)
auroc = roc_auc_score(label, score)
fig_roc = go.Figure()
fig_roc.add_trace(
go.Scatter(x=fpr, y=tpr, mode='lines', fill='tozeroy', name='ROC'))
fig_roc.add_shape(type='line',
line=dict(dash='dash'),
x0=0,
x1=1,
y0=0,
y1=1)
fig_roc.update_layout(
title_text=f'ROC Curve({auroc:.4f})',
title_x=0.5,
xaxis=go.layout.XAxis(title='False Positive Rate'),
yaxis=go.layout.YAxis(title='True Positive Rate'),
autosize=False,
width=600,
height=600)
# insert to visdom
try:
self.vis.plotlyplot(fig_roc, win=self.vis_roc, env=env)
except AttributeError:
self.vis_roc = self.vis.plotlyplot(fig_roc, env=env)
return auroc
def _update_loss_plot(self, epoch_list: list,
train_loss_list_per_epoch: list,
valid_loss_list: list, opts: dict, win: Callable,
env: str):
self.vis.line(X=np.array([epoch_list, epoch_list]).T,
Y=np.array([train_loss_list_per_epoch,
valid_loss_list]).T,
opts=opts,
win=win,
update='replace',
env=env)
def lof_score(self, env: str, lof_score: np.ndarray):
fig = go.Figure()
fig.add_trace(
go.Histogram(x=lof_score,
histnorm='probability density',
name='lof_score'))
fig.update_layout(title_text='LOF Score Distribution',
title_x=0.5,
barmode='overlay',
autosize=False,
width=600,
height=600)
fig.update_traces(opacity=0.5)
self.vis.plotlyplot(fig, env=env)
default=DEFAULT_HOSTNAME,
help='Server address of the target to run the demo on.')
FLAGS = parser.parse_args()
viz = Visdom(port=FLAGS.port, server=FLAGS.server)
hub = "/Users/afq/Google Drive/networks/"
eoses = [
"water_slgc_logp_64",
# "water_lg",
# "water_linear",
]
for eos in eoses:
# netplots = rout.plot_networks(surfs)
# Subplots
simplots = rout.make_simulation_plot_list(
hub + 'test_databases/' + eos + '_testing.db', eos)
for p in simplots:
viz.plotlyplot(p, env=eos)
# The monolithic plot
# simplots = rout.plot_simulations(hub+'test_databases/'+eos+'_testing.db',eos)
# viz.plotlyplot(simplots, win='win_'+eos, env=eos)
surfs = rout.read_networks(hub + 'training_' + eos)
viz.update_window_opts('win_' + eos, {'width': 500, 'height': 500})
netplots = rout.generate_trisurf_plots(surfs)
for n, p in netplots:
viz.plotlyplot(p, win='net_' + eos + n, env=eos)
viz.update_window_opts('net_' + eos + n, {'width': 200, 'height': 200})
