def __init__(self, fov, resolution, verts, faces, colors, translation,
rotation, near, far):
self.canvas = Canvas(size=(resolution, resolution))
self.view = self.canvas.central_widget.add_view()
self.view.camera = 'perspective'
self.view.camera.fov = fov
self.mesh = visuals.Mesh(shading=None)
self.mesh.attach(Alpha(1.0))
self.view.add(self.mesh)
self.tr = self.view.camera.transform
self.mesh.set_data(vertices=verts,
faces=faces,
vertex_colors=colors[:, :3])
self.near = near
self.far = far
self.transform(translation, rotation)
self.view_changed()
def __define_plane(self, w):
x, y, z = 0.5, 0.375, 1.0
v = np.array([[-x * w, -y * w, z * w], [-x * w, y * w, z * w],
[x * w, y * w, z * w], [-x * w, -y * w, z * w],
[x * w, y * w, z * w], [x * w, -y * w, z * w]])
colors = [(0.8, 0.5, 0.7) for i in range(len(v))]
return visuals.Mesh(v, color=(0.5, 0.5, 0.8, 0.8))
def init_canvas(self):
if self.verbose == 1:
print('init_canvas')
app.use_app('ipynb_webgl')
ca = vispy.scene.SceneCanvas(keys='interactive', show=True,size=(self.canvas_width,self.canvas_height),bgcolor=[1,1,1])
ca.show()
self.canvas = ca
view = ca.central_widget.add_view()
self.text = visuals.Text(anchor_x='right')
view.add(self.text)
self.label_texts = visuals.Text(anchor_x='right')
view.add(self.label_texts)
self.camera = CustomPanZoomCamera([self.text,self.label_texts])
view.camera = self.camera
axis = visuals.Axis(parent=view.scene)
self.view = view
self.line = visuals.Line()
self.line.set_data(pos=np.array([[0,0,0]]),color=np.array([[0,0,0,0]]))
self.line.order = 1
self.view.add(self.line)
self.mesh = visuals.Mesh()
self.mesh.order = 0
self.view.add(self.mesh)
app.run()
def __init__(self, filename, CG_pos=0.0):
self.CG_pos = np.array([CG_pos, 0.0, 0.0])
self.vertices, self.faces, self.normals, self.texcoords = self._load_obj(
filename)
self.visual = visuals.Mesh(self.vertices, self.faces, color='red')
self.visual.light_dir = [-0.3, -0.3, -1.0]
def createsphere(self, nrows=20, ncols=20, radius=1.0):
if not self.mesh:
mdata = create_sphere(nrows, ncols, radius)
self.mesh = visuals.Mesh(meshdata=mdata,
vertex_colors='black',
face_colors=(1.0, 0.0, 0.0, 1.0),
color=(0.5, 0.5, 0.5, .75))
self.view.add(self.mesh)
else:
print(dir(self.mesh))
def _updateMesh(self):
# add mesh data into the scene
# Vertices positions
# p = np.array([[1, 1, 1], [-1, 1, 1], [-1, -1, 1], [1, -1, 1],
# [1, -1, -1], [1, 1, -1], [-1, 1, -1], [-1, -1, -1]])
# # p = p * 1000.0
# faces = np.array([[0, 2, 1],
# [0, 3, 2],
# [0, 4, 3],
# [0, 5, 4],
# [0, 6, 5],
# [0, 1, 6],
# [1, 7, 6],
# [1, 2, 7],
# [7, 3, 4],
# [7, 2, 3],
# [4, 6, 7],
# [4, 5, 6]
# ])
p = self._verts.copy()
faces = []
for face in self._faces:
faces.append([face[2],face[1],face[0]])
faces = np.array(faces) # otherwise, it is dark.
# faces = self._faces
# for more deatils on color, please refer to vispy/colors/colorarray.py
# colors_p = np.array(
# [[1, 0, 0, 1], [0, 1, 0, 1], [0, 0, 1, 1], [0, 0, 0, 1], [0, 1, 1, 1], [1, 0, 1, 1], [1, 1, 0, 1],
# [1, 1, 1, 1]])
# colors_f = np.array([(1.0, 0.1, 0.0, 0.3) for k in range(faces.shape[0])])
color_mesh = np.array([[1.0, 1.0, 0.3,0.9]])
# for more details see vispy/visuals/mesh.py
mesh = visuals.Mesh(vertices=p, faces=faces,
# vertex_colors = colors_p,
# face_colors = colors_f,
color=color_mesh,
shading='flat',
mode='triangles')
self._mesh = mesh
def __init__(self, size, K, R, T):
h, w = size[:2]
self.size = size
self.canvas = scene.SceneCanvas(bgcolor='black', size=(w, h))
self.view = self.canvas.central_widget.add_view()
self.view.camera = 'perspective'
self.transform = self.view.camera.transform
self.view.camera.fov = camera_helper.calculate_fov(K, self.size)
self.mesh = visuals.Mesh(shading=None)
self.view.add(self.mesh)
# camera intrinsic parameter & extrinsic parameter
self.K = K
self.R = R
self.T = T
self.focal_length_pix = self.K[0, 0]
def set_render_infos(self, mesh_b=None):
"""
Set vertices, faces and colors.
"""
if mesh_b is not None:
vertices, colors, faces = mesh_helper.mesh_to_render_points_for_stereo(
self.mesh, mesh_b, self.get_K(), self.get_inv_K())
else:
vertices, colors, faces = mesh_helper.mesh_to_render_points(
self.mesh, self.get_K(), self.get_inv_K())
vertice = np.array(vertices).astype(np.float32)
colors = np.array(colors).astype(np.float32)
faces = np.array(faces).astype(np.long)
mesh = visuals.Mesh(shading=None)
mesh.set_data(vertices=vertice, faces=faces, vertex_colors=colors)
mesh.attach(Alpha(1.0))
self.virtual_camera.view.add(mesh)
def __init__(self, exp_df_path, trial_df_path, rig_leds_path):
exp_df = pd.read_pickle(exp_df_path)
trial_df = pd.read_pickle(trial_df_path)
self.df = exp_df.join(trial_df.drop('block', axis=1),
on='trial_number')
self.rig_leds = np.load(rig_leds_path)
self.precalculate_data()
verts, faces, normals, nothin = vispy.io.read_mesh(
os.path.join(fh.PACKAGE_DIR, '../datafiles', 'head.obj'))
verts = np.einsum('ni,ji->nj', (verts - verts.mean(axis=0)),
fh.from_yawpitchroll(180, 90, 0))
#verts = verts - verts.mean(axis=0)
# add SceneCanvas first to create QApplication object before widget
self.vispy_canvas = vispy.scene.SceneCanvas(create_native=True,
vsync=True,
show=True,
bgcolor=(0.2, 0.2, 0.2, 0))
super(ExperimentVisualization, self).__init__()
#self.setAttribute(Qt.WA_TranslucentBackground)
self.timer = vispy.app.Timer(1 / 30,
start=False,
connect=self.advance_frame)
self.n_trials = len(self.df)
self.current_trial = 0
self.i_frame = 0
self.current_row = self.df.iloc[self.current_trial]
self.current_R_helmet = None
self.current_gaze_normals = None
self.current_ref_points = None
self.vispy_view = self.vispy_canvas.central_widget.add_view()
self.vispy_view.camera = 'turntable'
self.vispy_view.camera.center = self.rig_leds[127, :] + (
self.rig_leds[0, :] - self.rig_leds[127, :]) + (
self.rig_leds[254, :] - self.rig_leds[127, :])
self.vispy_view.camera.fov = 40
self.vispy_view.camera.distance = 1500
self.main_layout = QtWidgets.QVBoxLayout()
self.setLayout(self.main_layout)
self.frame_slider = QtWidgets.QSlider(Qt.Horizontal)
self.frame_slider.setMinimum(0)
self.frame_slider.valueChanged.connect(self.on_slider_change)
self.trial_picker = QtWidgets.QSpinBox()
self.trial_picker.setMaximum(self.n_trials)
self.trial_picker.valueChanged.connect(self.on_picker_change)
self.trial_changed.connect(self.load_trial)
self.frame_changed.connect(self.load_frame)
self.picker_slider_layout = QtWidgets.QHBoxLayout()
self.main_layout.addWidget(self.vispy_canvas.native)
self.main_layout.addLayout(self.picker_slider_layout)
self.animation_button = QtWidgets.QPushButton('Start Animation')
self.picker_slider_layout.addWidget(self.animation_button)
self.animation_button.clicked.connect(self.toggle_animation)
self.frame_label = QtWidgets.QLabel('Frame')
self.picker_slider_layout.addWidget(self.frame_label)
self.picker_slider_layout.addWidget(self.frame_slider)
self.picker_slider_layout.addWidget(QtWidgets.QLabel('Trial'))
self.picker_slider_layout.addWidget(self.trial_picker)
self.rig_vis = visuals.Markers()
self.rig_vis.set_gl_state(depth_test=False)
self.rig_vis.antialias = 0
self.vispy_view.add(self.rig_vis)
self.helmet_vis = visuals.Markers()
self.vispy_view.add(self.helmet_vis)
self.gaze_vis = visuals.Line()
self.vispy_view.add(self.gaze_vis)
self.head_mesh = visuals.Mesh(vertices=verts,
shading='smooth',
faces=faces,
mode='triangles',
color=(0.5, 0.55, 0.7))
self.head_mesh.shininess = 0
self.head_mesh.light_dir = [0, 1, 1]
# self.head_mesh.light_color = np.array((1, 1, 0.95)) * 0.8
self.head_mesh.ambient_light_color = np.array((0.98, 0.98, 1)) * 0.2
self.head_mesh.attach(Alpha(0.3))
self.head_mesh.set_gl_state(depth_test=True, cull_face=True)
self.head_mesh_transform = MatrixTransform()
self.head_mesh.transform = self.head_mesh_transform
self.vispy_view.add(self.head_mesh)
self.trial_changed.emit(0)
self.frame_changed.emit(0)
self.show()
vispy.app.run()
def run3DVisualizationIPM(ipmPoints,
centroids,
violations,
frame,
render=False):
'''
Takes all the world coordinates produced by the IPM method, and their color values and plots them in 3D space, using vispy. Also draws halo cylinders around 3D points corresponding to people centroids (from efficientdet bounding boxes). Also draws 3D lines (tubes) between the halos that represent a pair of people which are violating the 6' restriction.
Parameters:
ipmPoints (list): World coordinates with color, generated by the ipm method (x,y,z,[r,g,b])
centroids (list): A list of lists, where each inner list object is a 3D world point, representing the centroid of a person (found using the efficientdet bounding boxes), in format [x,y,z]
violations (list): A list of lists, where each inner list is a pair of integers, which are two indices in the bBoxes list representing a pair of people violating the 6' restriction. Formatted as [[pi1,pi2],[pi3,pi4]]
frame (int): frame number for the filename to save to
render (bool): whether or not to render canvas to a file
Returns:
'''
# Create canvas to draw everything
canvas = vispy.scene.SceneCanvas(keys='interactive',
show=True,
size=(1920, 1080))
view = canvas.central_widget.add_view()
# Unpack ipm points
ipmPos = []
ipmColor = []
for point in ipmPoints:
ipmPos.append([point[0], point[1], point[2]])
r = point[3][0]
g = point[3][1]
b = point[3][2]
ipmColor.append([r, g, b])
pos = np.array(ipmPos)
colors = np.array(ipmColor)
# 3D scatter plot to show depth map pointcloud
scatter = visuals.Markers()
scatter.antialias = 0
scatter.set_data(pos, edge_color=None, face_color=colors, size=5)
view.add(scatter)
# Draw cylinders around centroids
for point in centroids:
x, y, z = point
cyl_mesh = vispy.geometry.create_cylinder(10,
10,
radius=[500, 500],
length=50)
# Move cylinder to correct location in 3D space
# Make sure to negate the y value, otherwise everything will be mirrored
vertices = cyl_mesh.get_vertices()
center = np.array([x, -y, z + 150], dtype=np.float32)
vtcs = np.add(vertices, center)
cyl_mesh.set_vertices(vtcs)
cyl = visuals.Mesh(meshdata=cyl_mesh, color='g')
view.add(cyl)
# Draw lines between violating people
for pair in violations:
x1, y1, z1 = centroids[pair[0]]
x2, y2, z2 = centroids[pair[1]]
#lin = visuals.Line(pos=np.array([[x1,-y1,z1+1],[x2,-y2,z2+1]]), color='r', method='gl')
#view.add(lin)
tube = visuals.Tube(points=np.array([[x1, -y1, z1 + 150],
[x2, -y2, z2 + 150]]),
radius=50,
color='red')
view.add(tube)
view.camera = 'turntable' # or try 'arcball'
view.camera.elevation = 30.5
view.camera.azimuth = -78.5
view.camera.distance = 8250.000000000002
view.camera.fov = 60
# Add a colored 3D axis for orientation
axis = visuals.XYZAxis(parent=view.scene)
if (render):
img = canvas.render()
fname = "out_renders/IPM" + str(frame) + ".png"
io.write_png(fname, img)
else:
vispy.app.run()
def reset(self):
""" Reset. """
# last key press (it should have a mutex, but visualization is not
# safety critical, so let's do things wrong)
self.action = "no" # no, next, back, quit are the possibilities
''' 3D points cloud or mesh SceneCanvas '''
# new canvas prepared for visualizing data
self.canvas = SceneCanvas(keys='interactive', show=True)
# interface (n next, b back, q quit, very simple)
self.canvas.events.key_press.connect(self.key_press)
self.canvas.events.draw.connect(self.draw)
# grid
self.grid = self.canvas.central_widget.add_grid()
# add point cloud views
self.scan_view = vispy.scene.widgets.ViewBox(border_color='white',
parent=self.canvas.scene)
self.grid.add_widget(self.scan_view, 0, 0)
# Camera location settings
self.scene_cam = vispy.scene.cameras.BaseCamera()
# self.scene_cam.center = (-10, -10, 10)
# self.scan_view.add(self.scene_cam)
# self.scene_cam.pre_transform.set_range()
canvas2 = vispy.app.Canvas()
w = QMainWindow()
widget = QWidget()
w.setCentralWidget(widget)
widget.setLayout(QVBoxLayout())
widget.layout().addWidget(canvas2.native)
widget.layout().addWidget(QPushButton())
w.show()
self.scan_vis = visuals.Mesh()
self.scan_vis_mean = visuals.Line()
self.scan_vis_cam = visuals.Line()
self.scan_bbox_3d = visuals.Line()
self.label_vis = visuals.Text()
self.scan_view.add(self.scan_vis)
self.scan_view.add(self.scan_vis_mean)
self.scan_view.add(self.scan_vis_cam)
self.scan_view.add(self.scan_bbox_3d)
self.scan_view.add(self.label_vis)
self.scan_view.camera = 'arcball'
self.tr = self.scan_vis.transforms.get_transform(map_from='visual',
map_to='canvas')
# self.scan_view.camera = self.scene_cam
# self.scan_view.camera = 'arcball' , 'turntable'
# self.scan_view.camera.transform.rotate(90, (0,1,0))
visuals.XYZAxis(parent=self.scan_view.scene)
''' 2D images SceneCanvas '''
# img canvas size
self.canvas_W = 320
self.canvas_H = 280
self.multiplier = 2
''' new canvas for RGB & Depth img '''
self.img_canvas = SceneCanvas(keys='interactive',
show=True,
size=(self.canvas_W,
self.canvas_H * self.multiplier))
self.img_grid = self.img_canvas.central_widget.add_grid()
# interface (n next, s start, p pause, q quit, )
self.img_canvas.events.key_press.connect(self.key_press)
self.img_canvas.events.draw.connect(self.draw)
# add rgb views
self.rgb_img_raw_view = vispy.scene.widgets.ViewBox(
border_color='white', parent=self.img_canvas.scene)
self.img_grid.add_widget(self.rgb_img_raw_view, 0, 0)
self.rgb_img_raw_vis = visuals.Image(cmap='viridis')
self.rgb_img_raw_view.add(self.rgb_img_raw_vis)
# add a view for the depth
self.depth_img_view = vispy.scene.widgets.ViewBox(
border_color='white', parent=self.img_canvas.scene)
self.img_grid.add_widget(self.depth_img_view, 1, 0)
self.depth_img_vis = visuals.Image(cmap='viridis')
self.depth_img_view.add(self.depth_img_vis)
''' new canvas for 3D scene graph img '''
self.scene_graph_canvas = SceneCanvas(keys='interactive',
show=True,
size=(640, 480))
self.scene_graph_grid = self.scene_graph_canvas.central_widget.add_grid(
)
self.scene_graph_canvas.events.key_press.connect(self.key_press)
self.scene_graph_canvas.events.draw.connect(self.draw)
# add a view for 3D scene graphs
self.scene_graph_view = vispy.scene.widgets.ViewBox(
border_color='white', parent=self.scene_graph_canvas.scene)
self.scene_graph_grid.add_widget(self.scene_graph_view, 0, 0)
self.scene_graph_vis = visuals.Image(cmap='viridis')
self.scene_graph_view.add(self.scene_graph_vis)
def reset(self):
""" Reset. """
# last key press (it should have a mutex, but visualization is not
# safety critical, so let's do things wrong)
self.action = "no" # no, next, back, quit are the possibilities
# 3D canvas
self.scan_canvas = SceneCanvas(keys='interactive',
show=True,
title='',
size=(1600, 600),
bgcolor='white')
self.scan_canvas.events.key_press.connect(self.key_press)
self.grid_view = self.scan_canvas.central_widget.add_grid()
# source laserscan 3D
self.scan_view = vispy.scene.widgets.ViewBox(
border_color='white', parent=self.scan_canvas.scene)
self.scan_vis = visuals.Markers()
self.scan_view.camera = 'turntable'
self.scan_view.add(self.scan_vis)
visuals.XYZAxis(parent=self.scan_view.scene)
self.grid_view.add_widget(self.scan_view, 0, 0)
# target laserscan 3D
if self.show_target is True:
self.back_view = vispy.scene.widgets.ViewBox(
border_color='white', parent=self.scan_canvas.scene)
self.back_vis = visuals.Markers()
self.back_view.camera = 'turntable'
self.back_view.camera.link(self.scan_view.camera)
self.back_view.add(self.back_vis)
visuals.XYZAxis(parent=self.back_view.scene)
self.grid_view.add_widget(self.back_view, 0, 1)
# self.grid_view.padding = 6
# Set height of images
h = 1
if self.show_range is True:
h += 1
if self.show_remissions is True:
h += 1
# source canvas 2D
source_canvas_title = 'Source ' + str(self.H[0]) + 'x' + str(self.W[0])
self.source_canvas = SceneCanvas(keys='interactive',
show=True,
title=source_canvas_title,
size=(self.W[0], h * self.H[0]))
self.source_canvas.events.key_press.connect(self.key_press)
self.source_view = self.source_canvas.central_widget.add_grid()
source_grid_idx = 0
# Add label image
if self.show_label:
self.img_view = vispy.scene.widgets.ViewBox(
border_color='white', parent=self.source_canvas.scene)
self.img_vis = visuals.Image(cmap='viridis')
self.img_view.add(self.img_vis)
self.source_view.add_widget(self.img_view, source_grid_idx, 0)
source_grid_idx += 1
# target canvas 2D
if self.show_target:
target_canvas_title = 'Target ' + str(self.H[1]) + 'x' + str(
self.W[1])
self.target_canvas = SceneCanvas(keys='interactive',
show=True,
title=target_canvas_title,
size=(self.W[1], h * self.H[1]))
self.target_canvas.events.key_press.connect(self.key_press)
self.target_view = self.target_canvas.central_widget.add_grid()
target_grid_idx = 0
# Add label image
if self.show_label:
self.test_view = vispy.scene.widgets.ViewBox(
border_color='white', parent=self.target_canvas.scene)
self.test_vis = visuals.Image(cmap='viridis')
self.test_view.add(self.test_vis)
self.target_view.add_widget(self.test_view, target_grid_idx, 0)
target_grid_idx += 1
if self.show_range:
# Add source range image
self.range_view_source = vispy.scene.widgets.ViewBox(
border_color='white', parent=self.source_canvas.scene)
self.range_image_source = visuals.Image()
self.range_view_source.add(self.range_image_source)
self.source_view.add_widget(self.range_view_source,
source_grid_idx, 0)
source_grid_idx += 1
if self.show_target:
self.range_view_target = vispy.scene.widgets.ViewBox(
border_color='white', parent=self.target_canvas.scene)
self.range_image_target = visuals.Image(cmap='viridis')
self.range_view_target.add(self.range_image_target)
self.target_view.add_widget(self.range_view_target,
target_grid_idx, 0)
target_grid_idx += 1
if self.show_remissions:
# Add source remissions image
self.remissions_view_source = vispy.scene.widgets.ViewBox(
border_color='white', parent=self.source_canvas.scene)
self.remissions_image_source = visuals.Image()
self.remissions_view_source.add(self.remissions_image_source)
self.source_view.add_widget(self.remissions_view_source,
source_grid_idx, 0)
source_grid_idx += 1
# Add target remissions image
if self.show_target:
self.remissions_view_target = vispy.scene.widgets.ViewBox(
border_color='white', parent=self.target_canvas.scene)
self.remissions_image_target = visuals.Image(cmap='viridis')
self.remissions_view_target.add(self.remissions_image_target)
self.target_view.add_widget(self.remissions_view_target,
target_grid_idx, 0)
target_grid_idx += 1
# 2D canvas for showing difference in range, labels and remissions
if self.show_diff:
self.diff_canvas = SceneCanvas(keys='interactive',
show=True,
title='Difference Range Image',
size=(self.W[1], self.H[1] * h))
self.diff_canvas.events.key_press.connect(self.key_press)
self.diff_view = self.diff_canvas.central_widget.add_grid()
grid_idx = 0
# Add label difference
if self.show_label:
self.diff_view_label = vispy.scene.widgets.ViewBox(
border_color='white', parent=self.diff_canvas.scene)
self.diff_image_label = visuals.Image(cmap='viridis')
self.diff_view_label.add(self.diff_image_label)
self.diff_view.add_widget(self.diff_view_label, grid_idx, 0)
grid_idx += 1
# Add range difference
if self.show_range:
self.diff_view_depth = vispy.scene.widgets.ViewBox(
border_color='white', parent=self.diff_canvas.scene)
self.diff_image_depth = visuals.Image()
self.diff_view_depth.add(self.diff_image_depth)
self.diff_view.add_widget(self.diff_view_depth, grid_idx, 0)
grid_idx += 1
# Add remissions difference
if self.show_remissions:
self.diff_view_remissions = vispy.scene.widgets.ViewBox(
border_color='white', parent=self.diff_canvas.scene)
self.diff_image_remissions = visuals.Image()
self.diff_view_remissions.add(self.diff_image_remissions)
self.diff_view.add_widget(self.diff_view_remissions, grid_idx,
0)
grid_idx += 1
if self.show_mesh:
self.mesh_view = vispy.scene.widgets.ViewBox(
border_color='white', parent=self.scan_canvas.scene)
self.mesh_vis = visuals.Mesh(shading=None)
self.mesh_view.camera = 'turntable'
self.mesh_view.camera.link(self.scan_view.camera)
self.mesh_view.add(self.mesh_vis)
visuals.XYZAxis(parent=self.mesh_view.scene)
self.grid_view.add_widget(self.mesh_view, 0, 2)
from vispy.scene import visuals
from vispy.visuals.filters import Alpha
import networkx
from utils.geometry import *
img = plt.imread('../source/my.jpg')
h, w = img.shape[:2]
depth = np.tile((np.arange(0, w)) / w, (1, h, 1))
canvas = scene.SceneCanvas(bgcolor='black', size=(w * 3, h * 3))
grid = canvas.central_widget.add_grid()
view = canvas.central_widget.add_view()
left_view = grid.add_view(name='left_view', border_color='yellow')
temp_mesh = visuals.Mesh()
canvas.scene.add(temp_mesh)
#grid.add(temp_mesh)
input()
view.camera = 'perspective'
view.camera.fov = 60
print(view.camera, type(view.camera))
tr = view.camera.transform
vertice = []
faces = []
colors = []
is_inside = lambda y, x: (y >= 0 and y < h and x >= 0 and x < w)
four_neighbor = lambda y, x: [(y + 1, x), (y, x - 1), (y - 1, x), (y, x + 1)]
