class Canvas(app.Canvas):
def __init__(self):
app.Canvas.__init__(self, 'Cube', keys='interactive',
size=(400, 400))
self.cube = CubeVisual((1.0, 0.5, 0.25), color='red',
edge_color='black')
self.quaternion = Quaternion()
# Create a TransformSystem that will tell the visual how to draw
self.cube_transform = transforms.AffineTransform()
self.cube_transform.scale((100, 100, 0.001))
self.cube_transform.translate((200, 200))
self.tr_sys = transforms.TransformSystem(self)
self.tr_sys.visual_to_document = self.cube_transform
self.show()
def on_draw(self, event):
gloo.set_viewport(0, 0, *self.physical_size)
gloo.clear('white')
self.tr_sys.auto_configure()
self.cube.draw(self.tr_sys)
def on_mouse_move(self, event):
if event.button == 1 and event.last_event is not None:
x0, y0 = event.last_event.pos
x1, y1 = event.pos
w, h = self.size
self.quaternion = (self.quaternion *
Quaternion(*_arcball(x0, y0, w, h)) *
Quaternion(*_arcball(x1, y1, w, h)))
self.cube_transform.matrix = self.quaternion.get_matrix()
self.cube_transform.scale((100, 100, 0.001))
self.cube_transform.translate((200, 200))
self.update()
class Canvas(app.Canvas):
def __init__(self):
app.Canvas.__init__(self, 'Cube', keys='interactive', size=(400, 400))
self.cube = BoxVisual(1.0, 0.5, 0.25, color='red', edge_color='black')
self.cube.transform = transforms.MatrixTransform()
self.cube.transform.scale((100, 100, 0.001))
self.cube.transform.translate((200, 200))
self.quaternion = Quaternion()
self.show()
def on_resize(self, event):
vp = (0, 0, self.physical_size[0], self.physical_size[1])
self.context.set_viewport(*vp)
self.cube.transforms.configure(canvas=self, viewport=vp)
def on_draw(self, event):
self.context.clear('white')
self.cube.draw()
def on_mouse_move(self, event):
if event.button == 1 and event.last_event is not None:
x0, y0 = event.last_event.pos
x1, y1 = event.pos
w, h = self.size
self.quaternion = (self.quaternion *
Quaternion(*_arcball(x0, y0, w, h)) *
Quaternion(*_arcball(x1, y1, w, h)))
self.cube.transform.matrix = self.quaternion.get_matrix()
self.cube.transform.scale((100, 100, 0.001))
self.cube.transform.translate((200, 200))
self.update()
class Canvas(app.Canvas):
def __init__(self):
app.Canvas.__init__(self, 'Cube', keys='interactive',
size=(400, 400))
self.cube = CubeVisual((1.0, 0.5, 0.25), color='red',
edge_color='black')
self.cube.transform = transforms.MatrixTransform()
self.cube.transform.scale((100, 100, 0.001))
self.cube.transform.translate((200, 200))
self.quaternion = Quaternion()
self.show()
def on_resize(self, event):
vp = (0, 0, self.physical_size[0], self.physical_size[1])
self.context.set_viewport(*vp)
self.cube.transforms.configure(canvas=self, viewport=vp)
def on_draw(self, event):
self.context.clear('white')
self.cube.draw()
def on_mouse_move(self, event):
if event.button == 1 and event.last_event is not None:
x0, y0 = event.last_event.pos
x1, y1 = event.pos
w, h = self.size
self.quaternion = (self.quaternion *
Quaternion(*_arcball(x0, y0, w, h)) *
Quaternion(*_arcball(x1, y1, w, h)))
self.cube.transform.matrix = self.quaternion.get_matrix()
self.cube.transform.scale((100, 100, 0.001))
self.cube.transform.translate((200, 200))
self.update()
class Canvas(app.Canvas):
def __init__(self):
app.Canvas.__init__(self, 'Cube', keys='interactive',
size=(400, 400))
self.cube = CubeVisual((1.0, 0.5, 0.25), color='red',
edge_color='black')
self.quaternion = Quaternion()
# Create a TransformSystem that will tell the visual how to draw
self.cube_transform = transforms.AffineTransform()
self.cube_transform.scale((100, 100, 0.001))
self.cube_transform.translate((200, 200))
self.tr_sys = transforms.TransformSystem(self)
self.tr_sys.visual_to_document = self.cube_transform
self.show()
def on_draw(self, event):
gloo.set_viewport(0, 0, *self.physical_size)
gloo.clear('white')
self.tr_sys.auto_configure()
self.cube.draw(self.tr_sys)
def on_mouse_move(self, event):
if event.button == 1 and event.last_event is not None:
x0, y0 = event.last_event.pos
x1, y1 = event.pos
w, h = self.size
self.quaternion = (self.quaternion *
Quaternion(*_arcball(x0, y0, w, h)) *
Quaternion(*_arcball(x1, y1, w, h)))
self.cube_transform.matrix = self.quaternion.get_matrix()
self.cube_transform.scale((100, 100, 0.001))
self.cube_transform.translate((200, 200))
self.update()
def transformCube(self, t, q):
quaternion = Quaternion(*q)
self._transform.reset()
self._transform.matrix = quaternion.get_matrix()
self._transform.translate(t)
x, y, z = t
self._xPos = np.append(self._xPos, x)
self._yPos = np.append(self._yPos, y)
self._zPos = np.append(self._zPos, z)
self._plot.set_data((self._xPos.transpose(), self._yPos.transpose(),
self._zPos.transpose()),
symbol=None)
class RGBDMultiView(app.Canvas):
""" """
def __init__(self, sensor_type="OrbbecAstra", sensor_scale=1.0):
app.Canvas.__init__(self, keys='interactive', size=(1280, 1024))
self.title = 'RGBDMultiView'
self.quaternion = Quaternion() # for mouse movement
self.cam_params = GetCameraParameters(sensor_type, sensor_scale)
self.fov = 30
ps = self.pixel_scale
self.playbackSpeed = 1
# view and model control variables
self.tshift = 0.05 # the amount to translate by
self.rshift = 0.05 # amount to increase rotation by
self.scaleFactor = 1000 # the amount to scale the input model by
self.default_camera_view()
self.view = np.dot(self.rotMat, self.transMat)
self.model = np.eye(4, dtype=np.float32)
self.projection = np.eye(4, dtype=np.float32)
self.GL_progs = {} # name : data for a single frame
self.apply_zoom()
gloo.set_state('translucent', clear_color='gray')
self.timer = app.Timer('auto', connect=self.on_timer, start=False)
def on_timer(self, event):
if self.nframe_curr < self.nframe_total - self.playbackSpeed: # if we still got images, keep loading them
self.nframe_curr += self.playbackSpeed
self.set_frame(self.nframe_curr)
else:
# no more images means we're at the end so we should stop the timer allowing the video to
# be reverse or restarted by through the spacebar
if self.timer.running: self.timer.stop()
self.update()
def on_mouse_move(self, event):
if event.button == 1 and event.last_event is not None:
x0, y0 = event.last_event.pos
x1, y1 = event.pos
w, h = self.size
self.quaternion = (self.quaternion *
Quaternion(*_arcball(x0, y0, w, h)) *
Quaternion(*_arcball(x1, y1, w, h)))
self.rotMat = self.quaternion.get_matrix()
self.view = np.dot(self.rotMat, self.transMat)
for k, prog in self.GL_progs.items():
prog['u_view'] = self.view
prog['u_model'] = self.model
self.update()
def default_camera_view(self):
""" Set the view matrics to their default values """
self.offX = 0.0 # actual amounts translated thus far
self.offY = -0.6
self.zoomAmount = 2.5
self.centerCoord = (0, 0, 0) # center of the current point cloud frame
self.rotX = 9.3
self.rotY = 6.3
self.rotZ = 0.0
self.transMat = translate((self.offX, self.offY, -self.zoomAmount))
self.rotMat = rotationMatrix((self.rotX, self.rotY, self.rotZ))
q = quaternion_from_matrix(self.rotMat) ## just for mouse arc ball
self.quaternion.w = q[0]
self.quaternion.x = q[1]
self.quaternion.y = q[2]
self.quaternion.z = q[3]
def print_camera_view(self):
print(self.offX, self.offY, self.zoomAmount)
print(self.rotX, self.rotY, self.rotZ)
print()
def on_key_press(self, event):
### pause and play
if event.text == ' ':
if self.timer.running: self.timer.stop()
else: self.timer.start()
### manually advance through the frames
elif event.text == ',' and self.nframe_curr >= self.playbackSpeed:
self.nframe_curr -= self.playbackSpeed
self.set_frame(self.nframe_curr)
self.update()
elif event.text == '.':
if self.nframe_curr < self.nframe_total - self.playbackSpeed:
self.nframe_curr += self.playbackSpeed
elif self.nframe_curr < self.nframe_total - 1:
self.nframe_curr += 1
self.set_frame(self.nframe_curr)
self.update()
### camera manipulation
if event.text in ['w', 'a', 's', 'd']: # planar translations
if event.text == 'w': self.offY += self.tshift
elif event.text == 'a': self.offX -= self.tshift
elif event.text == 's': self.offY -= self.tshift
elif event.text == 'd': self.offX += self.tshift
self.transMat = translate((self.offX, self.offY, -self.zoomAmount))
if event.text in ['q', 'e', 'z', 'c', 'r',
'v']: # rotation about the axis
if event.text == 'q': self.rotX += self.rshift
elif event.text == 'e': self.rotX -= self.rshift
elif event.text == 'z': self.rotY += self.rshift
elif event.text == 'c': self.rotY -= self.rshift
elif event.text == 'r': self.rotZ += self.rshift
elif event.text == 'v': self.rotZ -= self.rshift
self.rotMat = rotationMatrix((self.rotX, self.rotY, self.rotZ))
### reset camera to original default orientation
if event.text == 'x':
self.print_camera_view()
self.default_camera_view()
if event.text == 'p':
strTopFolder = "F:/data/deep-learning/additive_depth/98_vr_calib/"
strFolderName = "test01_img_seq"
strMultiviewFolder = os.path.join(strTopFolder, strFolderName)
strPosesFile = os.path.join(strMultiviewFolder, "poses.txt")
self.sensor_props = ReadManualCalibPoses(strPosesFile)
for s in sensor_props:
s.load_image_files(strMultiviewFolder)
self.view = np.dot(self.rotMat, self.transMat)
for k, prog in self.GL_progs.items():
prog['u_view'] = self.view
prog['u_model'] = self.model
self.update()
def on_resize(self, event):
self.apply_zoom()
def on_mouse_wheel(self, event):
self.zoomAmount -= event.delta[1] / 2.0
self.transMat = translate((self.offX, self.offY, -self.zoomAmount))
self.view = np.dot(self.rotMat, self.transMat)
for k, prog in self.GL_progs.items():
prog['u_view'] = self.view
self.update()
def on_draw(self, event):
gloo.clear()
for k, prog in self.GL_progs.items():
prog.draw('points')
def apply_zoom(self):
gloo.set_viewport(0, 0, self.physical_size[0], self.physical_size[1])
self.projection = perspective(self.fov,
self.size[0] / float(self.size[1]), 0.01,
10000.0)
for k, prog in self.GL_progs.items():
prog['u_projection'] = self.projection
def load_frame_data(self, frame_n):
'''
load_frame_data
Arguments:
frame_n: int obtain the frame number for the stream
Return:
frame_data: array contain three camera and contain [numVertice , xyz , rgb]
'''
frame_data = {}
for s in self.sensor_props:
depthImg = cv2.imread(s.imgs_depth[frame_n],
-1) ## load frames and compute point cloud
clrImg = cv2.imread(s.imgs_color[frame_n], -1)
ptcloud, nVertices = image_fusion(self.cam_params, depthImg,
clrImg)
transformedPtcloud = transform_pointcloud_vectorized(
ptcloud, s.rotationMatrix(), s.translationMatrix())
xyz, rgb = np.hsplit(transformedPtcloud, 2)
## TODO: extra post-processing to convert points close to the camera to a specific color
frame_data[s.name] = [nVertices, xyz, rgb]
return frame_data
def load_frame_to_stl(self, frame_n):
'''
load_frame_data
Arguments:
frame_n: int obtain the frame number for the stream
Return:
frame_data: array contain three camera and contain [numVertice , xyz , rgb]
'''
data = np.zeros(6, dtype=mesh.Mesh.dtype)
frame_data = {}
xyzpcd, rgbpcd = [], []
for i, s in enumerate(self.sensor_props):
depthImg = cv2.imread(s.imgs_depth[frame_n],
-1) ## load frames and compute point cloud
clrImg = cv2.imread(s.imgs_color[frame_n], -1)
ptcloud, nVertices = image_fusion(self.cam_params, depthImg,
clrImg)
transformedPtcloud = transform_pointcloud_vectorized(
ptcloud, s.rotationMatrix(), s.translationMatrix())
xyz, rgb = np.hsplit(transformedPtcloud, 2)
xyzpcd.insert(i, xyz)
rgbpcd.insert(i, rgb)
## TODO: extra post-processing to convert points close to the camera to a specific color
frame_data[s.name] = [nVertices, xyz, rgb]
rgbpcd = np.vstack((rgbpcd[0], rgbpcd[1], rgbpcd[2]))
xyzpcd = np.vstack((xyzpcd[0], xyzpcd[1], xyzpcd[2]))
print(rgbpcd.shape, xyzpcd.shape)
return xyzpcd, rgbpcd, frame_data
def set_frame(self, frame_number):
self.nframe_curr = frame_number
frame_data = self.load_frame_data(self.nframe_curr)
for k, prog in self.GL_progs.items():
nVertices, xyz, rgb = frame_data[k]
## load and bind the current frame data to the GL program
data = np.zeros(nVertices, [('a_position', np.float32, 3),
('a_bg_color', np.float32, 4),
('a_fg_color', np.float32, 4),
('a_size', np.float32, 1)])
data['a_position'] = np.array(xyz / self.scaleFactor)
data['a_bg_color'] = np.c_[
rgb / 255, np.ones(
nVertices
)] # make sure rgb is between [0,1] and gotta append an extra one to each row for the alpha value
data['a_fg_color'] = 0, 0, 0, 1
data['a_size'] = 4
prog['u_model'] = self.model
prog.bind(gloo.VertexBuffer(data))
self.update()
def set_multiview_source(self, sensor_props):
self.nframe_total = sys.maxsize
self.sensor_props = sensor_props
for s in self.sensor_props:
gl_prog = gloo.Program(VERTEX_SHADER, FRAGMENT_SHADER)
gl_prog['u_linewidth'] = 0
gl_prog['u_antialias'] = 0
gl_prog['u_model'] = self.model
gl_prog['u_view'] = self.view
gl_prog['u_size'] = 1
self.GL_progs[s.name] = gl_prog
self.nframe_total = min(self.nframe_total, len(s.imgs_color))
self.nframe_total = min(self.nframe_total, len(s.imgs_depth))
print(self.nframe_total)
self.apply_zoom()
self.set_frame(0)
class Canvas(app.Canvas):
visualization_modes = ['cpk', 'backbone', 'aminoacid', 'dssp']
def __init__(self, pdbdata, mode='cpk'):
#Startup
app.Canvas.__init__(self, keys='interactive', size=(W, H))
#Loading shaders
self.program = gloo.Program(vertex, fragment)
#Analyze pdb file
self.parser = PDBParser(QUIET=True, PERMISSIVE=True)
self.structure = self.parser.get_structure('model', pdbdata)
#Mode selection
if mode not in Canvas.visualization_modes:
raise Exception('Not recognized visualization mode %s' % mode)
self.mode = mode
#Get the data of our atoms
self.atom_information()
#Camera settings
self.translate = max(abs(np.concatenate(self.coordinates))) + 40
self.translate = max(-1, self.translate)
self.view = translate((0, 0, -self.translate), dtype=np.float32)
self.model = np.eye(4, dtype=np.float32)
self.projection = np.eye(4, dtype=np.float32)
self.program['u_projection'] = self.projection
self.quaternion = Quaternion()
#Load data depending on the mode
self.apply_zoom()
self.load_data()
#self.lines = visuals.LinePlotVisual(self.chain_coords[1], width= 5.0, marker_size= 0.0, color=self.chain_colors[1])
gloo.set_state(depth_test=True, clear_color='white')
self.show()
def atom_information(self):
"""Determines the coordinates, colors and sizes of the atoms depending on the mode"""
if self.mode == 'cpk':
#list of atoms
self.atoms = [atom for atom in self.structure.get_atoms()]
self.natoms = len(self.atoms)
#atom coordinates
self.coordinates = np.array([atom.coord for atom in self.atoms])
self.center = centroid(self.coordinates)
self.coordinates -= self.center
#atom color
self.color = [
np.array(colorrgba(atom.get_id())[0:3]) for atom in self.atoms
]
#atom radius
self.radius = [vrad(atom.get_id()) for atom in self.atoms]
elif self.mode == 'aminoacid':
#list of atoms
self.atoms = [
atom for atom in self.structure.get_atoms()
if atom.get_parent().resname != 'HOH'
]
self.natoms = len(self.atoms)
#atom coordinates
self.coordinates = np.array([atom.coord for atom in self.atoms])
self.center = centroid(self.coordinates)
self.coordinates -= self.center
#atom color
self.color = [
colorrgba(restype(atom.get_parent().resname))[0:3]
for atom in self.atoms
]
#atom radius
self.radius = [vrad(atom.get_id()) for atom in self.atoms]
elif self.mode == 'backbone':
#list of atoms
self.atoms = [
atom for atom in self.structure.get_atoms()
if atom.get_name() == 'CA' or atom.get_name() == 'N'
]
self.natoms = len(self.atoms)
#atom coordinates
self.coordinates = np.array([atom.coord for atom in self.atoms])
self.center = centroid(self.coordinates)
self.coordinates -= self.center
#atom color
self.color = []
self.chains = []
self.chain_coords = []
self.chain_colors = []
for chain in self.structure.get_chains():
self.chains.append(chain)
self.can_coord = np.array([
atom.coord for atom in chain.get_atoms()
if atom.get_name() == 'CA' or atom.get_name() == 'N'
])
self.can_coord -= self.center
self.chain_coords.append(self.can_coord)
self.chain_length = len(self.can_coord)
self.chain_color = np.random.rand(1, 3)
self.chain_colors.append(self.chain_color)
self.color.append(
np.tile(self.chain_color, (self.chain_length, 1)))
if len(self.chains) > 1:
self.color = np.concatenate(self.color)
#atom radius
self.radius = [vrad(atom.get_id()) for atom in self.atoms]
def load_data(self):
"""Make an array with all the data and load it into VisPy Gloo"""
data = np.zeros(self.natoms, [('a_position', np.float32, 3),
('a_color', np.float32, 3),
('a_radius', np.float32, 1)])
data['a_position'] = self.coordinates
data['a_color'] = self.color
data['a_radius'] = self.radius #*self.pixel_scale
self.program.bind(gloo.VertexBuffer(data))
self.program['u_model'] = self.model
self.program['u_view'] = self.view
self.program['u_light_position'] = 0., 0., 2.
self.program['u_light_spec_position'] = -5., 5., -5.
print 'Data loaded'
def on_resize(self, event):
width, height = event.size
def apply_zoom(self):
"""Determine the projection of the canvas"""
width, height = self.physical_size
gloo.set_viewport(0, 0, width, height)
self.projection = perspective(95.0, width / float(height), 1.0, 400.0)
self.program['u_projection'] = self.projection
def on_draw(self, event):
gloo.clear()
self.program.draw('points')
#draw separated chains
#if self.mode in ['backbone','dssp']:
#for i in range(len(self.chains)):
#self.lines = visuals.LinePlotVisual(self.chain_coords[i], width= 5.0, marker_size= 0.0, color=self.chain_colors[i])
#self.lines.draw()
def on_mouse_move(self, event):
#Mouse rotation
if event.button == 1 and event.last_event is not None:
x0, y0 = event.last_event.pos
x1, y1 = event.pos
w, h = self.size
self.quaternion = (self.quaternion *
Quaternion(*_arcball(x0, y0, w, h)) *
Quaternion(*_arcball(x1, y1, w, h)))
self.model = self.quaternion.get_matrix()
self.program['u_model'] = self.model
self.update()
#Mouse zoom
elif event.button == 2 and event.last_event is not None:
x0, y0 = event.last_event.pos
x1, y1 = event.pos
self.translate += (y1 - y0)
self.translate = max(-1, self.translate)
self.view = translate((0, 0, -self.translate), dtype=np.float32)
self.program['u_view'] = self.view
self.update()
def on_mouse_wheel(self, event):
self.translate -= event.delta[1]
self.translate = max(-1, self.translate)
self.view = translate((0, 0, -self.translate))
self.program['u_view'] = self.view
self.update()
class InteractiveViewportMixin(object):
""" Canvas mixin for interactive rotation, translation and zooming
This class implements on_resize, on_mouse_move, on_mouse_press,
on_mouse_release and on_mouse_wheel in such a way that the view matrix and
the projection matrix changes in response to user input.
The derived class should implement on_update_view and on_update_projection
to update the matrices in the shader.
"""
def __init__(self, rview, zview=None, zoom_speed=0.1, move_speed=0.01):
""" Constructor
Parameters
----------
rview : float
Radius of inscribed sphere within the field of view.
zview : float
Distance to the z-clip plane from the origin.
zoom_speed : float
How sensitive the field of view to mouse wheel rolling.
move_speed : float
How sensitive the translation of view to mouse dragging.
"""
self.rview = rview
self.zview = zview if zview else rview
self.zoom_speed = zoom_speed
self.move_speed = move_speed
self._translation = (0.0, 0.0, 0.0)
self._quaternion = Quaternion()
def on_update_view(self, view):
""" Handle view matrix change
"""
pass
def on_update_projection(self, proj):
""" Handle projection matrix change
"""
pass
def get_view_matrix(self):
""" Return current view matrix
"""
return np.dot(self._quaternion.get_matrix(),
transforms.translate(self._translation))
def get_projection_matrix(self):
""" Return current projection matrix
"""
# Determine the field of view of orthographic projection so that the
# sphere of radius rview centered at the origin inscribes the field
# of view.
width, height = self.physical_size
aspect = float(width) / height
if aspect > 1:
top = self.rview
right = self.rview * aspect
else:
top = self.rview / aspect
right = self.rview
assert min(top, right) == self.rview
left = -right
bottom = -top
near = self.zview
far = -self.zview
left -= 1e-6
right += 1e-6
bottom -= 1e-6
top += 1e-6
near -= 1e-6
far += 1e-6
return transforms.ortho(left, right, bottom, top, near, far)
def _update_view(self):
self.on_update_view(self.get_view_matrix())
def _update_projection(self):
self.on_update_projection(self.get_projection_matrix())
def on_resize(self, event):
gloo.set_viewport(0, 0, *event.physical_size)
self._update_projection()
def on_mouse_move(self, event):
""" Left drag to rotate, right drag to tranelate
"""
if event.is_dragging and event.last_event:
x0, y0 = event.last_event.pos
x1, y1 = event.pos
w, h = self.size
if event.button == 1:
self._quaternion *= arcball(x0, y0, w, h) * arcball(
x1, y1, w, h)
self._update_view()
if event.button == 2:
tx, ty, tz = self._translation
tx += self.move_speed * (x1 - x0)
ty += -self.move_speed * (y1 - y0)
self._translation = (tx, ty, tz)
self._update_view()
def on_mouse_wheel(self, event):
""" Roll mouse wheel to zoom in/out
"""
roll = event.delta[1]
self.rview *= math.exp(-self.zoom_speed * roll)
self._update_projection()
def on_mouse_press(self, event):
if event.button == 1:
cursor = QtGui.QCursor(QtCore.Qt.ClosedHandCursor)
QtGui.QApplication.setOverrideCursor(cursor)
if event.button == 2:
cursor = QtGui.QCursor(QtCore.Qt.SizeAllCursor)
QtGui.QApplication.setOverrideCursor(cursor)
def on_mouse_release(self, event):
QtGui.QApplication.restoreOverrideCursor()
