def getActiveCameraIndex():
idx = None
try:
idx = QApp.view().cameras.index(QApp.view().camera) - 1
except Exception as e:
renderLogger.error('Unable to determine active camera index: %s',
str(e))
return idx
def process_frame(deinterlacing, detectingWands, frame, opts, pair):
ci, md = pair
img = get_movie_frame(md, frame, deinterlacing)
#data = filter_movie_frame(img, small_blur, large_blur)
#img, data = get_processed_movie_frame(md, frame, small_blur, large_blur, deinterlacing)
QApp.view().cameras[ci + 1].invalidateImageData()
"""
if 1: # show the filtered image
img[:] = data
pass
if 0: # crush the image to see the blobs
lookup = np.zeros(256, dtype=np.uint8)
lookup[threshold_bright:] = 255
lookup[255 - threshold_dark_inv:threshold_bright] = 128
img[:] = lookup[img]
"""
if 1:
good_darks, pts0, good_lights, pts1, data = get_dark_and_light_points(
img, frame, ci, opts)
if 1: # show the filtered image
#print "data before insertion", type(data), data.shape
#sys.exit(0)
img[:] = data
if 0: # crush the image to see the blobs
lookup = np.zeros(256, dtype=np.uint8)
lookup[threshold_bright:] = 255
lookup[255 - threshold_dark_inv:threshold_bright] = 128
img[:] = lookup[img]
# good_darks, pts0 = Detect.detect_dots(255-data, opts['threshold_dark_inv'], opts)
# good_lights,pts1 = Detect.detect_dots(data, opts['threshold_bright'], opts)
print ci, frame, len(pts0), len(pts1), 'good points (darks,lights)'
if detectingWands:
ratio = 2.0
x2d_threshold = 0.5
straightness_threshold = 0.01 * 2
match_threshold = 0.07 * 2
x2ds_labels = -np.ones(pts1.shape[0], dtype=np.int32)
x2ds_splits = np.array([0, pts1.shape[0]], dtype=np.int32)
ISCV.label_T_wand(pts1, x2ds_splits, x2ds_labels, ratio,
x2d_threshold, straightness_threshold,
match_threshold)
print x2ds_labels
for r, li in zip(good_lights, x2ds_labels):
if li != -1: # make some red boxes
dx, dy = 10, 10
img[int(r.sy - dy):int(r.sy + dy),
int(r.sx - dx):int(r.sx + dx), 0] = 128
else:
pts0 = pts1 = []
return (pts0, pts1)
def set_frame_cb(fi):
view = QApp.view() # TODO should be part of the callback?
if fi != frame_number():
State.setKey('/frame_number', fi)
State.push('change frame')
img = get_frame_image(fi)
update_gui_image(view, img)
update_markup_mesh(view)
QApp.app.updateMenus() # trigger a full refesh here (TODO not needed?)
def set_frame_cb(frame):
global md
readFrame(md, seekFrame=frame)
img = np.frombuffer(md['vbuffer'],
dtype=np.uint8).reshape(md['vheight'],
md['vwidth'], 3)
image_mesh = QApp.app.getLayer('image_mesh')
image_mesh.setImage(img)
view = QApp.view()
view.refreshImageData()
view.updateGL()
def set_frame_cb(frame):
global g_aam_model, g_images, g_shapes, g_predictor, template_vs
indx = frame/4
which = frame%4
img = np.zeros((160,160,3),dtype=np.uint8)
if which == 0:
img = g_images[indx].copy()
shp = g_shapes[indx].copy()
tex = textures[indx]
elif which == 1:
img = g_images[indx].copy()
shp = g_shapes[indx].copy()
tex = textures[indx]
shp = Face.track_face(img, g_predictor, shp)
elif which == 2:
shp = g_aam_model['shapes'][indx]
tex = g_aam_model['textures'][indx]
else:
shp = g_aam_model['ref_shape'] + np.dot(g_aam_model['shapes_u'][indx]*g_aam_model['shapes_s'],\
g_aam_model['shapes_vt']).reshape(-1,2)
tex = g_aam_model['texture_mean'] + np.dot(g_aam_model['texture_u'][indx]*g_aam_model['texture_s'],\
g_aam_model['texture_vt']).reshape(-1,3)
width,height = img.shape[1],img.shape[0]
w,h = width*0.5,height*0.5
if which >= 2:
shp = Face.normalize_shape(shp, model['ref_pinv'])
shp = (shp + 1) * [w,h]
np.clip(tex,0,255,out=tex)
Face.render_texture(tex, img, shp, model['model_indices'], model['model_weights'])
geo_mesh = QApp.app.getLayer('geo_mesh')
size = 68+4
vs = np.zeros((size,3),dtype=np.float32)
vs[:size-4,:2] = shp
vs[size-4:size,:2] = Face.get_boundary(shp, template_vs)
geo_mesh.setVs(vs)
geo_mesh.transforms[0][:,:3] = [[1,0,0],[0,1,0],[0,0,1],[-w,1000-h,0.1]]
image_mesh = QApp.app.getLayer('image_mesh')
image_mesh.setVs(np.array([[-w,-h,0],[w,-h,0],[w,h,0],[-w,h,0]], dtype=np.float32))
image_mesh.setImage(img)
QApp.view().updateGL()
def dirtyCB(dirty):
# triggered by a state change, this allows the app to synch with the state
# TODO this is the correct place to deal with changes due to eg menu items or hot keys
# should be able to remove eg updateGL from everywhere else, really
#print 'dirty',dirty
if dirty:
outliner = QApp.app.qoutliner
outliner.set_root(outliner.root) # TODO this causes a total rebuild
global g_predictor, g_aam_model
if '/predictor' in dirty: g_predictor = None
if '/aam' in dirty: g_aam_model = None
if '/vnames' in dirty:
#print 'setting names',State.getKey('/vnames')
QApp.view().getLayer('markup_mesh').names = State.getKey('/vnames', [])
if '/markup_mesh_sel' in dirty:
QApp.view().getLayer('markup_mesh').selectedIndex = State.getKey(
'/markup_mesh_sel', -1)
if '/edges' in dirty:
QApp.view().getLayer('markup_mesh').edges = State.getKey(
'/edges', None)
QApp.view().getLayer('ref_mesh').edges = State.getKey('/edges', None)
def main():
# Program options
global actors
TransmitMulticast = False
HostName = "localhost:801"
import sys
if (len(sys.argv) > 1):
HostName = sys.argv[1]
print 'Setting hostname to ',HostName
from UI import QApp, QActorWidget, GLGrid, GLPoints3D, GLSkeleton
from PySide import QtCore, QtGui, QtOpenGL
import numpy as np
global app,win,view
app = QtGui.QApplication(sys.argv)
app.setStyle('plastique')
win = QApp.QApp()
win.setWindowTitle('Imaginarium ViconConnect')
view = win.view()
view.setMinimumWidth(640)
view.setMinimumHeight(480)
win.setCentralWidget(view)
actorsDock = QtGui.QDockWidget('Actors')
actors = QActorWidget.QActorWidget(setActorVisible)
area = QtGui.QScrollArea()
area.setMinimumWidth(200)
area.setWidgetResizable(True)
area.setWidget(actors)
actorsDock.setWidget(area)
actorsDock.setFeatures(QtGui.QDockWidget.DockWidgetMovable|QtGui.QDockWidget.DockWidgetFloatable)
win.addDockWidget(QtCore.Qt.RightDockWidgetArea, actorsDock)
win.show()
global unlabelledMarkers
unlabelledMarkers = GLPoints3D([])
view.primitives.append(unlabelledMarkers)
global MyClient, skels, points
skels = {}
points = {}
view.primitives.append(GLGrid())
timer = QtCore.QTimer(app)
app.connect(timer, QtCore.SIGNAL('timeout()'), nextFrame)
timer.start(20)
MyClient = viconConnect(HostName, TransmitMulticast)
app.connect(app, QtCore.SIGNAL('lastWindowClosed()') , app.quit)
sys.exit(app.exec_())
def main():
appIn = QtGui.QApplication(sys.argv)
appIn.setStyle('plastique')
global win
win = QApp.QApp()
win.setWindowTitle('Imaginarium MoCap Software')
win.updateMenus()
Runtime.addOpMenu(win)
win.dirtyCB = dirtyCB
Runtime.setCallbacks(win)
RenderCallback.Factory(win, Runtime)
appIn.connect(appIn, QtCore.SIGNAL('lastWindowClosed()'), appIn.quit)
QGLViewer.makeViewer(timeRange=(0, 1400),
callback=setFrame,
appIn=appIn,
win=win,
pickCallback=Runtime.picked)
def main():
# Program options
hostName = "localhost:18667"
if (len(sys.argv) > 1):
hostName = sys.argv[1]
print 'Setting hostName to ', hostName
global app, win, view, state, stateHash, actors
app = QtGui.QApplication(sys.argv)
app.setStyle('plastique')
win = QApp.QApp()
win.setWindowTitle('Imaginarium Client')
view = win.view()
win.resize(640, 480)
actorsDock = QtGui.QDockWidget('Actors')
actors = QActorWidget.QActorWidget(setActorVisible)
area = QtGui.QScrollArea()
area.setMinimumWidth(200)
area.setWidgetResizable(True)
area.setWidget(actors)
actorsDock.setWidget(area)
actorsDock.setFeatures(QtGui.QDockWidget.DockWidgetMovable
| QtGui.QDockWidget.DockWidgetFloatable)
win.addDockWidget(QtCore.Qt.RightDockWidgetArea, actorsDock)
win.show()
global unlabelledMarkers
unlabelledMarkers = GLPoints3D([])
view.primitives.append(unlabelledMarkers)
global skels, points
skels = {}
points = {}
view.primitives.append(GLGrid())
timer = QtCore.QTimer(app)
app.connect(timer, QtCore.SIGNAL('timeout()'), nextFrame)
timer.start(20)
global client
client = ReframeClient('localhost', 18667)
app.connect(app, QtCore.SIGNAL('lastWindowClosed()'), app.quit)
sys.exit(app.exec_())
def intersectRaysCB(fi, raw_frames, mats, primitives, primitives2D, track3d):
global prev_frame
skipping = prev_frame is None or abs(fi - prev_frame) > 10
prev_frame = fi
view = QApp.view()
points, altpoints = primitives
g2d = primitives2D[0]
frame = raw_frames[fi]
# frame = x2d_frames[fi]
x2ds_data, x2ds_splits = GiantReader.frameCentroidsToDets(frame, None)
g2d.setData(x2ds_data, x2ds_splits)
if skipping:
x3ds, x3ds_labels = track3d.boot(x2ds_data, x2ds_splits)
else:
x3ds, x3ds_labels = track3d.push(x2ds_data, x2ds_splits)
points.setData(x3ds)
view.updateGL()
def rbfn_view_cb(fi, attrs):
# g_mode = 1
global g_rbfn
group,gn,pn,slider_indices,slider_names,pose_splits = rbfn_info_from_frame(fi)
QApp.view().displayText = [(10, 100, gn), (10,125, pn)]
img = group['images'][pn]
img = JPEG.decompress(img)
h,wm = img.shape[0]*0.5,img.shape[1]*0.5
out_shape = extract_x2ds(group, pn, g_rbfn['marker_names'])
svs = group['slider_data'][pn][slider_indices]
State._setKey('/root/sliders/attrs', dict(zip(slider_names, svs))) # NO UNDO
# compensate for roll, translation and scale
norm_shape, head_pan, head_tilt, A = stabilize_shape(out_shape)
# extract angles from the measured values
mirror_scale = -1 if attrs['mirroring'] else 1
new_pose = np.degrees(np.arctan2([head_pan*mirror_scale, head_tilt, -mirror_scale*A[1][0]],[2,2,A[1][1]]))
head_roll = -np.arctan2(A[1][0],A[1][1])
head_pan = np.arctan2(head_pan, 2.0)
head_tilt = np.arctan2(head_tilt, 2.0)
#print head_roll, head_pan, head_tilt
slider_names, slider_values = applyRetarget(g_rbfn, norm_shape)
svs[np.where(svs < 1e-4)] = 0
slider_values[np.where(slider_values < 1e-4)] = 0
#print zip(slider_values,svs)
slider_names.extend(['NeckRoll','NeckPan','NeckTilt'])
svs = np.clip(svs,0,1)
slider_values = np.float32(list(svs)+list(np.degrees([head_roll,head_pan,head_tilt])))
return new_pose,out_shape,norm_shape,img,slider_names,slider_values,A
p = view.primitives[pn]
if isinstance(p,GLMeshes):
name = p.names[pi]
print "Picked:", name
QApp.app.select(name)
elif isinstance(p,GLTool):
p.setComponent(pi)
QApp.app.updateGL()
from UI import GLMeshes # TODO move this colin
from UI import QApp
from GCore import State
from PySide import QtGui
import sys
appIn = QtGui.QApplication(sys.argv)
appIn.setStyle('plastique')
win = QApp.QApp()
win.setWindowTitle('Imaginarium Maya File Browser')
from UI import QGLViewer
primitives = []
#QApp.fields = fields
#outliner = QApp.app.qobjects
#for i,(o,v) in enumerate(dobjs):
# outliner.insertItem(i,o)
# outliner.item(i).setData(1,'_OBJ_'+v['name'])
vs = np.array([[[-1,-1,-1],[-1,-1,1],[-1,1,-1],[-1,1,1],[1,-1,-1],[1,-1,1],[1,1,-1],[1,1,1]]],dtype=np.float32)*100.0
faces = np.array([[[0,1,3,2],[4,6,7,5],[2,3,7,6],[1,0,4,5],[3,1,5,7],[0,2,6,4]]],dtype=np.int32)
transforms = [np.array([[1,0,0,0],[0,1,0,1000],[0,0,1,0]],dtype=np.float32)]
p = GLMeshes(names=['my mesh'], verts=vs, faces=faces, transforms=transforms)
q = GLTool()
q.setTool('3D rotation', transform = transforms[0], component=None)
r = GLTool()
def drawCameras(win, locationName, attrs, interface, picked):
from UI.QGLViewer import Camera
from UI import GLCameras
# Get camera ids. That's all we need if we are just updating
if 'camera_ids' not in attrs:
print 'Error rendering cameras: No camera_ids found.'
return
camera_ids = attrs['camera_ids']
if camera_ids is None: return
# Create cameras if we don't have any at this point
# Get all the gobbin from the cameras
if 'mats' not in attrs: return
mats = attrs['mats']
if mats is None: return
if 'camera_names' in attrs:
camera_names = attrs['camera_names']
else:
camera_names = [str(n) for n in range(len(camera_ids))]
# Check if we've got some image data
imageData = False
imgs = None
if 'imgs' in attrs: imgs = attrs['imgs']
if imgs is not None:
vheights, vwidths = None, None
if 'vheight' in attrs: vheights = attrs['vheight']
if 'vwidth' in attrs: vwidths = attrs['vwidth']
if vheights is not None and vwidths is not None:
if len(imgs) == len(vheights) == len(vwidths):
imageData = True
else:
renderLogger.error(
'Image data not consistent: #imgs[%d] #vheights[%d] #vwidths[%d]'
% (len(imgs), len(vheights), len(vwidths)))
else:
renderLogger.error('vheights and vwidths for images not found')
layerExists = win.view().hasLayer(locationName)
if layerExists:
camsLayer = win.view().getLayer(locationName)
if 'colours' in attrs and attrs['colours'] is not None:
camsLayer.colours = attrs['colours']
else:
camsLayer.colours = None
# TODO : Sort out when images don't exist, so vheights and vwidths don't either
updateMats = 'updateMats' in attrs and attrs['updateMats']
if imgs is not None and layerExists and not updateMats:
if 'updateImage' in attrs and attrs['updateImage']:
for ci, img, h, w in zip(camera_ids, imgs, vheights, vwidths):
#cam = QApp.view().cameras[ci + 1]
cam = QApp.view().cameras[ci + 1]
cam.setImageData(img, h, w, 3)
return
distOverride = {}
if 'distortion' in attrs: distOverride = attrs['distortion']
# Go through what we've gathered and create the cameras (with image data if present)
for ci, (mat, cid, cname) in enumerate(zip(mats, camera_ids,
camera_names)):
P, distortion = mat[2], mat[3]
if ci in distOverride: distortion = distOverride[ci]
if cid in cameraMap and updateMats:
camera = QApp.view().cameras[ci + 1]
camera.setP(P, distortion=distortion, store=True)
elif cid not in cameraMap:
cameraName = "%s | %s" % (cname, cid)
camera = Camera(cameraName)
camera.setP(P, distortion=distortion, store=True)
if imageData and ci < len(imgs):
img, vheight, vwidth = imgs[ci], vheights[ci], vwidths[ci]
camera.setImageData(img, vheight, vwidth, 3)
win.view().addCamera(camera)
cameraMap[cid] = cname
cams = GLCameras(camera_ids, mats)
if 'colour' in attrs and attrs['colour'] is not None:
cams.colour = attrs['colour']
if 'colours' in attrs and attrs['colours'] is not None:
cams.colours = attrs['colours']
win.setLayer(locationName, cams)
def cb(frame):
global g_record, g_frame
g_frame = frame
global g_camera_rays, g_camera_mat
#print 'in cb'
img = freenect.sync_get_video()[0]
geom_mesh = QApp.app.getLayer('geom_mesh')
geom_mesh.setImage(img)
if 1:
depths = freenect.sync_get_depth(format=freenect.DEPTH_REGISTERED)[0]
#print 'depths',np.median(depths)
if 0: # recording
if frame not in g_record: return
img, depths = g_record[frame]['video'], g_record[frame]['depths']
g_record[frame] = {'video': img.copy(), 'depths': depths.copy()}
if frame == 99: IO.save('dump', g_record)
depths_sum = np.array(depths != 0, dtype=np.int32)
lookup = np.array([0, 1, 0.5, 1.0 / 3, 0.25], dtype=np.float32)
if 1: # average
depths_lo = np.array(depths[::2, ::2] + depths[1::2, ::2] +
depths[::2, 1::2] + depths[1::2, 1::2],
dtype=np.float32)
depths_lo = depths_lo * lookup[
(depths_sum[::2, ::2] + depths_sum[1::2, ::2] +
depths_sum[::2, 1::2] +
depths_sum[1::2, 1::2]).reshape(-1)].reshape(depths_lo.shape)
else: # fullsize
depths_lo = depths * lookup[depths_sum.reshape(-1)].reshape(
depths_lo.shape)
K, RT, P, ks, T, wh = g_camera_mat
vs = depths_to_points(g_camera_rays, T, depths_lo)
geom_mesh.setVs(vs.reshape(-1, 3))
#QApp.view().setImage(img, img.shape[0], img.shape[1], img.shape[2])
#camera = QApp.view().camera
#geom_mesh.image = camera.image
#geom_mesh.bindImage = camera.bindImage
#geom_mesh.bindId = camera.bindId
global g_predictor, reference_3d, geo_vs, geo_vts
h, w, _3 = img.shape
global g_prev_vs
try:
g_prev_vs
except:
g_prev_vs = None
use_prev_vs = True
if g_prev_vs is None:
reference_3d[:, :2] = g_predictor['ref_shape'] * [100, 100]
tmp = Face.detect_face(img,
g_predictor) if g_prev_vs is None else g_prev_vs
tmp = Face.track_face(img, g_predictor, tmp)
if use_prev_vs: g_prev_vs = tmp
if frame == 0 or Face.test_reboot(img, g_prev_vs): g_prev_vs = None
geo_vts[:len(tmp)] = tmp
geo_vts[:, 1] = img.shape[0] - geo_vts[:, 1]
current_shape = geo_vts[:len(tmp)].copy()
if 1:
ds = extract_depths(vs, current_shape * 0.5)
M, inliers = Calibrate.rigid_align_points_inliers(ds,
reference_3d,
scale=True,
threshold_ratio=5.0)
ds = np.dot(ds, M[:3, :3].T) + M[:, 3]
which = np.where(np.sum((reference_3d - ds)**2, axis=1) < 100 * 100)[0]
reference_3d[which] = reference_3d[which] * 0.99 + ds[which] * 0.01
reference_3d[
inliers] = reference_3d[inliers] * 0.95 + ds[inliers] * 0.05
ds[:] = reference_3d[:]
M[1, 3] += 1000
M[0, 3] -= 300
else:
M = np.eye(3, 4, dtype=np.float32)
M[1, 3] += 1000
geom_mesh.setPose(M.reshape(1, 3, 4))
ref_pinv = g_predictor['ref_pinv']
xform = np.dot(ref_pinv, current_shape)
ut, s, v = np.linalg.svd(xform)
s = (s[0] * s[1])**-0.5
xform_inv = np.dot(v.T, ut.T) * s
current_shape = np.dot(current_shape - np.mean(current_shape, axis=0),
xform_inv) * 100.
geo_vs[:] = 0
geo_vs[:len(current_shape), :2] = current_shape
geo_vs[:70] = reference_3d
#geo_vs[:68,:] += [0,100,5500]
#print geo_vts[:4],w,h
geo_mesh = QApp.app.getLayer('geo_mesh')
geo_mesh.setVs(geo_vs,
vts=geo_vts *
np.array([1.0 / w, 1.0 / h], dtype=np.float32))
geo_mesh.setImage(img)
#geo_mesh.transforms[0][:,:3] = [[1,0,0],[0,1,0],[0,0,1],[0,1000,0.1]]
if 1:
global g_model
w, h = 160, 160
shp = geo_vs[:68, :2]
shape_u, tex_u, A_inv, mn = Face.fit_aam(g_model, tmp, img)
Face.render_aam(g_model, A_inv * 0.5, mn * 0.5, shape_u, tex_u, img)
img_mesh = QApp.app.getLayer('img_mesh')
img_mesh.setImage(img)
QApp.view().updateGL()
def main():
global State, mats, movieFilenames, primitives
global movies, primitives2D, deinterlacing, detectingWands
import IO
import sys, os
deinterlacing = False
detectingWands = False
detectingTiara = False
dot_detections = None
detections_filename = None
frame_offsets = None
firstFrame, lastFrame = 0, 5000
drawDotSize = 4.0
fovX, (ox,
oy), pan_tilt_roll, tx_ty_tz, distortion = 50., (0,
0), (0, 0,
0), (0, 1250,
0), (0,
0)
mats = []
grip_directory = os.environ['GRIP_DATA']
if 0:
fovX, (ox, oy), pan_tilt_roll, tx_ty_tz, distortion = 37.9, (0, 0), (
-66.0, 3.5, -0.2), (4850, 1330, 3280), (0, 0) # roughed in
K, RT = Calibrate.composeK(fovX, ox, oy), Calibrate.composeRT(
Calibrate.composeR(pan_tilt_roll), tx_ty_tz, 0)
mat0 = [
K[:3, :3], RT[:3, :4],
np.dot(K, RT)[:3, :], distortion, -np.dot(RT[:3, :3].T, RT[:3, 3]),
[1920, 1080]
]
fovX, (ox, oy), pan_tilt_roll, tx_ty_tz, distortion = 55.8, (0, 0), (
-103.6, 3.5, -0.3), (2980, 1380, -2180), (0, 0) # roughed in
K, RT = Calibrate.composeK(fovX, ox, oy), Calibrate.composeRT(
Calibrate.composeR(pan_tilt_roll), tx_ty_tz, 0)
mat1 = [
K[:3, :3], RT[:3, :4],
np.dot(K, RT)[:3, :], distortion, -np.dot(RT[:3, :3].T, RT[:3, 3]),
[1920, 1080]
]
fovX, (ox, oy), pan_tilt_roll, tx_ty_tz, distortion = 49.3, (0, 0), (
27.9, 4.0, -0.2), (-5340, 1150, 5030), (0, 0) # roughed in
K, RT = Calibrate.composeK(fovX, ox, oy), Calibrate.composeRT(
Calibrate.composeR(pan_tilt_roll), tx_ty_tz, 0)
mat2 = [
K[:3, :3], RT[:3, :4],
np.dot(K, RT)[:3, :], distortion, -np.dot(RT[:3, :3].T, RT[:3, 3]),
[1920, 1080]
]
fovX, (ox, oy), pan_tilt_roll, tx_ty_tz, distortion = 50.6, (0, 0), (
-156.6, 4.9, 0.2), (-105, 1400, -4430), (0, 0) # roughed in
K, RT = Calibrate.composeK(fovX, ox, oy), Calibrate.composeRT(
Calibrate.composeR(pan_tilt_roll), tx_ty_tz, 0)
mat3 = [
K[:3, :3], RT[:3, :4],
np.dot(K, RT)[:3, :], distortion, -np.dot(RT[:3, :3].T, RT[:3, 3]),
[1920, 1080]
]
mats = [mat0, mat1, mat2, mat3]
xcp_filename = '154535_Cal168_Floor_Final.xcp'
directory = os.path.join(grip_directory, 'REFRAME')
movieFilenames = [
'001E0827_01.MP4', '001F0813_01.MP4', '001G0922_01.MP4',
'001H0191_01.MP4'
]
#mats,movieFilenames = mats[:1],movieFilenames[:1] # restrict to single-view
frame_offsets = [119 + 160, 260, 339, 161]
small_blur, large_blur = 1, 25
min_dot_size = 1.0
max_dot_size = 20.0
circularity_threshold = 3.0
threshold_bright, threshold_dark_inv = 250, 250 #135,135
elif 0:
xcp_filename = '201401211653-4Pico-32_Quad_Dialogue_01_Col_wip_01.xcp'
detections_filename = 'detections.dat'
detectingTiara = True
pan_tilt_roll = (0, 0, 90)
distortion = (0.291979, 0.228389)
directory = os.path.join(os.environ['GRIP_DATA'], 'ted')
movieFilenames = [
'201401211653-4Pico-32_Quad_Dialogue_01_%d.mpg' % xi
for xi in range(1)
]
firstFrame = 511
small_blur, large_blur = 1, 20
min_dot_size = 1.0
max_dot_size = 16.0
circularity_threshold = 3.0
threshold_bright, threshold_dark_inv = 0, 170
elif 1:
xcp_filename = '50_Grip_RoomCont_AA_02.xcp'
detections_filename = 'detections.dat'
pan_tilt_roll = (0, 0, 0)
distortion = (0.291979, 0.228389)
directory = os.path.join(os.environ['GRIP_DATA'], '151110')
movieFilenames = ['50_Grip_RoomCont_AA_02.v2.mov']
firstFrame = 0
small_blur, large_blur = 1, 20
min_dot_size = 1.0
max_dot_size = 16.0
circularity_threshold = 3.0
threshold_bright, threshold_dark_inv = 170, 170
attrs = dict([(v, eval(v)) for v in [
'small_blur', 'large_blur', 'threshold_bright', 'threshold_dark_inv',
'circularity_threshold', 'min_dot_size', 'max_dot_size'
]])
primitives2D = QGLViewer.makePrimitives2D(([], []), ([], []))
primitives = []
if len(movieFilenames) is 1:
# TODO: time_base, timecode
K, RT = Calibrate.composeK(fovX, ox, oy), Calibrate.composeRT(
Calibrate.composeR(pan_tilt_roll), tx_ty_tz, 0)
mats = [[
K[:3, :3], RT[:3, :4],
np.dot(K, RT)[:3, :], distortion, -np.dot(RT[:3, :3].T, RT[:3, 3]),
[1920, 1080]
]]
camera_ids = ['video']
movies = [
MovieReader.open_file(os.path.join(directory, movieFilenames[0]),
audio=False)
]
else: # hard coded cameras
if xcp_filename.endswith('.xcp'):
if detectingTiara: # gruffalo
c3d_filename = os.path.join(
directory,
'201401211653-4Pico-32_Quad_Dialogue_01_Col_wip_02.c3d')
from IO import C3D
c3d_dict = C3D.read(c3d_filename)
global c3d_frames
c3d_frames, c3d_fps, c3d_labels = c3d_dict['frames'], c3d_dict[
'fps'], c3d_dict['labels']
c3d_subject = '' #'TedFace'
which = np.where(
[s.startswith(c3d_subject) for s in c3d_labels])[0]
c3d_frames = c3d_frames[:, which, :]
c3d_labels = [c3d_labels[i] for i in which]
print len(c3d_frames)
xcp, xcp_data = ViconReader.loadXCP(
os.path.join(directory, xcp_filename))
mats.extend(xcp)
elif xcp_filename.endswith('.cal'):
from IO import OptitrackReader
xcp, xcp_data = OptitrackReader.load_CAL(
os.path.join(directory, xcp_filename))
mats = xcp
print 'mats', len(mats), len(movieFilenames)
assert (len(mats) == len(movieFilenames))
camera_ids = []
movies = []
for ci, mf in enumerate(movieFilenames):
fo = 0 if frame_offsets is None else frame_offsets[ci]
movies.append(
MovieReader.open_file(os.path.join(directory, mf),
audio=False,
frame_offset=fo))
camera_ids = ['cam_%d' % ci for ci in xrange(len(mats))]
print len(mats), len(movies), len(camera_ids)
primitives.append(GLPoints3D([]))
primitives.append(GLPoints3D([]))
primitives.append(GLPoints3D([]))
primitives[0].colour = (0, 1, 1, 0.5) # back-projected "cyan" points
primitives[1].colour = (0, 0, 1, 0.5)
primitives[1].pointSize = 5
primitives[2].colour = (1, 0, 0, 0.99)
if len(movieFilenames) != 1 and detections_filename != None:
try:
dot_detections = IO.load(detections_filename)[1]
except:
numFrames = len(c3d_frames) # TODO HACK HACK
dot_detections = movies_to_detections(movies, range(numFrames),
deinterlacing, attrs)
IO.save(detections_filename, dot_detections)
if detectingTiara:
x3ds_seq = {}
for fi in dot_detections.keys():
frame = c3d_frames[(fi - 55) % len(c3d_frames)]
which = np.array(np.where(frame[:, 3] == 0)[0], dtype=np.int32)
x3ds_seq[fi] = np.concatenate((VICON_tiara_x3ds + np.array([150,-100,0],dtype=np.float32),frame[which,:3])), \
np.concatenate((np.arange(len(VICON_tiara_x3ds),dtype=np.int32),which+len(VICON_tiara_x3ds)))
dot_labels = get_labels(dot_detections.keys(),
x3ds_seq,
dot_detections,
mats,
x2d_threshold=0.05)
calibration_fi = 546 - 2 - 6
RT = tighten_calibration(x3ds_seq[calibration_fi],
dot_labels[calibration_fi], mats)
for v in c3d_frames:
v[:, :3] = np.dot(v[:, :3], RT[:3, :3].T) + RT[:, 3]
if True:
dot_detections = IO.load(detections_filename)[1]
x3ds_seq = {}
for fi in dot_detections.keys():
frame = c3d_frames[(fi - 55) % len(c3d_frames)]
which = np.array(np.where(frame[:, 3] == 0)[0],
dtype=np.int32)
x3ds_seq[fi] = np.concatenate((VICON_tiara_x3ds + np.array([0,1000,0],dtype=np.float32),frame[which,:3])), \
np.concatenate((np.arange(len(VICON_tiara_x3ds),dtype=np.int32),which+len(VICON_tiara_x3ds)))
#dot_labels = get_labels(dot_detections.keys(), x3ds_seq, dot_detections, mats, x2d_threshold = 0.05)
if detectingTiara:
primitives.append(GLPoints3D(VICON_tiara_x3ds + [0, 1000, 0]))
primitives[-1].pointSize = 5
global track3d, prev_frame, booting, trackGraph
track3d = Label.Track3D(mats[:len(movies)],
x2d_threshold=0.03,
x3d_threshold=5.0,
min_rays=3,
boot_interval=2) #tilt_threshold = 0.01, gruffalo
trackGraph = Label.TrackGraph()
prev_frame = 0
booting = 1
from UI import QApp
from PySide import QtGui
from GCore import State
# Modified the options parameter for fields to be the range of acceptable values for the box
# Previously would crash if small_blur got too low
QApp.fields = {
'image filter': [
('small_blur', 'Small blur radius',
'This is part of the image filter which controls the size of smallest detected features.',
'int', small_blur, {
"min": 0,
"max": None
}),
('large_blur', 'Large blur radius',
'This is part of the image filter which controls the size of largest detected features.',
'int', large_blur, {
"min": 0,
"max": None
}),
('threshold_bright', 'threshold_bright',
'This is part of the image filter which controls the size of smallest detected features.',
'int', threshold_bright, {
"min": 0,
"max": 255
}),
('threshold_dark_inv', 'threshold_dark_inv',
'This is part of the image filter which controls the size of largest detected features.',
'int', threshold_dark_inv, {
"min": 0,
"max": 255
}),
('circularity_threshold', 'circularity_threshold',
'How circular?.', 'float', circularity_threshold, {
"min": 0,
"max": 100
}),
('min_dot_size', 'min_dot_size',
'min_dot_size smallest detected features.', 'float', min_dot_size,
{
"min": 0,
"max": 100
}),
('max_dot_size', 'max_dot_size',
'max_dot_size largest detected features.', 'float', max_dot_size,
{
"min": 0,
"max": 100
}),
]
}
State.addKey('dotParams', {'type': 'image filter', 'attrs': attrs})
State.setSel('dotParams')
appIn = QtGui.QApplication(sys.argv)
appIn.setStyle('plastique')
win = QApp.QApp()
win.setWindowTitle('Imaginarium Dots Viewer')
QGLViewer.makeViewer(primitives=primitives,
primitives2D=primitives2D,
timeRange=(firstFrame, lastFrame),
callback=setFrame,
mats=mats,
camera_ids=camera_ids,
movies=movies,
pickCallback=picked,
appIn=appIn,
win=win)
def animateHead(newFrame):
global ted_geom, ted_geom2, ted_shape, tony_geom, tony_shape, tony_geom2, tony_obj, ted_obj, diff_geom, c3d_frames, extract
global tony_shape_vector, tony_shape_mat, ted_lo_rest, ted_lo_mat, c3d_points
global md, movies
tony_geom.image, tony_geom.bindImage, tony_geom.bindId = ted_geom.image, ted_geom.bindImage, ted_geom.bindId # reuse the texture!
fo = 55
MovieReader.readFrame(md, seekFrame=((newFrame + fo) / 2))
view = QApp.view()
for ci in range(0, 4):
view.cameras[ci + 1].invalidateImageData()
ci = view.cameras.index(view.camera) - 1
if ci >= 0:
MovieReader.readFrame(movies[ci],
seekFrame=(newFrame +
fo)) # only update the visible camera
frac = (newFrame % 200) / 100.
if (frac > 1.0): frac = 2.0 - frac
fi = newFrame % len(c3d_frames)
if ted_skel: # move the skeleton
dofs = ted_anim['dofData'][fi * 2 - 120]
Character.pose_skeleton(ted_skel['Gs'], ted_skel, dofs)
ted_glskel.setPose(ted_skel['Gs'])
offset = ted_skel['Gs'][13] # ted_skel['jointNames'].index('VSS_Head')
cams = QApp.app.getLayers()['cameras']
tmp = np.eye(4, 4, dtype=np.float32)
tmp[:3, :] = offset
cams.setTransform(tmp)
if ci >= 0: # move the camera view to be correct
camRT = mats[ci][1]
RT = np.dot(camRT, np.linalg.inv(tmp))
view.cameras[ci + 1].setRT(RT)
# update the face geometries to fit the skeleton
ted_geom.setPose(offset.reshape(1, 3, 4))
tony_geom.setPose(offset.reshape(1, 3, 4))
#TODO head_points,c3d_points,surface_points,ted_geom2
frame = c3d_frames[fi][extract]
which = np.where(frame[:, 3] == 0)[0]
x3ds = frame[which, :3]
#print which,x3ds.shape,ted_lo_rest.shape,ted_lo_mat.shape
bnds = np.array([[0, 1]] * ted_lo_mat.shape[0], dtype=np.float32)
tony_shape_vector[:] = OBJReader.fitLoResShapeMat(ted_lo_rest,
ted_lo_mat,
x3ds,
Aoffset=10.0,
Boffset=3.0,
x_0=tony_shape_vector,
indices=which,
bounds=bnds)
#global tony_shape_vectors; tony_shape_vector[:] = tony_shape_vectors[newFrame%len(tony_shape_vectors)]
#tony_shape_vector *= 0.
#tony_shape_vector += (np.random.random(len(tony_shape_vector)) - 0.5)*0.2
if 1:
ted_shape_v = np.dot(ted_shape_mat_T, tony_shape_vector).reshape(-1, 3)
else:
ted_shape_v = np.zeros_like(ted_obj['v'])
ISCV.dot(ted_shape_mat_T, tony_shape_vector, ted_shape_v.reshape(-1))
tony_shape_v = ted_shape_v
#tony_shape_v = tony_shape['v']*frac
ted_geom.setVs(ted_obj['v'] + ted_shape_v) #ted_shape['v'] * frac)
tony_geom.setVs(tony_obj['v'] + tony_shape_v -
np.array([200, 0, 0], dtype=np.float32))
ted_geom2.setVs(ted_obj['v'] * (1.0 - frac) +
tony_tedtopo_obj['v'] * frac +
np.array([200, 0, 0], dtype=np.float32))
#if len(ted_shape_v) == len(tony_shape_v):
# tony_geom2.setVs(tony_obj['v'] + ted_shape_v - [400,0,0])
# diff_geom.setVs(ted_obj['v'] + tony_shape_v - ted_shape_v - [600,0,0])
#print [c3d_labels[i] for i in which]
surface_points.vertices = np.dot(ted_lo_mat.T,
tony_shape_vector).T + ted_lo_rest
surface_points.colour = [0, 1, 0, 1] # green
c3d_points.vertices = x3ds
c3d_points.colour = [1, 0, 0, 1] # red
QApp.app.refreshImageData()
QApp.app.updateGL()
def setFrame(frame):
global State, mats, movieFilenames, primitives
global movies, primitives2D, deinterlacing, detectingWands, dot_detections, track3d, prev_frame, booting, trackGraph
key = State.getKey('dotParams/attrs')
skipping, prev_frame = (frame != prev_frame
and frame - 1 != prev_frame), frame
booting = 10 if skipping else booting - 1
p0, p1 = [], []
if True: #dot_detections is None:
for pair in enumerate(movies):
pts = process_frame(deinterlacing, detectingWands, frame, key,
pair)
p0.append(pts[0])
p1.append(pts[1])
def make_bounds(lens):
return np.array([sum(lens[:x]) for x in xrange(len(lens) + 1)],
dtype=np.int32)
data0 = np.array(np.concatenate(p0),
dtype=np.float32).reshape(-1, 2), make_bounds(
map(len, p0))
data1 = np.array(np.concatenate(p1),
dtype=np.float32).reshape(-1, 2), make_bounds(
map(len, p1))
else:
#dot_detections = movies_to_detections(movies, [frame], deinterlacing, key)
data0, data1 = dot_detections[frame] if dot_detections.has_key(
frame) else dot_detections.values()[0]
for ci, md in enumerate(movies):
try:
MovieReader.readFrame(md, seekFrame=frame)
except:
print 'oops', frame
return None, None
#img = np.frombuffer(md['vbuffer'],dtype=np.uint8).reshape(md['vheight'],md['vwidth'],3)
QApp.view().cameras[ci + 1].invalidateImageData()
data0 = data0[0].copy(), data0[
1] # so that undistort doesn't modify the raw detections
data1 = data1[0].copy(), data1[1]
# TODO, move this to the viewer...
data0 = ViconReader.frameCentroidsToDets(data0, mats)
data1 = ViconReader.frameCentroidsToDets(data1, mats)
primitives2D[0].setData(data0[0], data0[1])
primitives2D[1].setData(data1[0], data1[1])
#print x2ds_labels
if len(movieFilenames) is not 1:
if 1:
#x2ds_data, x2ds_splits = data0 # dark points only
x2ds_data, x2ds_splits = data1 # light points only
if skipping:
x3ds, x3ds_labels = track3d.boot(x2ds_data, x2ds_splits)
#trackGraph = Label.TrackGraph()
else:
x3ds, x3ds_labels = track3d.push(x2ds_data, x2ds_splits)
# coarse bounding box
if False:
for xi, x in zip(x3ds_labels, x3ds):
if x[0] < -200 or x[0] > 200 or x[1] < 800 or x[
1] > 1200 or x[2] < -50 or x[2] > 300:
track3d.x2ds_labels[np.where(
track3d.x2ds_labels == xi)[0]] = -1
x[:] = 0
primitives[0].setData(x3ds)
#trackGraph.push(x3ds,x3ds_labels)
#primitives[0].graph = trackGraph.drawing_graph()
elif False:
Ps = np.array([m[2] / (m[0][0, 0]) for m in mats],
dtype=np.float32)
data = data0 # dark points
#data = data1 # light points
x3ds, x2ds_labels = Recon.intersect_rays(data[0],
data[1],
Ps,
mats,
tilt_threshold=0.003,
x2d_threshold=0.02,
x3d_threshold=5.0,
min_rays=2)
primitives[0].setData(x3ds)
if detectingTiara:
global c3d_frames
frame = c3d_frames[(frame - 55) % len(c3d_frames)]
which = np.where(frame[:, 3] == 0)[0]
x3ds = frame[which, :3]
#print frame,'len',len(x3ds)
primitives[1].setData(x3ds)
QApp.app.refreshImageData()
QApp.app.updateGL()
def intersectRaysCB(fi):
global x2d_frames, mats, Ps, c3d_frames, view, primitives, primitives2D, track3d, prev_frame, track_orn, orn_graph, boot, g_all_skels, md, orn_mapper, mar_mapper
skipping = prev_frame is None or np.abs(fi - prev_frame) > 10
prev_frame = fi
view = QApp.view()
points, altpoints = primitives
g2d = primitives2D[0]
frame = x2d_frames[fi]
x2ds_data, x2ds_splits = ViconReader.frameCentroidsToDets(frame, mats)
g2d.setData(x2ds_data, x2ds_splits)
if skipping:
x3ds, x3ds_labels = track3d.boot(x2ds_data, x2ds_splits)
#trackGraph = Label.TrackGraph()
boot = -10
else:
x3ds, x3ds_labels = track3d.push(x2ds_data, x2ds_splits)
if False:
boot = boot + 1
if boot == 0:
x2d_threshold_hash = 0.01
penalty = 10.0 # the penalty for unlabelled points. this number should be about 10. to force more complete labellings, set it higher.
maxHyps = 500 # the number of hypotheses to maintain.
print "booting:"
numLabels = len(orn_graph[0])
l2x = -np.ones(numLabels, dtype=np.int32)
label_score = ISCV.label_from_graph(x3ds, orn_graph[0],
orn_graph[1], orn_graph[2],
orn_graph[3], maxHyps, penalty,
l2x)
clouds = ISCV.HashCloud2DList(x2ds_data, x2ds_splits,
x2d_threshold_hash)
which = np.array(np.where(l2x != -1)[0], dtype=np.int32)
pras_score, x2d_labels, vels = Label.project_assign(
clouds,
x3ds[l2x[which]],
which,
Ps,
x2d_threshold=x2d_threshold_hash)
print fi, label_score, pras_score
labelled_x3ds = x3ds[l2x[which]]
print track_orn.bootPose(x2ds_data, x2ds_splits, x2d_labels)
if boot > 0:
track_orn.push(x2ds_data, x2ds_splits, its=4)
#x3ds,x2ds_labels = Recon.intersect_rays(x2ds_data, x2ds_splits, Ps, mats, seed_x3ds = None)
points.setData(x3ds)
if c3d_frames != None:
c3ds = c3d_frames[(fi - 832) / 2]
true_labels = np.array(np.where(c3ds[:, 3] == 0)[0], dtype=np.int32)
x3ds_true = c3ds[true_labels, :3]
altpoints.setData(x3ds_true)
ci = view.cameraIndex() - 1
if True: #ci == -1:
MovieReader.readFrame(md, seekFrame=max((fi - 14) / 4, 0))
QApp.app.refreshImageData()
(orn_skel_dict, orn_t) = g_all_skels['orn']
orn_mesh_dict, orn_skel_mesh, orn_geom_mesh = orn_t
orn_anim_dict = orn_skel_dict['anim_dict']
orn_skel_dict['chanValues'][:] = orn_anim_dict['dofData'][fi]
Character.updatePoseAndMeshes(orn_skel_dict, orn_skel_mesh, orn_geom_mesh)
(mar_skel_dict, mar_t) = g_all_skels['mar']
mar_anim_dict = mar_skel_dict['anim_dict']
mar_mesh_dict, mar_skel_mesh, mar_geom_mesh = mar_t
Character.updatePoseAndMeshes(mar_skel_dict, mar_skel_mesh, mar_geom_mesh,
mar_anim_dict['dofData'][fi])
from PIL import Image
#orn_geom_mesh.setImage((md['vbuffer'],(md['vheight'],md['vwidth'],3)))
#orn_geom_mesh.refreshImage()
w, h = 1024, 1024
cam = view.cameras[0]
cam.refreshImageData(view)
aspect = float(max(1, cam.bindImage.width())) / float(
cam.bindImage.height()) if cam.bindImage is not None else 1.0
orn_mapper.project(orn_skel_dict['geom_Vs'], aspect)
data = Opengl.renderGL(w, h, orn_mapper.render, cam.bindId)
orn_geom_mesh.setImage(data)
mar_mapper.project(mar_skel_dict['geom_Vs'], aspect)
data = Opengl.renderGL(w, h, mar_mapper.render, cam.bindId)
mar_geom_mesh.setImage(data)
#image = Image.fromstring(mode='RGB', size=(w, h), data=data)
#image = image.transpose(Image.FLIP_TOP_BOTTOM)
#image.save('screenshot.png')
if 0:
global g_screen
image = Opengl.renderGL(1920, 1080, Opengl.quad_render,
(cam.bindId, g_screen))
import pylab as pl
pl.imshow(image)
pl.show()
view.updateGL()
def paintGL(self, p0=0, p1=None, drawOpts=DRAWOPT_ALL): ''' :param drawOpts: OR combination of draw flags. default is :data:`UI.DRAWOPT_ALL` ''' #if not self.d['draw'] or not self.d['visible']: return doingSelection = (p1 is not None) if p1 is None: p1 = len(self) if not self.GL_is_initialised: self.initializeGL() if p1 == 0: return # don't render if no vertices if self.image != self.bindImage: if self.image == []: self.image = QtGui.QPixmap(self.imageFilename).toImage() self.imageFlipped = False if self.bindImage is not None: self.deleteTexture(self.bindId) self.bindId,self.bindImage = long(0),None if self.image is not None: global win if self.view == None: from UI import QApp; self.view = QApp.view() # TODO self.bindId = self.view.bindTexture(self.image) self.bindImage = self.image if self.bindImage is not None: GL.glEnable(GL.GL_TEXTURE_2D) GL.glBindTexture(GL.GL_TEXTURE_2D, self.bindId) GL.glEnable(GL.GL_BLEND) GL.glEnable(GL.GL_CULL_FACE) GL.glCullFace(GL.GL_BACK) GL.glFrontFace(GL.GL_CCW) GL.glEnable(GL.GL_LIGHTING) GL.glEnable(GL.GL_LIGHT0) Pmat = GL.glGetDoublev(GL.GL_PROJECTION_MATRIX) lightDir = -Pmat[:3,2] # the direction the camera is looking GL.glLightfv(GL.GL_LIGHT0, GL.GL_POSITION, lightDir) GL.glShadeModel(GL.GL_SMOOTH) if self.colour is not None: GL.glMaterialfv(GL.GL_FRONT, GL.GL_AMBIENT_AND_DIFFUSE, self.colour) if self.vs is not None: GL.glEnableClientState(GL.GL_VERTEX_ARRAY) self.vs.bind() GL.glVertexPointerf(self.vs) if self.vts is not None: GL.glEnableClientState(GL.GL_TEXTURE_COORD_ARRAY) self.vts.bind() GL.glTexCoordPointerf(self.vts) if self.vns is not None: GL.glEnableClientState(GL.GL_NORMAL_ARRAY) self.vns.bind() GL.glNormalPointerf(self.vns) if self.tris is not None and DRAWOPT_GEOMS & drawOpts: if not doingSelection: GL.glUseProgram(self.shader2) self.tris.bind() if self.drawStyle == 'wire': GL.glShadeModel(GL.GL_FLAT) GL.glLineWidth(1) GL.glDrawElements(GL.GL_LINES, (p1-p0)*3, GL.GL_UNSIGNED_INT, self.tris + p0*12) GL.glShadeModel(GL.GL_SMOOTH) elif self.drawStyle == 'smooth': GL.glDrawElements(GL.GL_TRIANGLES, (p1-p0)*3, GL.GL_UNSIGNED_INT, self.tris + p0*12) #GL.glDrawElementsui(GL.GL_TRIANGLES, self.tris) elif self.drawStyle == 'wire_over_smooth': GL.glDrawElements(GL.GL_TRIANGLES, (p1-p0)*3, GL.GL_UNSIGNED_INT, self.tris + p0*12) #GL.glDrawElementsui(GL.GL_TRIANGLES, self.tris) GL.glShadeModel(GL.GL_FLAT) GL.glMaterialfv(GL.GL_FRONT, GL.GL_AMBIENT_AND_DIFFUSE, [0,0,0,1]) GL.glLineWidth(1) GL.glDrawElementsui(GL.GL_LINES, self.tris) GL.glMaterialfv(GL.GL_FRONT, GL.GL_AMBIENT_AND_DIFFUSE, self.colour) GL.glShadeModel(GL.GL_SMOOTH) self.tris.unbind() if self.transformData is not None and self.transforms is not None and DRAWOPT_GEOMS & drawOpts: if not doingSelection: GL.glUseProgram(self.shader) GL.glUniformMatrix4fv(self.shader_myMat, len(self.transforms), GL.GL_FALSE, self.transforms) # put the transforms in myMat GL.glEnableVertexAttribArray(self.shader_bi) self.vtis.bind() GL.glVertexAttribIPointer(self.shader_bi, 1, GL.GL_UNSIGNED_INT, 0, self.vtis) # write the vtis to bi self.tis.bind() if self.drawStyle == 'wire': GL.glShadeModel(GL.GL_FLAT) GL.glLineWidth(1) GL.glDrawElementsui(GL.GL_LINES, self.tis) # this is wrong GL.glShadeModel(GL.GL_SMOOTH) elif self.drawStyle == 'smooth': GL.glDrawElementsui(GL.GL_TRIANGLES, self.tis) elif self.drawStyle == 'wire_over_smooth': GL.glDrawElementsui(GL.GL_TRIANGLES, self.tis) GL.glShadeModel(GL.GL_FLAT) GL.glMaterialfv(GL.GL_FRONT, GL.GL_AMBIENT_AND_DIFFUSE, [0,0,0,1]) GL.glLineWidth(1) GL.glDrawElementsui(GL.GL_LINES, self.tis) GL.glMaterialfv(GL.GL_FRONT, GL.GL_AMBIENT_AND_DIFFUSE, self.colour) GL.glShadeModel(GL.GL_SMOOTH) self.tis.unbind() self.vtis.unbind() #for ti,tris in self.transformData: #if len(tris)==0: continue #GL.glUniformMatrix4fv(self.myMat, 1, GL.GL_FALSE, self.transforms[ti]) #GL.glDrawElementsui(self.drawStyle, tris) if self.vs is not None: self.vs.unbind() GL.glDisableClientState(GL.GL_VERTEX_ARRAY) if self.vts is not None: self.vts.unbind() GL.glDisableClientState(GL.GL_TEXTURE_COORD_ARRAY) if self.vns is not None: self.vns.unbind() GL.glDisableClientState(GL.GL_NORMAL_ARRAY) GL.glDisable(GL.GL_LIGHTING) GL.glDisable(GL.GL_BLEND) GL.glDisable(GL.GL_TEXTURE_2D) GL.glBindTexture(GL.GL_TEXTURE_2D, 0) GL.glUseProgram( 0 )
def setFrame_cb(fi):
attrs = State.getKey('/root/ui/attrs/')
global g_setting_frame
if g_setting_frame: return
g_setting_frame = True
try: # within this loop we handle the timeline, which could trigger calling this function recursively
global g_mode, g_frame, g_TIS_server, g_neutral_corrective_shape
global g_smooth_pose
view = QApp.view()
cid = view.cameraIndex()
if cid != g_mode: # deal with changing modes
g_mode = cid
if g_mode == 0:
if g_md is not None: QApp.app.qtimeline.setRange(0, g_md['vmaxframe'])
elif g_mode == 1:
pose_splits = rbfn_pose_splits()
QApp.app.qtimeline.setRange(0, pose_splits[-1]-1)
new_frame = g_frame.get(g_mode,fi)
if new_frame != fi:
QApp.app.qtimeline.frame = new_frame
fi = new_frame
except Exception as e:
print 'exc setFrame',e
g_setting_frame = False
g_frame[g_mode] = fi
if not attrs['setting_neutral']: g_neutral_corrective_shape = 0
new_pose,new_shape,norm_shape,img,slider_names,slider_values,A = [track_view_cb,rbfn_view_cb][g_mode](fi,attrs)
mirror_scale = -1 if attrs['mirroring'] else 1
h,wm = img.shape[0]*0.5,img.shape[1]*0.5*mirror_scale
geo_vs = np.zeros((new_shape.shape[0],3), dtype=np.float32)
if attrs['debugging']: # display the stabilised data
geo_vs[:,:2] = norm_shape
geo_vs *= 200
geo_vs[:,:2] += np.int32(np.mean(new_shape, axis=0)/200)*200
else: # display the tracking data
geo_vs[:,:2] = new_shape
geo_mesh,image_mesh,bs_mesh = QApp.app.getLayers(['geo_mesh', 'image_mesh', 'bs_mesh'])
bs_mesh.visible = attrs['show_harpy']
if bs_mesh.visible:
global g_bs_vs, g_bs_shape_mat_T
bs_mesh.setVs(g_bs_vs + np.dot(g_bs_shape_mat_T, np.clip(slider_values[:-3],0,1)))
# compute the Harpy position
R = Calibrate.composeR(new_pose*[1,-1,-1])
if g_mode == 1: R = np.eye(3) # TODO
bs_ts = Calibrate.composeRT(R,[0,1720,0],0) # compensate for the offset of the Harpy (temples ~1720mm above origin)
scale = 1.0/np.linalg.norm(160.*A) # IPD (64mm) / 0.4 (ref_shape) = 160.
off = np.mean(new_shape[[0,16]],axis=0) # get the position of the temples (pixels)
g_smooth_pose[g_mode] = filter_data(np.float32([scale,off[0],off[1]]), g_smooth_pose.setdefault(g_mode,None), 10.0)
pose = g_smooth_pose[g_mode]
bs_ts[:3] *= pose[0]
bs_ts[:3,3] += [pose[1]-abs(wm),1000+pose[2]-h,0]
# offset screen-right 300mm
bs_ts[:3,3] += (pose[0]*attrs['harpy_xoffset'])*np.float32([np.cos(np.radians(view.camera.cameraRoll)),-np.sin(np.radians(view.camera.cameraRoll)),0.0])
bs_mesh.transforms[0] = bs_ts.T
geo_mesh.setVs(geo_vs)
geo_mesh.colour=[0 if attrs['streaming_TIS'] else 1,1 if attrs['streaming_TIS'] else 0,0,1]
geo_mesh.transforms[0][:,:3] = [[mirror_scale,0,0],[0,1,0],[0,0,1],[-wm,1000-h,0.1]]
image_mesh.setVs(np.float32([[-wm,-h,0],[wm,-h,0],[wm,h,0],[-wm,h,0]]))
image_mesh.setImage(img)
if attrs['unreal']:
if not attrs['streaming_TIS']: toggle_unreal()
ret, activeConnections = g_TIS_server.WriteAll(PyTISStream.getBlendshapeData(slider_names, slider_values))
if not ret:
print "Server is not Initialised"
State._setKey('/root/ui/attrs/streaming_TIS', False)
else:
# Turn off streaming
if attrs['streaming_TIS']: toggle_unreal()
QApp.app.updateGL()
def track_view_cb(fi, attrs):
# g_mode = 0
global g_webcam, g_md, g_rbfn, g_predictor
# runtime options and state
global g_prev_smooth_shape, g_prev_vs, g_hmc_boot, g_settle, g_head_pan_tilt_roll
if attrs['using_webcam']:
if g_webcam is None:
g_webcam = WebCam()
g_webcam.Open(State.getKey('/root/ui/attrs/cam_offset') + State.getKey('/root/ui/attrs/webcam_index'))
g_webcam.SetProperty('FPS', State.getKey('/root/ui/attrs/cam_fps'))
g_webcam.SetProperty('FRAME_WIDTH', State.getKey('/root/ui/attrs/cam_width'))
g_webcam.SetProperty('FRAME_HEIGHT', State.getKey('/root/ui/attrs/cam_height'))
if g_webcam is None:
img = np.zeros((16,16,3),dtype=np.uint8)
else:
img = g_webcam.GetFrame()
if img is None:
img = np.zeros((16,16,3),dtype=np.uint8)
elif g_md is not None:
MovieReader.readFrame(g_md, seekFrame=fi) # only update the visible camera
img = np.frombuffer(g_md['vbuffer'], dtype=np.uint8).reshape(g_md['vheight'],g_md['vwidth'],3)
#QApp.app.qtimeline.setRange(0, g_md['vmaxframe'])
else:
img = np.zeros((16,16,3),dtype=np.uint8)
mirror_scale = -1 if attrs['mirroring'] else 1
rotate = attrs['rotate']
if g_settle >= 0:
if g_settle == 0 and g_prev_vs is not None:
g_hmc_boot = g_prev_vs.copy()
g_settle = g_settle - 1
else:
if attrs['HMC_mode'] and g_hmc_boot is not None: g_prev_vs = g_hmc_boot.copy()
if attrs['booting'] or Face.test_reboot(img, g_prev_vs):
g_prev_vs = Face.detect_face(img, g_predictor, 2, rotate)
g_hmc_boot = None # in case we didn't detect a face
g_settle = 10 # go into settle mode (10 frames)
if g_prev_vs is not None:
State.setKey('/root/ui/attrs/booting',False)
if attrs['HMC_mode']: g_hmc_boot = g_prev_vs.copy()
g_prev_vs = Face.track_face(img, g_predictor, g_prev_vs, rotate=rotate)
# compensate for roll, translation and scale
norm_shape, head_pan, head_tilt, A = stabilize_shape(g_prev_vs, setting_neutral=attrs['setting_neutral'])
# dejitter
if attrs['filtering']:
g_prev_smooth_shape = filter_data(norm_shape, g_prev_smooth_shape)
else:
g_prev_smooth_shape = norm_shape.copy()
# extract angles from the measured values
head_pan_tilt_roll = np.degrees(np.arctan2([head_pan*mirror_scale, head_tilt, -mirror_scale*A[1][0]],[2,2,A[1][1]]))
g_head_pan_tilt_roll = filter_data(head_pan_tilt_roll, g_head_pan_tilt_roll, 3.0)
camera = QApp.view().camera
camera.lockedUpright = False
camera.cameraRoll = (-90*rotate if rotate != -1 else g_head_pan_tilt_roll[2])
ret = g_prev_smooth_shape.copy()
if attrs['mirroring']:
flip_order = [16,15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0, 26,25,24,23,22,21,20,19,18,17, 27,28,29,30, 35,34,33,32,31, \
45,44,43,42, 47,46, 39,38,37,36, 41,40, 54,53,52,51,50,49,48, 59,58,57,56,55, 64,63,62,61,60, 67,66,65, 69,68]
ret = ret[flip_order]
slider_names, slider_values = applyRetarget(g_rbfn, ret)
#State._setKey('/root/sliders/attrs', dict(zip(slider_names, slider_values))) # NO UNDO
slider_names.extend(['NeckPan','NeckTilt','NeckRoll'])
slider_values = np.float32(list(slider_values)+list(g_head_pan_tilt_roll))
return g_head_pan_tilt_roll.copy(),g_prev_vs.copy(),norm_shape,img,slider_names,slider_values,A
def set_selected_vertex(vi):
State.setKey('/markup_mesh_sel', vi)
markup_mesh = QApp.view().getLayer('markup_mesh')
markup_mesh.selectedIndex = vi
def set_frame_cb2(frame):
global g_predictor, g_predictor_dlib, g_detector
size = (len(g_predictor['ref_shape'])+4)
geo_vs = np.zeros((size,3), dtype=np.float32)
ref_vs = np.zeros((size,3), dtype=np.float32)
global g_prev_vs
try: g_prev_vs
except: g_prev_vs = None
if 0: # show_images
global g_jpgs; fn = g_jpgs[frame%len(g_jpgs)]
img = Face.load_image(fn)
img = Face.fix_image(img, max_size=640)
use_prev_vs = False # images need booting every frame
else: # show_movies
global md; MovieReader.readFrame(md, seekFrame=frame) # only update the visible camera
img = np.frombuffer(md['vbuffer'], dtype=np.uint8).reshape(md['vheight'],md['vwidth'],3)
use_prev_vs = True
if 0: # undistort_stuff
global g_screen
global g_tid, g_bid
g_tid,g_bid = Opengl.bind_streaming_image(img, g_tid, g_bid)
img = Opengl.renderGL(img.shape[1], img.shape[0], Opengl.quad_render, (g_tid, g_screen, 0.85))
#Opengl.unbind_image(bid)
if 0: # rotated_image
img = img.transpose((1,0,2)).copy()
if 0: # gamma_image
lookup = np.array([int(((x/255.0)**0.4545)*255.0) for x in range(256)], dtype=np.uint8)
img = lookup[img]
#img[:,600:1000] = 0 #img[:,200:600].copy()
if 0: # test_rotate
import scipy; img = scipy.misc.imrotate(img, frame, interp='bilinear')
if 0: # test_rotate_right
import scipy; img[:,-img.shape[0]:] = scipy.misc.imrotate(img[:,-img.shape[0]:], frame, interp='bilinear')
if 0: # test_filter_image
img = ISCV.filter_image(img,4,16)
w,h = img.shape[1]*0.5,img.shape[0]*0.5
boot = g_prev_vs
if boot is None: boot = Face.detect_face(img, g_predictor, 2) # ,-1) # put -1 at end to boot at any angle
tmp = Face.track_face(img, g_predictor, boot)
if use_prev_vs and boot is not None: g_prev_vs = tmp
if frame == 0 or Face.test_reboot(img, g_prev_vs): g_prev_vs = None
global template_vs
geo_vs[:size-4,:2] = tmp
geo_vs[size-4:size,:2] = Face.get_boundary(geo_vs[:size-4,:2], template_vs)
if 0: # show_aam
global g_aam_model
shape_u, tex_u, A_inv, mn = Face.fit_aam(g_aam_model, tmp, img)
Face.render_aam(g_aam_model, A_inv*0.1, mn*0.1, shape_u, tex_u, img)
su,tu = Face.normalized_aam_coords(g_aam_model, shape_u, tex_u)
res = Face.aam_residual(g_aam_model, tmp, img)
QApp.view().displayText = [(10,100,'%f' % np.linalg.norm(tu)),(10,125,'%f' % np.linalg.norm(su)),(10,150,'%f'%res)]
if 0: # show_extracted_texture
global g_aam_model_indices,g_aam_model_weights
pixels = Face.extract_texture(img, geo_vs[:size,:2], g_aam_model_indices, g_aam_model_weights)
global template_vs
Face.render_texture(pixels, img, template_vs, g_aam_model_indices, g_aam_model_weights)
geo_mesh = QApp.app.getLayer('geo_mesh')
geo_mesh.setVs(geo_vs)
geo_mesh.transforms[0][:,:3] = [[1,0,0],[0,1,0],[0,0,1],[-w,1000-h,0.1]]
image_mesh = QApp.app.getLayer('image_mesh')
image_mesh.setVs(np.array([[-w,-h,0],[w,-h,0],[w,h,0],[-w,h,0]], dtype=np.float32))
image_mesh.setImage(img)
QApp.view().updateGL()
def drawX3ds(win, locationName, attrs, interface, picked):
from UI import GLPoints3D
if 'x3ds' not in attrs: return
layerName = locationName
if attrs['type'] == 'points': layerName += '_x3ds'
if not win.hasLayer(layerName):
layer = GLPoints3D([])
win.setLayer(layerName, layer)
layer = win.getLayer(layerName)
if 'x3ds_pointSize' in attrs: layer.pointSize = attrs['x3ds_pointSize']
else: layer.pointSize = 8
if 'x3ds_colour' in attrs: layer.colour = attrs['x3ds_colour']
else: layer.colour = (1, 0.5, 0, 0.7)
colours = np.array([], dtype=np.float32)
if 'x3ds_colours' in attrs and attrs['x3ds_colours'].any():
colours = attrs['x3ds_colours']
if 'drawStyle' in attrs and attrs['drawStyle']:
layer.drawStyle = attrs['drawStyle']
x3ds = np.array(attrs['x3ds'], dtype=np.float32)
x3ds_labels = None
if 'x3ds_labels' in attrs:
x3ds_labels = attrs['x3ds_labels']
layer.setData(x3ds,
names=Interface.getLabelNames(x3ds_labels),
colours=colours)
else:
layer.setData(x3ds, colours=colours)
if layer: layer.visible = isVisible(attrs)
if 'normals' in attrs:
layer.normals = attrs['normals']
if 'edges' in attrs and attrs['edges'] is not None:
layer.edges = attrs['edges']
# layer.graph = attrs['trackGraph']
# Check if we want to draw the camera ray contributions
# Make sure:
# - We have a 3D object that has been picked
# - The picked object is a 3D point
# - The picked layer matches the one we are processing
if picked is not None and picked['type'] == '3d' and picked[
'primitiveType'] == 'GLPoints3D' and win.view().layers.keys()[
picked['primitiveIndex']] == layerName:
if 'index' not in picked: return
if picked['isLabel']:
li = picked['index']
if x3ds_labels is not None:
liIdx = np.where(x3ds_labels == li)[0]
xi = liIdx[0] if liIdx else li
else:
xi = li
else:
xi = picked['index']
li = x3ds_labels[xi]
if xi >= len(x3ds): return
selected_x3d = x3ds[xi]
# Create a yellow highlight around (behind) the picked 3D point
highlightLayer = GLPoints3D(np.array([selected_x3d], dtype=np.float32),
colour=(1, 1, 0, 0.9))
highlightLayer.pointSize = layer.pointSize + 6.
win.setLayer('x3d_selected', highlightLayer, selection=True)
if selected_x3d.any() and 'showCameraContributions' in attrs and attrs[
'showCameraContributions']:
if 'camerasLocation' not in attrs: return
if 'x3ds_labels' not in attrs: return
if 'labels' not in attrs or 'x2ds_splits' not in attrs: return
camsLoc = attrs['camerasLocation']
cams = interface.location(camsLoc)
if cams is None:
print 'Render Callback: No cameras found when showing 3D point ray contributions.'
return
x2ds_labels = attrs['labels']
x2ds_splits = attrs['x2ds_splits']
camIds = [
interface.findCameraIdFromRayId(rayId, x2ds_splits)
for rayId in np.where(x2ds_labels == li)[0]
]
camNames = [
cam.name for ci, cam in enumerate(win.view().cameras[1:])
if ci in camIds
]
camPositions = np.array([m[4] for m in cams['mats']],
dtype=np.float32)[camIds]
print '3D Point', li, '|', 'Cameras:', camIds, camNames
pts = [selected_x3d]
pts.extend(camPositions)
edges = [[0, idx + 1] for idx in range(len(camPositions))]
selLayer = GLPoints3D(pts, edges=edges, colour=(0, 1, 1, 0.5))
win.setLayer('x3d_cameraContributions', selLayer, selection=True)
# Camera panels test
if False:
try:
if win.cameraPanels is None:
win.cameraPanels = QApp.CameraPanels(
mainView=win.view())
win.cameraPanels.resetCurrPos()
for c in camIds:
win.cameraPanels.setCameraLayer(c + 1)
win.cameraPanels.show()
win.cameraPanels.update()
except Exception as e:
print e
