def drawMesh(win, locationName, attrs, interface, picked):
from UI import GLMeshes
# TODO: Add support for multiple meshes. The mesh ops should create lists of vs etc. and we iterate over the lot
if 'vs' not in attrs or 'tris' not in attrs: return
vs = attrs['vs']
tris = attrs['tris']
vts = attrs['vts'] if 'vts' in attrs else None
drawStyle = attrs['drawStyle'] if 'drawStyle' in attrs else 'smooth'
colour = attrs['colour'] if 'colour' in attrs else (0.9, 0.9, 0.9, 1.0)
names = attrs['names'] if 'names' in attrs else ['default']
transforms = attrs['transforms'] if 'transforms' in attrs else None
layerName = locationName + '_mesh'
if layerName not in win.getLayers():
mesh = GLMeshes(names=names,
verts=vs,
faces=tris,
drawStyle=drawStyle,
colour=colour,
vts=vts,
transforms=transforms)
win.setLayer(layerName, mesh)
else:
for _vs, _name in zip(vs, names):
layer = win.getLayer(layerName)
layer.setVs(_vs, _name)
layer.colour = colour
layer.drawStyle = drawStyle
layer = win.getLayer(layerName)
if layer: layer.visible = isVisible(attrs)
if 'texture' in attrs:
layer.setImage(attrs['texture'])
QApp.app.select('/fbx/' + name)
if __name__ == '__main__':
import sys
from GCore import State
from UI import GLMeshes
payload = load(sys.argv[1])
#from pprint import pprint; pprint(payload, open('fbx.txt','w'))
fbx = decode(payload)
IO.save('fbx.tmp', fbx)
skel_dict = extract_skeleton(fbx)
anim_dict = extract_animation(fbx, skel_dict)
#print skel_dict['jointNames']
skel_mesh = GLMeshes(**skelDictToMesh(skel_dict))
global g_anim_dict, g_skel_dict
g_anim_dict = anim_dict
g_skel_dict = skel_dict
#State.setKey('/doc',payload)
#fbx = decode(payload)
#State.setKey('/fbx',fbx)
#for k,v in fbx.iteritems(): State.setKey(k,v)
from UI import QApp, QGLViewer, GLMeshes, GLPoints3D
app, win = QGLViewer.makeApp(appName='Imaginarium FBX')
#outliner = QApp.app.qobjects
#for gi,(key,value) in enumerate(fbx.items()):
geo_bs.append((p0,p1))
geo_bs.append((p1,p2))
geo_bs.append((p2,p0))
size = len(template_vs)
geo_vs = np.zeros((size,3), dtype=np.float32)
geo_fs = []
geo_ts = np.array([[1,0,0,0],[0,1,0,1000],[0,0,1,0]], dtype=np.float32)
geo_vts = [[0,0]]*size
img_vs = [[-1000,-1000,0],[1000,-1000,0],[1000,1000,0],[-1000,1000,0]]
img_fs = [[0,1,2,3]]
img_ts = np.array([[1,0,0,0],[0,1,0,1000],[0,0,1,0]], dtype=np.float32)
img_vts = [[0,1],[1,1],[1,0],[0,0]]
geo_mesh = GLMeshes(names=['geo_mesh'],verts=[geo_vs],faces=[geo_fs],transforms=[geo_ts],bones=[geo_bs], vts=[geo_vts], colour=[1,0,0,1])
image_mesh = GLMeshes(names=['image_mesh'],verts=[img_vs],faces=[img_fs],transforms=[img_ts],bones = [[]], vts=[img_vts])
layers = {'image_mesh':image_mesh,'geo_mesh':geo_mesh}
if 0:
ref_vs = np.zeros((size,3), dtype=np.float32)
ref_vs[:,:2] = template_vs
ref_mesh = GLMeshes(names=['ref_mesh'],verts=[ref_vs],faces=[geo_fs],transforms=[geo_ts],bones=[geo_bs], vts=[geo_vts], colour=[0,1,0,1])
layers['ref_mesh'] = ref_mesh
app,win = QGLViewer.makeApp(appName='Imaginarium FaceTrack')
QApp.app.addMenuItem({'menu':'&File','item':'Import &movie','tip':'Import a movie file','cmd':import_movie_cb})
QApp.app.qoutliner.set_root('')
#for gi,geo in enumerate(layers.keys()): outliner.addItem(geo, data='_OBJ_'+geo, index=gi)
global g_aam_model_indices,g_aam_model_weights
g_aam_model_indices,g_aam_model_weights = Face.make_sample_model(template_vs, ref_fs, grid_size=1.0)
QApp.app.select(name)
elif isinstance(p,GLTool):
p.setComponent(pi)
QApp.app.updateGL()
from UI import GLMeshes # TODO move this colin
from UI import QApp
from GCore import State
from PySide import QtGui
import sys
appIn = QtGui.QApplication(sys.argv)
appIn.setStyle('plastique')
win = QApp.QApp()
win.setWindowTitle('Imaginarium Maya File Browser')
from UI import QGLViewer
primitives = []
#QApp.fields = fields
#outliner = QApp.app.qobjects
#for i,(o,v) in enumerate(dobjs):
# outliner.insertItem(i,o)
# outliner.item(i).setData(1,'_OBJ_'+v['name'])
vs = np.array([[[-1,-1,-1],[-1,-1,1],[-1,1,-1],[-1,1,1],[1,-1,-1],[1,-1,1],[1,1,-1],[1,1,1]]],dtype=np.float32)*100.0
faces = np.array([[[0,1,3,2],[4,6,7,5],[2,3,7,6],[1,0,4,5],[3,1,5,7],[0,2,6,4]]],dtype=np.int32)
transforms = [np.array([[1,0,0,0],[0,1,0,1000],[0,0,1,0]],dtype=np.float32)]
p = GLMeshes(names=['my mesh'], verts=vs, faces=faces, transforms=transforms)
q = GLTool()
q.setTool('3D rotation', transform = transforms[0], component=None)
r = GLTool()
r.setTool('3D translation', transform = transforms[0], component=None)
primitives = [p,q,r]
QGLViewer.makeViewer(primitives=primitives, timeRange = (1,100), callback=setFrameCB, pickCallback=pickedCB, appIn=appIn, win=win)
if len(sys.argv) > 1:
filename = sys.argv[1]
md = open_file(filename)
if len(sys.argv) > 2: setVolume(md, int(sys.argv[2]))
img = np.frombuffer(md['vbuffer'],
dtype=np.uint8).reshape(md['vheight'],
md['vwidth'], 3)
h, w = md['vheight'] / 2, md['vwidth'] / 2
img_vs = [[-w, -h, 0], [w, -h, 0], [w, h, 0], [-w, h, 0]]
img_fs = [[0, 1, 2, 3]]
img_ts = np.array([[1, 0, 0, 0], [0, 1, 0, 1000], [0, 0, 1, 0]],
dtype=np.float32)
img_vts = [[0, 1], [1, 1], [1, 0], [0, 0]]
image_mesh = GLMeshes(names=['image_mesh'],
verts=[img_vs],
faces=[img_fs],
transforms=[img_ts],
bones=[[]],
vts=[img_vts])
image_mesh.setImage(img)
QGLViewer.makeViewer(timeRange=(0, md['vmaxframe'], 1, md['fps']),
callback=set_frame_cb,
layers={'image_mesh': image_mesh})
exit()
#'Imaginarium Movie Playback Tool'
if len(sys.argv) > 1:
import cv2
appname = sys.argv.pop(0)
filename = sys.argv.pop(0)
rotate = False
if filename == '-r180':
rotate = True
def main():
grip_dir = os.environ['GRIP_DATA']
global g_model
g_model = IO.load(os.path.join(grip_dir, 'aam.new.io'))[1]
global g_predictor, reference_3d, geo_vs, geo_vts, rect
rect = None
pred_fn = os.path.join(grip_dir, 'pred.new.io')
g_predictor = Face.load_predictor(pred_fn) #, cutOff=15)
reference_shape = g_predictor['ref_shape']
size = reference_shape.shape[0]
geo_vs = np.zeros((size, 3), dtype=np.float32)
geo_vs[:size, :2] = reference_shape
geo_vts = np.zeros((size, 2), dtype=np.float32)
geo_vts[:size] = reference_shape + 0.5
geo_ts = np.array([[1, 0, 0, 0], [0, 1, 0, 1000], [0, 0, 1, 0]],
dtype=np.float32)
geo_fs = Face.triangulate_2D(reference_shape)
geo_bs = []
for p0, p1, p2 in geo_fs:
geo_bs.append((p0, p1))
geo_bs.append((p1, p2))
geo_bs.append((p2, p0))
reference_3d = np.zeros((reference_shape.shape[0], 3), dtype=np.float32)
reference_3d[:, :2] = reference_shape * [100, 100]
img_vs = np.array([[0, 0, 0], [640, 0, 0], [640, 480, 0], [0, 480, 0]],
dtype=np.float32)
img_vts = np.array([[0, 1], [1, 1], [1, 0], [0, 0]], dtype=np.float32)
img_fs = np.array([[0, 1, 2, 3]], dtype=np.int32)
img_ts = np.array([[1, 0, 0, 0], [0, 1, 0, 1000], [0, 0, 1, 0]],
dtype=np.float32)
geo_mesh = GLMeshes(names=['geo_mesh'],
verts=[geo_vs],
faces=[geo_fs],
transforms=[geo_ts],
bones=[geo_bs],
vts=[geo_vts])
img_mesh = GLMeshes(names=['img_mesh'],
verts=[img_vs],
faces=[img_fs],
transforms=[img_ts],
bones=[None],
vts=[img_vts])
kinect = freenect.init()
tilt, roll = 0, 0
if 1:
kdev = freenect.open_device(kinect, 0)
freenect.set_led(kdev, 0) # turn off LED
freenect.set_tilt_degs(kdev, 25)
kstate = freenect.get_tilt_state(kdev)
freenect.update_tilt_state(kdev)
tilt_angle, tilt_status = kstate.tilt_angle, kstate.tilt_status
ax, ay, az = kstate.accelerometer_x, kstate.accelerometer_y, kstate.accelerometer_z
#bottom facing down: (85, 743, 369, 52, 0)
#right side down: (916, 71, 96, 112, 0)
#front side down: (52, 63, -863, -128, 0)
freenect.close_device(kdev)
y_axis = np.array((ax, ay, az), dtype=np.float32)
y_axis = y_axis / np.linalg.norm(y_axis)
roll = np.degrees(np.arctan2(ax, ay))
tilt = -np.degrees(np.arctan2(az, (ax**2 + ay**2)**0.5))
fovX = 62.0
pan_tilt_roll = (0, tilt, roll)
tx_ty_tz = (0, 1000, 6000)
P = Calibrate.composeP_fromData((fovX, ), (pan_tilt_roll), (tx_ty_tz), 0)
global g_camera_rays, g_camera_mat
h, w = 480 // 2, 640 // 2
coord, pix_coord = make_coords(h, w)
#P = np.eye(3,4,dtype=np.float32)
#P[0,0] = P[1,1] = 2.0
k1, k2 = 0, 0
g_camera_mat = Calibrate.makeMat(P, (k1, k2), [w, h])
K, RT, P, ks, T, wh = g_camera_mat
coord_undist = coord.copy()
Calibrate.undistort_points_mat(coord.reshape(-1, 2), g_camera_mat,
coord_undist.reshape(-1, 2))
g_camera_rays = np.dot(coord_undist,
RT[:2, :3]) # ray directions (unnormalized)
g_camera_rays -= np.dot([-K[0, 2], -K[1, 2], K[0, 0]], RT[:3, :3])
g_camera_rays /= (np.sum(g_camera_rays**2, axis=-1)**0.5).reshape(
h, w, 1) # normalized ray directions
names = ['kinect']
vs = [np.zeros((h * w, 3), dtype=np.float32)]
ts = [np.eye(3, 4, dtype=np.float32)]
vts = [pix_coord * (1.0 / w, 1.0 / h)]
faces = [make_faces(h, w)]
mats = None
geom_mesh = GLMeshes(names=names,
verts=vs,
faces=faces,
transforms=ts,
vts=vts)
layers = {
'geom_mesh': geom_mesh,
'geo_mesh': geo_mesh,
'img_mesh': img_mesh
}
QGLViewer.makeViewer(layers=layers,
mats=mats,
callback=cb,
timeRange=(0, 10000))
c3d_points.vertices = x3ds
c3d_points.colour = [1, 0, 0, 1] # red
QApp.app.updateGL()
if __name__ == '__main__':
import UI
from UI import QApp, QGLViewer
from UI import GLMeshes
import os, sys
if len(sys.argv) > 1:
filename = sys.argv[1]
geom_dict = flatten_OBJ_and_x10(read_OBJ(filename))
ted_geom = GLMeshes(['ted'], [geom_dict['v']], [geom_dict['tris']],
vts=[geom_dict['vt']],
transforms=[np.eye(3, 4)])
primitives = [ted_geom]
QGLViewer.makeViewer(primitives=primitives)
exit()
from GCore import Calibrate
import MovieReader
import C3D
global ted_geom, ted_geom2, ted_shape, tony_geom, tony_shape, tony_geom2, tony_obj, ted_obj, diff_geom, c3d_frames
global tony_shape_vector, tony_shape_mat, ted_lo_rest, ted_lo_mat, c3d_points
global md, movies
ted_dir = os.path.join(os.environ['GRIP_DATA'], 'ted')
def main():
from UI import QGLViewer
from UI import GLMeshes, GLPoints3D
global g_setting_frame
g_setting_frame = False
# static data
global g_webcam, g_md, g_rbfn, g_predictor, g_head_pan_shape, g_head_tilt_shape
# runtime options and state
global g_prev_smooth_shape, g_prev_vs, g_hmc_boot, g_neutral_corrective_shape, g_settle, g_head_pan_tilt_roll, g_smooth_pose
global g_directory, g_TIS_server, g_mode, g_frame
g_TIS_server = SocketServer.SocketServer()
g_mode, g_frame = 0,{}
grip_dir = os.environ['GRIP_DATA']
g_directory = grip_dir
g_webcam,g_md = None,None
g_prev_vs, g_prev_smooth_shape = None,None
g_hmc_boot = None
#clear_neutral()
g_neutral_corrective_shape = IO.load(os.path.join(g_directory,'neutral.out'))[1]
g_settle = -1
g_head_pan_tilt_roll = None
g_smooth_pose = {}
aam = IO.load(os.path.join(g_directory,'aam.out'))[1]
if 0:
svt = np.float32(aam['shapes']).reshape(-1,140)
svt = np.dot(aam['shapes_u'],aam['shapes_s'].reshape(-1,1)*aam['shapes_vt'])
svt = aam['shapes_s'].reshape(-1,1)*aam['shapes_vt']
tmp = svt.reshape(svt.shape[0],-1,2)
Sx,Sy = tmp[:,:,0],tmp[:,:,1]
tmp = np.dot(np.dot(Sy.T,np.dot(Sx,Sx.T)),Sy)
u,s,vt = np.linalg.svd(tmp, full_matrices=False)
print s
g_head_pan_shape = np.zeros((svt.shape[1]/2,2),dtype=np.float32)
g_head_tilt_shape = np.zeros((svt.shape[1]/2,2),dtype=np.float32)
g_head_pan_shape[:,0] = g_head_tilt_shape[:,1] = vt[0]
print np.sum(g_head_pan_shape * aam['shapes_vt'][0].reshape(-1,2))
print np.sum(g_head_tilt_shape * aam['shapes_vt'][1].reshape(-1,2))
g_head_pan_shape = aam['shapes_vt'][0].reshape(-1,2)
g_head_tilt_shape = aam['shapes_vt'][1].reshape(-1,2)
g_head_tilt_shape = g_head_pan_shape[:,::-1]*np.float32([1,-1])
print np.sum(g_head_pan_shape*g_head_tilt_shape)
g_head_pan_shape *= np.linalg.norm(g_head_pan_shape)**-0.5
g_head_tilt_shape *= np.linalg.norm(g_head_tilt_shape)**-0.5
if np.sum(g_head_pan_shape[:,0] < 1): g_head_pan_shape = -g_head_pan_shape
if np.sum(g_head_tilt_shape[:,1] > 1): g_head_tilt_shape = -g_head_tilt_shape
#print np.sum(g_head_pan_shape * g_head_tilt_shape)
#print np.dot(g_head_pan_shape[:,0],g_head_tilt_shape[:,1])
g_predictor = Face.load_predictor(os.path.join(g_directory,'train.out'))
rbfn_filename = os.path.join(g_directory,'rbfn.out')
g_rbfn = IO.load(rbfn_filename)[1]
#g_rbfn = convert_rbfn(rbfn_in_filename)
#IO.save(rbfn_filename, g_rbfn)
ref_shape = g_predictor['ref_shape']
cx,cy = np.mean(ref_shape,axis=0)
vx,vy = (np.var(ref_shape,axis=0)**0.5) * 2.5
geo_bs = []
ref_fs = Face.triangulate_2D(ref_shape)
for p0,p1,p2 in ref_fs:
geo_bs.append((p0,p1))
geo_bs.append((p1,p2))
geo_bs.append((p2,p0))
geo_vs = np.zeros((len(ref_shape),3), dtype=np.float32)
geo_fs = []
geo_ts = np.float32([[1,0,0,0],[0,1,0,1000],[0,0,1,0]])
geo_vts = np.zeros_like(ref_shape)
img_vs = np.float32([[-1000,-1000,0],[1000,-1000,0],[1000,1000,0],[-1000,1000,0]])
img_fs = np.int32([[0,1,2,3]])
img_ts = np.float32([[1,0,0,0],[0,1,0,1000],[0,0,1,0]])
img_vts = np.float32([[0,1],[1,1],[1,0],[0,0]])
markup_mesh = GLPoints3D(vertices=geo_vs, edges=np.int32(geo_bs), names=[], colour=[0,1,0,1],edgeColour=[1,1,1,1])
geo_mesh = GLMeshes(names=['geo_mesh'],verts=[geo_vs],faces=[geo_fs],transforms=[geo_ts],bones=[geo_bs], vts=[geo_vts], colour=[1,0,0,1])
image_mesh = GLMeshes(names=['image_mesh'],verts=[img_vs],faces=[img_fs],transforms=[img_ts],vts=[img_vts])
global g_bs_vs, g_bs_shape_mat, g_bs_fs, g_bs_vts, g_bs_shape_mat_T
bs_dict = IO.load(os.path.join(g_directory,'harpy_ma.out'))[1]['blendShapes']['Harpy_cFace_GEOShape']
obj_scale = 10.0
g_bs_vs = np.float32(bs_dict['vs']*obj_scale)
bs_dict['pts'] = [b*obj_scale for b in bs_dict['pts']]
g_bs_fs = bs_dict['fs'] # warning: mix of quads and triangles :-(
assert bs_dict['vts'].keys() == range(len(bs_dict['vts'].keys()))
g_bs_vts = bs_dict['vts'].values()
g_bs_ts = np.float32([[1,0,0,800],[0,1,0,-600],[0,0,1,300]])
bs_mesh = GLMeshes(names=['bs_mesh'],verts=[g_bs_vs],faces=[g_bs_fs],transforms=[g_bs_ts],vts=[g_bs_vts],visible=False)
rbfn_groups, rbfn_slider_splits, rbfn_slider_names, rbfn_marker_names = extract_groups(g_rbfn)
slider_names = [(x[8:-2]+'.translateY' if x.startswith('get_ty') else x) for x in bs_dict['wt_names']]
try:
slider_order = [slider_names.index(x) for x in rbfn_slider_names]
except Exception as e:
print 'error',e
slider_order = []
g_bs_shape_mat = bs_dict['matrix'] = np.zeros((len(bs_dict['pts']), len(bs_dict['vs']), 3),dtype=np.float32)
for m,ct,pt in zip(g_bs_shape_mat,bs_dict['cts'],bs_dict['pts']): m[ct] = pt
g_bs_shape_mat = g_bs_shape_mat[slider_order]
g_bs_shape_mat_T = g_bs_shape_mat.transpose(1,2,0).copy()
layers = {'image_mesh':image_mesh,'geo_mesh':geo_mesh,'bs_mesh':bs_mesh,'markup_mesh':markup_mesh}
app,win = QGLViewer.makeApp()
outliner = win.qoutliner
#for gi,geo in enumerate(layers.keys()): outliner.addItem(geo, data='_OBJ_'+geo, index=gi)
State.setKey('ui',{'type':'ui','attrs':{\
'harpy_xoffset':300.0,'show_harpy':True,'rotate':0,'mirroring':False,'unreal':True,'streaming_TIS':False,\
'using_webcam':False,'HMC_mode':True,'booting':True,'filtering':True,'setting_neutral':True,'debugging':False, \
'webcam_index':0,'cam_offset':700,'cam_fps':50,'cam_width':1280,'cam_height':720, 'movie_filename':''}})
if True: # running on deployed face machine at 720p50
State.setKey('/root/ui',{'type':'ui','attrs':{\
'harpy_xoffset':300.0,'show_harpy':False,'rotate':1,'mirroring':False,'unreal':True,'streaming_TIS':False,\
'using_webcam':True,'HMC_mode':True,'booting':True,'filtering':True,'setting_neutral':True,'debugging':False, \
'webcam_index':0,'cam_offset':700,'cam_fps':50,'cam_width':1280,'cam_height':720, 'movie_filename':''}})
win.setFields('ui', [
('show_harpy', 'show_harpy','Whether to display the Harpy','bool', False),
('harpy_xoffset', 'xoffset', 'Pixels to offset Harpy to right', 'float', 300.0),
('rotate', 'rotation','Rotate image 0=up,1=left,2=down,3=right,-1=any angle','int', 0),
('mirroring', 'mirror', 'Show reversed', 'bool', False),
('unreal', 'unreal', 'Whether to connect to unreal', 'bool', True),
#('streaming_TIS', 'streaming_TIS', 'Whether currently streaming', 'bool', False),
('using_webcam', 'webcam', 'Whether using the webcam', 'bool', False),
('HMC_mode', 'HMC_mode','Boot every frame', 'bool', True),
('booting', 'boot', 'Boot at next chance', 'bool', True),
('filtering', 'filter', 'Whether to filter noise', 'bool', True),
('setting_neutral', 'neutral', 'Set neutral at next chance', 'bool', False),
('debugging', 'debug', 'Show rbfn input for debugging', 'bool', False),
('webcam_index', 'camindex', 'The index of the webcam', 'int', 0),
('cam_offset', 'camoffset','The offset of the webcam', 'int', 700),
('cam_fps', 'fps', 'The frame rate of the webcam', 'int', 50),
('cam_width', 'width', 'The width of the webcam image', 'int', 1280),
('cam_height', 'height', 'The height of the webcam image', 'int', 720),
('movie_filename', 'movie', 'The filename of the movie', 'string', ''),
])
slider_names = sorted(g_rbfn['slider_names'])
win.setFields('sliders', [(sn,sn,'Slider %d'%si,'float',0.0) for si,sn in enumerate(slider_names)])
State.setKey('/root/sliders', {'type':'sliders','attrs':{sn:0.0 for sn in slider_names}})
outliner.set_root('/root')
#outliner.addItem('sliders', data='sliders', index=1)
win.outliner.raise_()
#win.select('ui')
QApp.app.dirtyCB = dirty_cb
QApp.app.addMenuItem({'menu':'&File','item':'Import &movie','tip':'Import a movie file','cmd':import_movie})
QApp.app.addMenuItem({'menu':'&Edit','item':'Retrain rbfn','tip':'Train the rbfn','cmd':retrain_RBFN})
QApp.app.addMenuItem({'menu':'&Edit','item':'Retrain rbfn (no linear)','tip':'Train the rbfn with no linear part','cmd':retrain_RBFN_no_linear})
QApp.app.addMenuItem({'menu':'&Edit','item':'Retrack refresh rbfn','tip':'Refresh the rbfn','cmd':retrack_refresh_rbfn})
QApp.app.addMenuItem({'menu':'&Edit','item':'Retrack remap rbfn','tip':'Rebuild the rbfn','cmd':retrack_remap_rbfn})
QApp.app.addMenuItem({'menu':'&File','item':'Export rbfn','tip':'Export the rbfn','cmd':export_rbfn})
State.clearUndoStack()
QGLViewer.makeViewer(appName='StreamVideoTrack',timeRange=(0,100), callback=setFrame_cb, keyCallback=keypress_cb, layers=layers, mats=[Calibrate.makeMat(Calibrate.composeRT(np.eye(3)*[10,10,1],[0,1000,6000],1000),[0,0],[1920,1080])], camera_ids=['RBFN'])
# Ensure the server has stopped when program terminates
g_TIS_server.Stop()