size = len(template_vs)
geo_vs = np.zeros((size,3), dtype=np.float32)
geo_fs = []
geo_ts = np.array([[1,0,0,0],[0,1,0,1000],[0,0,1,0]], dtype=np.float32)
geo_vts = [[0,0]]*size
img_vs = [[-1000,-1000,0],[1000,-1000,0],[1000,1000,0],[-1000,1000,0]]
img_fs = [[0,1,2,3]]
img_ts = np.array([[1,0,0,0],[0,1,0,1000],[0,0,1,0]], dtype=np.float32)
img_vts = [[0,1],[1,1],[1,0],[0,0]]
geo_mesh = GLMeshes(names=['geo_mesh'],verts=[geo_vs],faces=[geo_fs],transforms=[geo_ts],bones=[geo_bs], vts=[geo_vts], colour=[1,0,0,1])
image_mesh = GLMeshes(names=['image_mesh'],verts=[img_vs],faces=[img_fs],transforms=[img_ts],bones = [[]], vts=[img_vts])
layers = {'image_mesh':image_mesh,'geo_mesh':geo_mesh}
if 0:
ref_vs = np.zeros((size,3), dtype=np.float32)
ref_vs[:,:2] = template_vs
ref_mesh = GLMeshes(names=['ref_mesh'],verts=[ref_vs],faces=[geo_fs],transforms=[geo_ts],bones=[geo_bs], vts=[geo_vts], colour=[0,1,0,1])
layers['ref_mesh'] = ref_mesh
app,win = QGLViewer.makeApp(appName='Imaginarium FaceTrack')
QApp.app.addMenuItem({'menu':'&File','item':'Import &movie','tip':'Import a movie file','cmd':import_movie_cb})
QApp.app.qoutliner.set_root('')
#for gi,geo in enumerate(layers.keys()): outliner.addItem(geo, data='_OBJ_'+geo, index=gi)
global g_aam_model_indices,g_aam_model_weights
g_aam_model_indices,g_aam_model_weights = Face.make_sample_model(template_vs, ref_fs, grid_size=1.0)
QGLViewer.makeViewer(timeRange=(0,8000), callback=set_frame_cb2, keyCallback=keyCB, layers=layers)
anim_dict = extract_animation(fbx, skel_dict)
#print skel_dict['jointNames']
skel_mesh = GLMeshes(**skelDictToMesh(skel_dict))
global g_anim_dict, g_skel_dict
g_anim_dict = anim_dict
g_skel_dict = skel_dict
#State.setKey('/doc',payload)
#fbx = decode(payload)
#State.setKey('/fbx',fbx)
#for k,v in fbx.iteritems(): State.setKey(k,v)
from UI import QApp, QGLViewer, GLMeshes, GLPoints3D
app, win = QGLViewer.makeApp(appName='Imaginarium FBX')
#outliner = QApp.app.qobjects
#for gi,(key,value) in enumerate(fbx.items()):
# outliner.addItem(str(key)+'='+repr(value)[:200], data=str(key), index=gi)
display_nodes = skel_to_nodes(skel_dict)
#print zip(*display_nodes)[1]
#for gi,(disp,key) in enumerate(display_nodes): outliner.addItem(disp, data='_OBJ_'+key, index=gi)
#global g_fields
QApp.fields = dict([(k, sorted(v.items()))
for k, v in g_fields.iteritems()])
for gi, (k, v) in enumerate(fbx['objs'].items()):
State.setKey('/fbx/' + str(k), v)
QApp.app.qoutliner.set_root('/fbx')
#for gi,key in enumerate(sorted(State.allKeys())): outliner.addItem(key+'='+repr(State.getKey(key))[:80], data=key, index=gi)
def main():
from UI import QGLViewer
from UI import GLMeshes, GLPoints3D
global g_setting_frame
g_setting_frame = False
# static data
global g_webcam, g_md, g_rbfn, g_predictor, g_head_pan_shape, g_head_tilt_shape
# runtime options and state
global g_prev_smooth_shape, g_prev_vs, g_hmc_boot, g_neutral_corrective_shape, g_settle, g_head_pan_tilt_roll, g_smooth_pose
global g_directory, g_TIS_server, g_mode, g_frame
g_TIS_server = SocketServer.SocketServer()
g_mode, g_frame = 0,{}
grip_dir = os.environ['GRIP_DATA']
g_directory = grip_dir
g_webcam,g_md = None,None
g_prev_vs, g_prev_smooth_shape = None,None
g_hmc_boot = None
#clear_neutral()
g_neutral_corrective_shape = IO.load(os.path.join(g_directory,'neutral.out'))[1]
g_settle = -1
g_head_pan_tilt_roll = None
g_smooth_pose = {}
aam = IO.load(os.path.join(g_directory,'aam.out'))[1]
if 0:
svt = np.float32(aam['shapes']).reshape(-1,140)
svt = np.dot(aam['shapes_u'],aam['shapes_s'].reshape(-1,1)*aam['shapes_vt'])
svt = aam['shapes_s'].reshape(-1,1)*aam['shapes_vt']
tmp = svt.reshape(svt.shape[0],-1,2)
Sx,Sy = tmp[:,:,0],tmp[:,:,1]
tmp = np.dot(np.dot(Sy.T,np.dot(Sx,Sx.T)),Sy)
u,s,vt = np.linalg.svd(tmp, full_matrices=False)
print s
g_head_pan_shape = np.zeros((svt.shape[1]/2,2),dtype=np.float32)
g_head_tilt_shape = np.zeros((svt.shape[1]/2,2),dtype=np.float32)
g_head_pan_shape[:,0] = g_head_tilt_shape[:,1] = vt[0]
print np.sum(g_head_pan_shape * aam['shapes_vt'][0].reshape(-1,2))
print np.sum(g_head_tilt_shape * aam['shapes_vt'][1].reshape(-1,2))
g_head_pan_shape = aam['shapes_vt'][0].reshape(-1,2)
g_head_tilt_shape = aam['shapes_vt'][1].reshape(-1,2)
g_head_tilt_shape = g_head_pan_shape[:,::-1]*np.float32([1,-1])
print np.sum(g_head_pan_shape*g_head_tilt_shape)
g_head_pan_shape *= np.linalg.norm(g_head_pan_shape)**-0.5
g_head_tilt_shape *= np.linalg.norm(g_head_tilt_shape)**-0.5
if np.sum(g_head_pan_shape[:,0] < 1): g_head_pan_shape = -g_head_pan_shape
if np.sum(g_head_tilt_shape[:,1] > 1): g_head_tilt_shape = -g_head_tilt_shape
#print np.sum(g_head_pan_shape * g_head_tilt_shape)
#print np.dot(g_head_pan_shape[:,0],g_head_tilt_shape[:,1])
g_predictor = Face.load_predictor(os.path.join(g_directory,'train.out'))
rbfn_filename = os.path.join(g_directory,'rbfn.out')
g_rbfn = IO.load(rbfn_filename)[1]
#g_rbfn = convert_rbfn(rbfn_in_filename)
#IO.save(rbfn_filename, g_rbfn)
ref_shape = g_predictor['ref_shape']
cx,cy = np.mean(ref_shape,axis=0)
vx,vy = (np.var(ref_shape,axis=0)**0.5) * 2.5
geo_bs = []
ref_fs = Face.triangulate_2D(ref_shape)
for p0,p1,p2 in ref_fs:
geo_bs.append((p0,p1))
geo_bs.append((p1,p2))
geo_bs.append((p2,p0))
geo_vs = np.zeros((len(ref_shape),3), dtype=np.float32)
geo_fs = []
geo_ts = np.float32([[1,0,0,0],[0,1,0,1000],[0,0,1,0]])
geo_vts = np.zeros_like(ref_shape)
img_vs = np.float32([[-1000,-1000,0],[1000,-1000,0],[1000,1000,0],[-1000,1000,0]])
img_fs = np.int32([[0,1,2,3]])
img_ts = np.float32([[1,0,0,0],[0,1,0,1000],[0,0,1,0]])
img_vts = np.float32([[0,1],[1,1],[1,0],[0,0]])
markup_mesh = GLPoints3D(vertices=geo_vs, edges=np.int32(geo_bs), names=[], colour=[0,1,0,1],edgeColour=[1,1,1,1])
geo_mesh = GLMeshes(names=['geo_mesh'],verts=[geo_vs],faces=[geo_fs],transforms=[geo_ts],bones=[geo_bs], vts=[geo_vts], colour=[1,0,0,1])
image_mesh = GLMeshes(names=['image_mesh'],verts=[img_vs],faces=[img_fs],transforms=[img_ts],vts=[img_vts])
global g_bs_vs, g_bs_shape_mat, g_bs_fs, g_bs_vts, g_bs_shape_mat_T
bs_dict = IO.load(os.path.join(g_directory,'harpy_ma.out'))[1]['blendShapes']['Harpy_cFace_GEOShape']
obj_scale = 10.0
g_bs_vs = np.float32(bs_dict['vs']*obj_scale)
bs_dict['pts'] = [b*obj_scale for b in bs_dict['pts']]
g_bs_fs = bs_dict['fs'] # warning: mix of quads and triangles :-(
assert bs_dict['vts'].keys() == range(len(bs_dict['vts'].keys()))
g_bs_vts = bs_dict['vts'].values()
g_bs_ts = np.float32([[1,0,0,800],[0,1,0,-600],[0,0,1,300]])
bs_mesh = GLMeshes(names=['bs_mesh'],verts=[g_bs_vs],faces=[g_bs_fs],transforms=[g_bs_ts],vts=[g_bs_vts],visible=False)
rbfn_groups, rbfn_slider_splits, rbfn_slider_names, rbfn_marker_names = extract_groups(g_rbfn)
slider_names = [(x[8:-2]+'.translateY' if x.startswith('get_ty') else x) for x in bs_dict['wt_names']]
try:
slider_order = [slider_names.index(x) for x in rbfn_slider_names]
except Exception as e:
print 'error',e
slider_order = []
g_bs_shape_mat = bs_dict['matrix'] = np.zeros((len(bs_dict['pts']), len(bs_dict['vs']), 3),dtype=np.float32)
for m,ct,pt in zip(g_bs_shape_mat,bs_dict['cts'],bs_dict['pts']): m[ct] = pt
g_bs_shape_mat = g_bs_shape_mat[slider_order]
g_bs_shape_mat_T = g_bs_shape_mat.transpose(1,2,0).copy()
layers = {'image_mesh':image_mesh,'geo_mesh':geo_mesh,'bs_mesh':bs_mesh,'markup_mesh':markup_mesh}
app,win = QGLViewer.makeApp()
outliner = win.qoutliner
#for gi,geo in enumerate(layers.keys()): outliner.addItem(geo, data='_OBJ_'+geo, index=gi)
State.setKey('ui',{'type':'ui','attrs':{\
'harpy_xoffset':300.0,'show_harpy':True,'rotate':0,'mirroring':False,'unreal':True,'streaming_TIS':False,\
'using_webcam':False,'HMC_mode':True,'booting':True,'filtering':True,'setting_neutral':True,'debugging':False, \
'webcam_index':0,'cam_offset':700,'cam_fps':50,'cam_width':1280,'cam_height':720, 'movie_filename':''}})
if True: # running on deployed face machine at 720p50
State.setKey('/root/ui',{'type':'ui','attrs':{\
'harpy_xoffset':300.0,'show_harpy':False,'rotate':1,'mirroring':False,'unreal':True,'streaming_TIS':False,\
'using_webcam':True,'HMC_mode':True,'booting':True,'filtering':True,'setting_neutral':True,'debugging':False, \
'webcam_index':0,'cam_offset':700,'cam_fps':50,'cam_width':1280,'cam_height':720, 'movie_filename':''}})
win.setFields('ui', [
('show_harpy', 'show_harpy','Whether to display the Harpy','bool', False),
('harpy_xoffset', 'xoffset', 'Pixels to offset Harpy to right', 'float', 300.0),
('rotate', 'rotation','Rotate image 0=up,1=left,2=down,3=right,-1=any angle','int', 0),
('mirroring', 'mirror', 'Show reversed', 'bool', False),
('unreal', 'unreal', 'Whether to connect to unreal', 'bool', True),
#('streaming_TIS', 'streaming_TIS', 'Whether currently streaming', 'bool', False),
('using_webcam', 'webcam', 'Whether using the webcam', 'bool', False),
('HMC_mode', 'HMC_mode','Boot every frame', 'bool', True),
('booting', 'boot', 'Boot at next chance', 'bool', True),
('filtering', 'filter', 'Whether to filter noise', 'bool', True),
('setting_neutral', 'neutral', 'Set neutral at next chance', 'bool', False),
('debugging', 'debug', 'Show rbfn input for debugging', 'bool', False),
('webcam_index', 'camindex', 'The index of the webcam', 'int', 0),
('cam_offset', 'camoffset','The offset of the webcam', 'int', 700),
('cam_fps', 'fps', 'The frame rate of the webcam', 'int', 50),
('cam_width', 'width', 'The width of the webcam image', 'int', 1280),
('cam_height', 'height', 'The height of the webcam image', 'int', 720),
('movie_filename', 'movie', 'The filename of the movie', 'string', ''),
])
slider_names = sorted(g_rbfn['slider_names'])
win.setFields('sliders', [(sn,sn,'Slider %d'%si,'float',0.0) for si,sn in enumerate(slider_names)])
State.setKey('/root/sliders', {'type':'sliders','attrs':{sn:0.0 for sn in slider_names}})
outliner.set_root('/root')
#outliner.addItem('sliders', data='sliders', index=1)
win.outliner.raise_()
#win.select('ui')
QApp.app.dirtyCB = dirty_cb
QApp.app.addMenuItem({'menu':'&File','item':'Import &movie','tip':'Import a movie file','cmd':import_movie})
QApp.app.addMenuItem({'menu':'&Edit','item':'Retrain rbfn','tip':'Train the rbfn','cmd':retrain_RBFN})
QApp.app.addMenuItem({'menu':'&Edit','item':'Retrain rbfn (no linear)','tip':'Train the rbfn with no linear part','cmd':retrain_RBFN_no_linear})
QApp.app.addMenuItem({'menu':'&Edit','item':'Retrack refresh rbfn','tip':'Refresh the rbfn','cmd':retrack_refresh_rbfn})
QApp.app.addMenuItem({'menu':'&Edit','item':'Retrack remap rbfn','tip':'Rebuild the rbfn','cmd':retrack_remap_rbfn})
QApp.app.addMenuItem({'menu':'&File','item':'Export rbfn','tip':'Export the rbfn','cmd':export_rbfn})
State.clearUndoStack()
QGLViewer.makeViewer(appName='StreamVideoTrack',timeRange=(0,100), callback=setFrame_cb, keyCallback=keypress_cb, layers=layers, mats=[Calibrate.makeMat(Calibrate.composeRT(np.eye(3)*[10,10,1],[0,1000,6000],1000),[0,0],[1920,1080])], camera_ids=['RBFN'])
# Ensure the server has stopped when program terminates
g_TIS_server.Stop()