def compute_pre_transformation(self, file_name):
translation = [0, 0, 0]
if self.config['pre-align']['align_center_of_mass']:
# hack to avoid how the objimporter deals with multiple polydata
pd = Utils3D.multi_read_surface(file_name)
if pd.GetNumberOfPoints() < 1:
print('Could not read', file_name)
return None
vtk_cm = vtk.vtkCenterOfMass()
vtk_cm.SetInputData(pd)
vtk_cm.SetUseScalarsAsWeights(False)
vtk_cm.Update()
cm = vtk_cm.GetCenter()
translation = [-cm[0], -cm[1], -cm[2]]
t = vtk.vtkTransform()
t.Identity()
rx = self.config['pre-align']['rot_x']
ry = self.config['pre-align']['rot_y']
rz = self.config['pre-align']['rot_z']
# Scale is handling by doing magic with the view frustrum elsewhere
# s = self.config['pre-align']['scale']
# t.Scale(s, s, s)
t.RotateY(ry)
t.RotateX(rx)
t.RotateZ(rz)
t.Translate(translation)
t.Update()
return t
def process_files_in_dir(config, dir_name):
print('Processing files in ', dir_name)
names = Utils3D.get_mesh_files_in_dir(dir_name)
print('Processing ', len(names), ' meshes')
dm = deepmvlm.DeepMVLM(config)
for file_name in names:
print('Processing ', file_name)
name_lm_txt = os.path.splitext(file_name)[0] + '_landmarks.txt'
landmarks = dm.predict_one_file(file_name)
dm.write_landmarks_as_text(landmarks, name_lm_txt)
def predict_one_file(self, file_name):
render_3d = Render3D(self.config)
image_stack, transform_stack = render_3d.render_3d_file(file_name)
predict_2d = Predict2D(self.config, self.model, self.device)
heatmap_maxima = predict_2d.predict_heatmaps_from_images(image_stack)
u3d = Utils3D(self.config)
u3d.heatmap_maxima = heatmap_maxima
u3d.transformations_3d = transform_stack
u3d.compute_lines_from_heatmap_maxima()
# u3d.visualise_one_landmark_lines(65)
u3d.compute_all_landmarks_from_view_lines()
u3d.project_landmarks_to_surface(file_name)
return u3d.landmarks
def predict_one_subject(config, file_name):
device, model = get_device_and_load_model(config)
render_3d = Render3D(config)
image_stack, transform_stack = render_3d.render_3d_file(file_name)
predict_2d = Predict2D(config, model, device)
heatmap_maxima = predict_2d.predict_heatmaps_from_images(image_stack)
u3d = Utils3D(config)
u3d.heatmap_maxima = heatmap_maxima
# u3d.read_heatmap_maxima()
# u3d.read_3d_transformations()
u3d.transformations_3d = transform_stack
u3d.compute_lines_from_heatmap_maxima()
# u3d.visualise_one_landmark_lines(40, 'saved/temp/DeepMVLM_DTU3D/0904_104414')
u3d.compute_all_landmarks_from_view_lines()
u3d.project_landmarks_to_surface(file_name)
# u3d.write_landmarks_as_vtk_points()
return u3d.landmarks
def visualise_mesh_and_landmarks(mesh_name, landmarks=None):
print(
'################### utils3d/ render3d/ visualise mesh and landmarks ################### \n'
)
file_type = os.path.splitext(mesh_name)[1]
win_size = 512
ren = vtk.vtkRenderer()
ren.SetBackground(1, 1, 1)
# Initialize RenderWindow
ren_win = vtk.vtkRenderWindow()
ren_win.AddRenderer(ren)
ren_win.SetSize(win_size, win_size)
file_read = False
if file_type == ".obj":
#print('############# if file type == .obj ############# \n')
mtl_name = os.path.splitext(mesh_name)[0] + '.mtl'
# only do this for textured files
if os.path.isfile(mtl_name):
obj_dir = os.path.dirname(mesh_name)
obj_in = vtk.vtkOBJImporter()
obj_in.SetFileName(mesh_name)
if os.path.isfile(mtl_name):
obj_in.SetFileNameMTL(mtl_name)
obj_in.SetTexturePath(obj_dir)
obj_in.Update()
obj_in.SetRenderWindow(ren_win)
obj_in.Update()
props = vtk.vtkProperty()
props.SetColor(1, 1, 1)
props.SetDiffuse(0)
props.SetSpecular(0)
props.SetAmbient(1)
actors = ren.GetActors()
actors.InitTraversal()
actor = actors.GetNextItem()
while actor:
actor.SetProperty(props)
actor = actors.GetNextItem()
del props
file_read = True
if not file_read and file_type in [
".vtk", ".stl", ".ply", ".wrl", ".obj"
]:
#print('############## if not file_read and file_type in ############################### \n ')
pd = Utils3D.multi_read_surface(mesh_name)
if pd.GetNumberOfPoints() < 1:
print('Could not read', mesh_name)
return None
texture_img = Utils3D.multi_read_texture(mesh_name)
if texture_img is not None:
#print('################ if texture_img is not None. Set 1 ##################### \n')
pd.GetPointData().SetScalars(None)
texture = vtk.vtkTexture()
texture.SetInterpolate(1)
texture.SetQualityTo32Bit()
texture.SetInputData(texture_img)
mapper = vtk.vtkPolyDataMapper()
mapper.SetInputData(pd)
actor_text = vtk.vtkActor()
actor_text.SetMapper(mapper)
if texture_img is not None:
#print('################ if texture_img is not None. Set 2 ##################### \n')
actor_text.SetTexture(texture)
actor_text.GetProperty().SetColor(1, 1, 1)
actor_text.GetProperty().SetAmbient(1.0)
actor_text.GetProperty().SetSpecular(0)
actor_text.GetProperty().SetDiffuse(0)
ren.AddActor(actor_text)
if landmarks is not None:
#print('########### if landmarks is not None ################ \n')
lm_pd = Render3D.get_landmarks_as_spheres(landmarks)
mapper = vtk.vtkPolyDataMapper()
mapper.SetInputData(lm_pd)
actor_lm = vtk.vtkActor()
actor_lm.SetMapper(mapper)
actor_lm.GetProperty().SetColor(0, 0, 1)
ren.AddActor(actor_lm)
# ren.GetActiveCamera().SetPosition(0, 0, 1)
# ren.GetActiveCamera().SetFocalPoint(0, 0, 0)
# ren.GetActiveCamera().SetViewUp(0, 1, 0)
# ren.GetActiveCamera().SetParallelProjection(1)
iren = vtk.vtkRenderWindowInteractor()
style = vtk.vtkInteractorStyleTrackballCamera()
iren.SetInteractorStyle(style)
iren.SetRenderWindow(ren_win)
print(
'############## ren_win.Render & iren.Start ####################### \n '
)
ren_win.Render()
iren.Start()
def render_3d_multi_rgb_geometry_depth(self, transform_stack, file_name):
write_image_files = self.config['process_3d']['write_renderings']
off_screen_rendering = self.config['process_3d'][
'off_screen_rendering']
n_views = self.config['data_loader']['args']['n_views']
img_size = self.config['data_loader']['args']['image_size']
win_size = img_size
slack = 5
start = time.time()
self.logger.debug('Rendering')
n_channels = 5 # 3 for RGB, 1 for depth and 1 for geometry
image_stack = np.zeros((n_views, win_size, win_size, n_channels),
dtype=np.float32)
pd = Utils3D.multi_read_surface(file_name)
if pd.GetNumberOfPoints() < 1:
print('Could not read', file_name)
return None
pd = self.apply_pre_transformation(pd)
texture_img = Utils3D.multi_read_texture(file_name)
if texture_img is not None:
pd.GetPointData().SetScalars(None)
texture = vtk.vtkTexture()
texture.SetInterpolate(1)
texture.SetQualityTo32Bit()
texture.SetInputData(texture_img)
# Initialize Camera
ren = vtk.vtkRenderer()
ren.SetBackground(1, 1, 1)
ren.GetActiveCamera().SetPosition(0, 0, 1)
ren.GetActiveCamera().SetFocalPoint(0, 0, 0)
ren.GetActiveCamera().SetViewUp(0, 1, 0)
ren.GetActiveCamera().SetParallelProjection(1)
# Initialize RenderWindow
ren_win = vtk.vtkRenderWindow()
ren_win.AddRenderer(ren)
ren_win.SetSize(win_size, win_size)
ren_win.SetOffScreenRendering(off_screen_rendering)
t = vtk.vtkTransform()
t.Identity()
t.Update()
# Transform (assuming only one mesh)
trans = vtk.vtkTransformPolyDataFilter()
trans.SetInputData(pd)
trans.SetTransform(t)
trans.Update()
mapper = vtk.vtkPolyDataMapper()
mapper.SetInputData(trans.GetOutput())
actor_text = vtk.vtkActor()
actor_text.SetMapper(mapper)
if texture_img is not None:
actor_text.SetTexture(texture)
actor_text.GetProperty().SetColor(1, 1, 1)
actor_text.GetProperty().SetAmbient(1.0)
actor_text.GetProperty().SetSpecular(0)
actor_text.GetProperty().SetDiffuse(0)
ren.AddActor(actor_text)
actor_geometry = vtk.vtkActor()
actor_geometry.SetMapper(mapper)
ren.AddActor(actor_geometry)
w2if = vtk.vtkWindowToImageFilter()
w2if.SetInput(ren_win)
writer_png = vtk.vtkPNGWriter()
writer_png.SetInputConnection(w2if.GetOutputPort())
scale = vtk.vtkImageShiftScale()
scale.SetOutputScalarTypeToUnsignedChar()
scale.SetInputConnection(w2if.GetOutputPort())
scale.SetShift(0)
scale.SetScale(-255)
writer_png_2 = vtk.vtkPNGWriter()
writer_png_2.SetInputConnection(scale.GetOutputPort())
for view in range(n_views):
name_rgb = str(self.config.temp_dir /
('rendering' + str(view) + '_RGB.png'))
name_depth = str(self.config.temp_dir /
('rendering' + str(view) + '_zbuffer.png'))
name_geometry = str(self.config.temp_dir /
('rendering' + str(view) + '_geometry.png'))
# print('Rendering ', name_rgb)
rx, ry, rz, s, tx, ty = transform_stack[view]
t.Identity()
t.RotateY(ry)
t.RotateX(rx)
t.RotateZ(rz)
t.Update()
trans.Update()
xmin = -150
xmax = 150
ymin = -150
ymax = 150
zmin = trans.GetOutput().GetBounds()[4]
zmax = trans.GetOutput().GetBounds()[5]
xlen = xmax - xmin
ylen = ymax - ymin
cx = 0
cy = 0
extend_factor = 1.0
side_length = max([xlen, ylen]) * extend_factor
# zoom_fac = win_size / side_length
ren.GetActiveCamera().SetParallelScale(side_length / 2)
ren.GetActiveCamera().SetPosition(cx, cy, 500)
ren.GetActiveCamera().SetFocalPoint(cx, cy, 0)
ren.GetActiveCamera().SetClippingRange(500 - zmax - slack,
500 - zmin + slack)
# Save textured image
w2if.SetInputBufferTypeToRGB()
actor_geometry.SetVisibility(False)
actor_text.SetVisibility(True)
mapper.Modified()
ren.Modified() # force actors to have the correct visibility
ren_win.Render()
if write_image_files:
w2if.Modified(
) # Needed here else only first rendering is put to file
writer_png.SetFileName(name_rgb)
writer_png.Write()
else:
w2if.Modified(
) # Needed here else only first rendering is put to file
w2if.Update()
# add rendering to image stack
im = w2if.GetOutput()
rows, cols, _ = im.GetDimensions()
sc = im.GetPointData().GetScalars()
a = vtk_to_numpy(sc)
components = sc.GetNumberOfComponents()
a = a.reshape(rows, cols, components)
a = np.flipud(a)
# get RGB data - 3 first channels
image_stack[view, :, :, 0:3] = a[:, :, :]
actor_text.SetVisibility(False)
actor_geometry.SetVisibility(True)
mapper.Modified()
ren.Modified() # force actors to have the correct visibility
ren_win.Render()
if write_image_files:
w2if.Modified(
) # Needed here else only first rendering is put to file
writer_png.SetFileName(name_geometry)
writer_png.Write()
else:
w2if.Modified(
) # Needed here else only first rendering is put to file
w2if.Update()
# add rendering to image stack
im = w2if.GetOutput()
rows, cols, _ = im.GetDimensions()
sc = im.GetPointData().GetScalars()
a = vtk_to_numpy(sc)
components = sc.GetNumberOfComponents()
a = a.reshape(rows, cols, components)
a = np.flipud(a)
# get geometry data
image_stack[view, :, :, 3:4] = a[:, :, 0:1]
ren.Modified() # force actors to have the correct visibility
ren_win.Render()
w2if.SetInputBufferTypeToZBuffer()
w2if.Modified()
if write_image_files:
w2if.Modified(
) # Needed here else only first rendering is put to file
writer_png_2.SetFileName(name_depth)
writer_png_2.Write()
else:
w2if.Modified(
) # Needed here else only first rendering is put to file
w2if.Update()
scale.Update()
im = scale.GetOutput()
rows, cols, _ = im.GetDimensions()
sc = im.GetPointData().GetScalars()
a = vtk_to_numpy(sc)
components = sc.GetNumberOfComponents()
a = a.reshape(rows, cols, components)
a = np.flipud(a)
# get depth data
image_stack[view, :, :, 4:5] = a[:, :, 0:1]
actor_geometry.SetVisibility(False)
actor_text.SetVisibility(True)
ren.Modified()
del writer_png_2, writer_png, ren_win, actor_geometry, actor_text, mapper, w2if, t, trans
if texture_img is not None:
del texture_img
del texture
# del texture_image
end = time.time()
self.logger.debug("File load and rendering time: " + str(end - start))
return image_stack
def write_landmarks_as_text(landmarks, file_name):
Utils3D.write_landmarks_as_text_external(landmarks, file_name)
def write_landmarks_as_vtk_points(landmarks, file_name):
Utils3D.write_landmarks_as_vtk_points_external(landmarks, file_name)
def test_on_bu_3d_fe(config):
test_set_file = config['data_loader']['args'][
'data_dir'] + '/dataset_test.txt'
# test_set_file = config['data_loader']['args']['data_dir'] + '/face_dataset_debug.txt'
result_file = config.temp_dir / 'results.csv'
device, model = get_device_and_load_model(config)
files = []
with open(test_set_file) as f:
for line in f:
line = line.strip("/n")
line = line.strip("\n")
clean_name = os.path.splitext(line)[0]
if len(clean_name) > 0:
files.append(clean_name)
print('Read', len(files), 'files to run test on')
bu_3dfe_dir = config['preparedata']['raw_data_dir']
idx = 0
res_f = open(result_file, "w")
start_time = time.time()
for f_name in files:
lm_name = bu_3dfe_dir + f_name + '_RAW_84_LMS.txt'
wrl_name = bu_3dfe_dir + f_name + '_RAW.wrl'
# bmp_name = bu_3dfe_dir + f_name + '_F3D.bmp'
gt_lms = read_3d_landmarks(lm_name)
if os.path.isfile(wrl_name):
print('Computing file ', idx, ' of ', len(files))
render_3d = Render3D(config)
image_stack, transform_stack = render_3d.render_3d_file(wrl_name)
predict_2d = Predict2D(config, model, device)
heatmap_maxima = predict_2d.predict_heatmaps_from_images(
image_stack)
print('Computing 3D landmarks')
u3d = Utils3D(config)
u3d.heatmap_maxima = heatmap_maxima
u3d.transformations_3d = transform_stack
u3d.compute_lines_from_heatmap_maxima()
# u3d.visualise_one_landmark_lines(83)
# u3d.visualise_one_landmark_lines(26)
u3d.compute_all_landmarks_from_view_lines()
u3d.project_landmarks_to_surface(wrl_name)
pred_lms = u3d.landmarks
res_f.write(f_name + ', ')
write_landmark_accuracy(gt_lms, pred_lms, res_f)
res_f.flush()
base_name = os.path.basename(f_name)
sphere_file = config.temp_dir / (base_name +
'_landmarkAccuracy.vtk')
visualise_landmarks_as_spheres_with_accuracy(
gt_lms, pred_lms, str(sphere_file))
idx = idx + 1
time_per_test = (time.time() - start_time) / idx
time_left = (len(files) - idx) * time_per_test
print('Time left in test: ',
str(datetime.timedelta(seconds=time_left)))
else:
print('File', wrl_name, ' does not exists')
def write_landmarks_as_ply_for_recognition(file_name):
Utils3D.write_landmarks_as_ply_external_for_recognition(file_name)
def write_landmarks_as_ply(file_name):
Utils3D.write_landmarks_as_ply_external(file_name)