def init(Obj_Name, params): vertices_1, faces_1, textures_1 = nr.load_obj( "./3D_objects/{}.obj".format(Obj_Name), load_texture=True) #, texture_size=4) # print(vertices_1.shape) # print(faces_1.shape) vertices_1 = vertices_1[None, :, :] # add dimension faces_1 = faces_1[None, :, :] #add dimension textures_1 = textures_1[None, :, :] #add dimension nb_vertices = vertices_1.shape[0] # define extrinsic parameter alpha = params[0] beta = params[1] gamma = params[2] x = params[3] y = params[4] z = limit(params[5]) R = np.array([alpha, beta, gamma]) # angle in degree t = np.array([x, y, z]) # translation in meter Rt = np.concatenate( (R, t), axis=None ) # create one array of parameter in radian, this arraz will be saved in .npy file cam = camera_setttings( R=R, t=t, vert=nb_vertices ) # degree angle will be converted and stored in radian renderer = nr.Renderer(image_size=512, camera_mode='projection', dist_coeffs=None, K=cam.K_vertices, R=cam.R_vertices, t=cam.t_vertices, near=1, background_color=[1, 1, 1], far=1000, orig_size=512, light_intensity_ambient=1.0, light_intensity_directional=0, light_direction=[0, 1, 0], light_color_ambient=[1, 1, 1], light_color_directional=[1, 1, 1]) # [1,1,1] return vertices_1, faces_1, textures_1, renderer
def renderingGivenTm(self): # print('rendering the 3D cad tool') instrument_to_camera_transform = self.T_m Extracted_X, Extracted_Y, Extracted_Z, Extracted_theta1_deg, Extracted_theta2_deg, Extracted_theta3_deg = self.matrix2angle( self.T_m) # define transfomration parameter from json file alpha = Extracted_theta1_deg + 90 #adapt from openCV to renderer axis system beta = Extracted_theta2_deg gamma = Extracted_theta3_deg x = Extracted_X y = Extracted_Y z = Extracted_Z # print('parameter found are: ',x, y, z, alpha, beta, gamma) #renderer the 3D cad model vertices_1, faces_1, textures_1 = nr.load_obj( "3D_objects/shaftshortOnly.obj", load_texture=True, normalization=False) # , texture_size=4) vertices_1 = vertices_1[None, :, :] # add dimension faces_1 = faces_1[None, :, :] # add dimension textures_1 = textures_1[None, :, :] # add dimension nb_vertices = vertices_1.shape[0] R = np.array([np.radians(alpha), np.radians(beta), np.radians(gamma)]) # angle in degree t = np.array([x, y, z]) # translation in meter # define transformation by transformation matrix self.Rt = np.concatenate((R, t), axis=None).astype( np.float16 ) # create one array of parameter in radian, this arraz will be saved in .npy file cam = camera_setttings( R=R, t=t, PnPtm=self.T_m, PnPtmFlag=False, vert=nb_vertices, resolutionx=1280, resolutiony=1024, cx=c_x, cy=c_y, fx=f_x, fy=f_y) # degree angle will be converted and stored in radian renderer = nr.Renderer( image_size=1280, camera_mode='projection', dist_coeffs=None, anti_aliasing=True, fill_back=True, perspective=False, K=cam.K_vertices, R=cam.R_vertices, t=cam.t_vertices, near=0, background_color=[1, 1, 1], # background is filled now with value 0-1 instead of 0-255 # changed from 0-255 to 0-1 far=1, orig_size=1280, light_intensity_ambient=1, light_intensity_directional=0.5, light_direction=[0, 1, 0], light_color_ambient=[1, 1, 1], light_color_directional=[1, 1, 1]) images_1 = renderer( vertices_1, faces_1, textures_1, K=torch.cuda.FloatTensor(cam.K_vertices), R=torch.cuda.FloatTensor(cam.R_vertices), t=torch.cuda.FloatTensor( cam.t_vertices)) # [batch_size, RGB, image_size, image_size] image = images_1[0].detach().cpu().numpy()[0].transpose( (1, 2, 0)) #float32 from 0-1 image = (image * 255).astype( np.uint8) #cast from float32 255.0 to 255 uint8 self.image = image[0:1024, 0:1280, :] #modifification done in the rasterize.py file for the default far and near value DEFAULT_FAR = 1 , DEFAULT_EPS = 1e-4 sils_1 = renderer( vertices_1, faces_1, textures_1, mode='silhouettes', K=torch.cuda.FloatTensor(cam.K_vertices), R=torch.cuda.FloatTensor(cam.R_vertices), t=torch.cuda.FloatTensor( cam.t_vertices)) # [batch_size, RGB, image_size, image_size] sil = sils_1.detach().cpu().numpy().transpose((1, 2, 0)) sil = np.squeeze((sil * 255)).astype( np.uint8 ) # change from float 0-1 [512,512,1] to uint8 0-255 [512,512] self.sil = sil[0:1024, 0:1280] #create window of the overlap of the tool and renderer backgroundImage = Image.open("{}/frameL{}.jpg".format( self.pathfile, self.currentFrameId)) self.backgroundIm = np.array(backgroundImage) toolbck = backgroundImage.load() sil3d = self.sil[:, :, np.newaxis] renderim = np.concatenate((sil3d, sil3d, sil3d), axis=2) toolIm = Image.fromarray(np.uint8(renderim)) alpha = 0.2 size = 10 #ellipse size out = Image.blend(backgroundImage, toolIm, alpha) draw = ImageDraw.Draw(out) for i in range(len(self.pinhole_point2)): px = self.pinhole_point2[i, 0] py = self.pinhole_point2[i, 1] draw.ellipse( [px - size / 2, py - size / 2, px + size / 2, py + size / 2], fill='blue') # draw.show() draw.ellipse( [c_x - size / 2, c_y - size / 2, c_x + size / 2, c_y + size / 2], fill='red') self.out = np.array(out)
def __init__(self, filename_obj=None, *args, **kwargs): super(ModelResNet50, self).__init__(Bottleneck, [3, 4, 6, 3], num_classes=6, **kwargs) # resnet part self.seq1 = nn.Sequential(self.conv1, self.bn1, self.relu, self.maxpool, self.layer1, self.layer2) self.seq2 = nn.Sequential( self.layer3, self.layer4, self.avgpool, ) self.fc # self.fc1 = nn.Linear(2*6, 6) # self.fc2 = nn.Linear(8, 6) # render part vertices, faces, textures = nr.load_obj(filename_obj, load_texture=True, normalization=False) vertices = vertices[ None, :, :] # [num_vertices, XYZ] -> [batch_size=1, num_vertices, XYZ] faces = faces[ None, :, :] # [num_faces, 3] -> [batch_size=1, num_faces, 3 textures = textures[None, :, :] nb_vertices = vertices.shape[0] self.register_buffer('vertices', vertices) self.register_buffer('faces', faces) self.register_buffer('textures', textures) # --------------------------------------------------------------------------------- # extrinsic parameter, link world/object coordinate to camera coordinate # --------------------------------------------------------------------------------- alpha = np.radians(0) beta = np.radians(0) gamma = np.radians(0) x = 0 # uniform(-2, 2) y = 0 # uniform(-2, 2) z = 0.08 # uniform(5, 10) #1000t was done with value between 7 and 10, Rot and trans between 5 10 R = np.array([np.radians(alpha), np.radians(beta), np.radians(gamma)]) # angle in degree t = np.array([x, y, z]) # translation in meter batch = vertices.shape[0] # --------------------------------------------------------------------------------- # intrinsic parameter # --------------------------------------------------------------------------------- c_x = 590 c_y = 508 f_x = 1067 f_y = 1067 # camera_calibration = np.zeros((4, 4)) # camera_calibration[0, 0] = f_x # camera_calibration[1, 1] = f_y # camera_calibration[0, 2] = c_x # camera_calibration[1, 2] = c_y # camera_calibration[2, 2] = 1 cam = camera_setttings( R=R, t=t, PnPtm=0, PnPtmFlag=False, vert=nb_vertices, resolutionx=1280, resolutiony=1024, cx=c_x, cy=c_y, fx=f_x, fy=f_y) # degree angle will be converted and stored in radian # K = np.array([[f / pix_sizeX, 0, Cam_centerX], # [0, f / pix_sizeY, Cam_centerY], # [0, 0, 1]]) # shape of [nb_vertice, 3, 3] # # K = np.repeat(K[np.newaxis, :, :], batch, axis=0) # shape of [batch=1, 3, 3] # R = np.repeat(R[np.newaxis, :, :], batch, axis=0) # shape of [batch=1, 3, 3] # t = np.repeat(t[np.newaxis, :], 1, axis=0) # shape of [1, 3] # # self.K = K # self.R = R # # -------------------------- working block translation # self.tx = torch.from_numpy(np.array(x, dtype=np.float32)).cuda() # self.ty = torch.from_numpy(np.array(y, dtype=np.float32)).cuda() # self.tz = torch.from_numpy(np.array(z, dtype=np.float32)).cuda() # self.t =torch.from_numpy(np.array([self.tx, self.ty, self.tz], dtype=np.float32)).unsqueeze(0) # -------------------------- # setup renderer renderer = nr.Renderer( image_size=1280, camera_mode='projection', dist_coeffs=None, anti_aliasing=True, fill_back=True, perspective=False, K=cam.K_vertices, R=cam.R_vertices, t=cam.t_vertices, near=0, background_color=[1, 1, 1], # background is filled now with value 0-1 instead of 0-255 # changed from 0-255 to 0-1 far=1, orig_size=1280, light_intensity_ambient=1, light_intensity_directional=0.5, light_direction=[0, 1, 0], light_color_ambient=[1, 1, 1], light_color_directional=[1, 1, 1]) self.renderer = renderer
def main(): cubes_database = [] sils_database = [] params_database = [] im_nr = 1 vertices_1, faces_1, textures_1 = nr.load_obj( "3D_objects/wrist.obj", load_texture=True) #, texture_size=4) print(vertices_1.shape) print(faces_1.shape) vertices_1 = vertices_1[None, :, :] # add dimension faces_1 = faces_1[None, :, :] #add dimension textures_1 = textures_1[None, :, :] #add dimension nb_vertices = vertices_1.shape[0] print(vertices_1.shape) print(faces_1.shape) file_name_extension = 'ExampleDatabaseOf10images' nb_im = 10 #init and create renderer object R = np.array([np.radians(0), np.radians(0), np.radians(0)]) # angle in degree t = np.array([0, 0, 0]) # translation in meter cam = camera_setttings(R=R, t=t, vert=nb_vertices) renderer = nr.Renderer( image_size=512, camera_mode='projection', dist_coeffs=None, K=cam.K_vertices, R=cam.R_vertices, t=cam.t_vertices, near=1, background_color=[ 0, 0, 0 ], #background is filled now with value 0-1 instead of 0-255 # changed from 0-255 to 0-1 far=1000, orig_size=512, light_intensity_ambient=1.0, light_intensity_directional=0, light_direction=[0, 1, 0], light_color_ambient=[1, 1, 1], light_color_directional=[1, 1, 1]) loop = tqdm.tqdm(range(0, nb_im)) for i in loop: # define transfomration parameter randomly uniform alpha = uniform(0, 180) beta = uniform(0, 180) gamma = uniform(0, 180) x = uniform(-1.5, 1.5) y = uniform(-1.5, 1.5) z = uniform( 5, 7 ) #1000t was done with value between 7 and 10, Rot and trans between 5 10 R = np.array([np.radians(alpha), np.radians(beta), np.radians(gamma)]) # angle in degree t = np.array([x, y, z]) # translation in meter Rt = np.concatenate((R, t), axis=None).astype( np.float16 ) # create one array of parameter in radian, this arraz will be saved in .npy file cam = camera_setttings( R=R, t=t, vert=nb_vertices ) # degree angle will be converted and stored in radian images_1 = renderer( vertices_1, faces_1, textures_1, K=torch.cuda.FloatTensor(cam.K_vertices), R=torch.cuda.FloatTensor(cam.R_vertices), t=torch.cuda.FloatTensor( cam.t_vertices)) # [batch_size, RGB, image_size, image_size] image = images_1[0].detach().cpu().numpy()[0].transpose( (1, 2, 0)) #float32 from 0-1 image = (image * 255).astype( np.uint8) #cast from float32 255.0 to 255 uint8 sils_1 = renderer( vertices_1, faces_1, textures_1, mode='silhouettes', K=torch.cuda.FloatTensor(cam.K_vertices), R=torch.cuda.FloatTensor(cam.R_vertices), t=torch.cuda.FloatTensor( cam.t_vertices)) # [batch_size, RGB, image_size, image_size] sil = sils_1.detach().cpu().numpy().transpose((1, 2, 0)) sil = np.squeeze((sil * 255)).astype( np.uint8 ) # change from float 0-1 [512,512,1] to uint8 0-255 [512,512] #grow the list of cube, silhouette and parameters cubes_database.extend(image) sils_database.extend(sil) params_database.extend(Rt) im_nr = im_nr + 1 # #put surgical iamge as a background # BinarySil3layermask = (np.array([sil, sil,sil])).transpose((1, 2, 0))/255 # # plt.imshow(BinarySil3layermask) # numberbackground = random.randint(1,8) # backgroundImg = mpimg.imread("3D_objects/background{}.jpg".format(numberbackground)) # # sx = backgroundImg.shape[0] # sy = backgroundImg.shape[1] # # moveX = random.randint(0,sx-512) # moveY = random.randint(0,sy-512) # # print(moveX, moveY) # cropedbackgroundImg = backgroundImg[moveX:moveX+512, moveY:moveY+512, :] # maskedbackground = np.multiply((BinarySil3layermask *-1+1), cropedbackgroundImg/255) # imWithBackground = (image + (maskedbackground*255)).astype(np.uint8) # # plt.imshow(imWithBackground) # image = imWithBackground # cubes_database.extend(image) # if(im_nr%1 == 0): # fig = plt.figure() # fig.add_subplot(2, 1, 1) # plt.imshow(image) # imageio.imwrite("3D_objects/{}_ref.png".format(file_name_extension), image) # # fig.add_subplot(2, 1, 2) # plt.imshow(sil, cmap='gray') # plt.show() # plt.close(fig) # save database # reshape in the form (nbr of image, x dim, y dim, layers) cubes_database = np.reshape(cubes_database, (im_nr - 1, 512, 512, 3)) # 3 channel rgb sils_database = np.reshape(sils_database, (im_nr - 1, 512, 512)) #binary mask monochannel params_database = np.reshape(params_database, (im_nr - 1, 6)) #array of 6 params np.save('Npydatabase/cubes_{}.npy'.format(file_name_extension), cubes_database) np.save('Npydatabase/sils_{}.npy'.format(file_name_extension), sils_database) np.save('Npydatabase/params_{}.npy'.format(file_name_extension), params_database) print('images saved')
def main(): cubes_database = [] sils_database = [] params_database = [] im_nr = 1 vertices_1, faces_1, textures_1 = nr.load_obj( "3D_objects/wrist.obj", load_texture=True) #, texture_size=4) print(vertices_1.shape) print(faces_1.shape) vertices_1 = vertices_1[None, :, :] # add dimension faces_1 = faces_1[None, :, :] #add dimension textures_1 = textures_1[None, :, :] #add dimension nb_vertices = vertices_1.shape[0] print(vertices_1.shape) print(faces_1.shape) nb_im = 1000 file_name_extension = 'exampleOfNameForTheTimelapse'.format(nb_im) parser = argparse.ArgumentParser() parser.add_argument('-or', '--filename_output', type=str, default=os.path.join( result_dir, 'Animation_{}.gif'.format(file_name_extension))) args = parser.parse_args() #init and create renderer object R = np.array([np.radians(0), np.radians(0), np.radians(0)]) # angle in degree t = np.array([0, 0, 0]) # translation in meter cam = camera_setttings(R=R, t=t, vert=nb_vertices) renderer = nr.Renderer( image_size=512, camera_mode='projection', dist_coeffs=None, K=cam.K_vertices, R=cam.R_vertices, t=cam.t_vertices, near=1, background_color=[ 0, 0, 0 ], #background is filled now with value 0-1 instead of 0-255 # changed from 0-255 to 0-1 far=1000, orig_size=512, light_intensity_ambient=1.0, light_intensity_directional=0, light_direction=[0, 1, 0], light_color_ambient=[1, 1, 1], light_color_directional=[1, 1, 1]) prevA = 0 prevB = 0 prevG = 0 circlepoint = PointsInCircum(1.4, nb_im) Z_sinus = np.linspace(0, 360, nb_im) Z_sinus = 6 + np.sin(np.radians(Z_sinus)) circlepoint_alpha = np.linspace(0, 180, nb_im) circlepoint_beta = np.linspace(180, 0, nb_im) circlepoint_gamma = np.linspace(0, 180, nb_im) print('z axis translation min max are {} and {}'.format( Z_sinus.min(), Z_sinus.max())) #5.000000015261379 and 6.999998763828597 loop = tqdm.tqdm(range(0, nb_im)) for i in loop: # define transfomration parameter randomly uniform #------INSERT HERE SPECIFIC VALUE OF TRANSLATION AND ROTATION ----------------------------------------------- alpha = 0 # circlepoint_alpha[i] beta = 0 #circlepoint_beta[i] gamma = circlepoint_gamma[i] x = circlepoint[i][0] y = circlepoint[i][1] z = Z_sinus[ i] #1000t was done with value between 7 and 10, Rot and trans between 5 10 R = np.array([np.radians(alpha), np.radians(beta), np.radians(gamma)]) # angle in degree t = np.array([x, y, z]) # translation in meter Rt = np.concatenate((R, t), axis=None).astype( np.float16 ) # create one array of parameter in radian, this arraz will be saved in .npy file cam = camera_setttings( R=R, t=t, vert=nb_vertices ) # degree angle will be converted and stored in radian images_1 = renderer( vertices_1, faces_1, textures_1, K=torch.cuda.FloatTensor(cam.K_vertices), R=torch.cuda.FloatTensor(cam.R_vertices), t=torch.cuda.FloatTensor( cam.t_vertices)) # [batch_size, RGB, image_size, image_size] image = images_1[0].detach().cpu().numpy()[0].transpose( (1, 2, 0)) #float32 from 0-1 img = image image = (image * 255).astype( np.uint8) #cast from float32 255.0 to 255 uint8 sils_1 = renderer( vertices_1, faces_1, textures_1, mode='silhouettes', K=torch.cuda.FloatTensor(cam.K_vertices), R=torch.cuda.FloatTensor(cam.R_vertices), t=torch.cuda.FloatTensor( cam.t_vertices)) # [batch_size, RGB, image_size, image_size] sil = sils_1.detach().cpu().numpy().transpose((1, 2, 0)) sil = np.squeeze((sil * 255)).astype( np.uint8 ) # change from float 0-1 [512,512,1] to uint8 0-255 [512,512] #grow the list of cube, silhouette and parameters cubes_database.extend(image) sils_database.extend(sil) params_database.extend(Rt) im_nr = im_nr + 1 # #put surgical iamge as a background # BinarySil3layermask = (np.array([sil, sil,sil])).transpose((1, 2, 0))/255 # # plt.imshow(BinarySil3layermask) # numberbackground = random.randint(1,8) # backgroundImg = mpimg.imread("3D_objects/background{}.jpg".format(numberbackground)) # # sx = backgroundImg.shape[0] # sy = backgroundImg.shape[1] # # moveX = random.randint(0,sx-512) # moveY = random.randint(0,sy-512) # # print(moveX, moveY) # cropedbackgroundImg = backgroundImg[moveX:moveX+512, moveY:moveY+512, :] # maskedbackground = np.multiply((BinarySil3layermask *-1+1), cropedbackgroundImg/255) # imWithBackground = (image + (maskedbackground*255)).astype(np.uint8) # # plt.imshow(imWithBackground) # image = imWithBackground # cubes_database.extend(image) # if(im_nr%50 == 0): # fig = plt.figure() # fig.add_subplot(2, 1, 1) # plt.imshow(image) # imageio.imwrite("3D_objects/{}_ref.png".format(file_name_extension), image) # # fig.add_subplot(2, 1, 2) # plt.imshow(sil, cmap='gray') # plt.show() # plt.close(fig) # imsave('/tmp/_tmp_%04d.png' % i, img) # save database make_gif(args.filename_output) # reshape in the form (nbr of image, x dim, y dim, layers) cubes_database = np.reshape(cubes_database, (im_nr - 1, 512, 512, 3)) # 3 channel rgb sils_database = np.reshape(sils_database, (im_nr - 1, 512, 512)) #binary mask monochannel params_database = np.reshape(params_database, (im_nr - 1, 6)) #array of 6 params np.save('Npydatabase/cubes_{}.npy'.format(file_name_extension), cubes_database) np.save('Npydatabase/sils_{}.npy'.format(file_name_extension), sils_database) np.save('Npydatabase/params_{}.npy'.format(file_name_extension), params_database) print('images saved')
def main(): cubes_database = [] sils_database = [] params_database = [] im_nr = 1 vertices_1, faces_1, textures_1 = nr.load_obj("3D_objects/shaft.obj", load_texture=True) #, texture_size=4) print(vertices_1.shape) print(faces_1.shape) vertices_1 = vertices_1[None, :, :] # add dimension faces_1 = faces_1[None, :, :] #add dimension textures_1 = textures_1[None, :, :] #add dimension nb_vertices = vertices_1.shape[0] print(vertices_1.shape) print(faces_1.shape) file_name_extension = 'shaft_1im_180_M15_15_5_7' nb_im = 1 #init and create renderer object # R = np.array([np.radians(0), np.radians(0), np.radians(0)]) # angle in degree # t = np.array([0, 0, 0]) # translation in meter cam = camera_setttings() renderer = nr.Renderer(image_size=1280, camera_mode='projection', dist_coeffs=None, K=cam.K_vertices, R=cam.R_vertices, t=cam.t_vertices, near=1, background_color=[1, 1, 1], #background is filled now with value 0-1 instead of 0-255 # changed from 0-255 to 0-1 far=1000, orig_size=674, light_intensity_ambient=0.7, light_intensity_directional=0.5, light_direction=[0, 1, 0], light_color_ambient=[1, 1, 1], light_color_directional=[1, 1, 1]) ## -------------------------read json file ------------------------------------------- with open('data/data.json') as json_file: data = json.load(json_file) data_len = len(data) # usm_camera = data[0:data_len]['usm-1'] ## -------------------------------------------------------------------------------- loop = tqdm.tqdm(range(0, nb_im)) for i in loop: ## -------------------------extract json frame matrix ------------------------------------------- usm_camera = data[i]['usm-1'] usm_inst = data[i]['usm-2'] instrument_to_camera_transform = np.asarray([list(map(float, usm_inst['pose'][0])), list(map(float, usm_inst['pose'][1])), list(map(float, usm_inst['pose'][2])), list(map(float, usm_inst['pose'][3]))], dtype=np.float64) #to test the conversion degree to radian to transformation matrix and then back to euler angle in radian R_test = np.array([np.radians(0),np.radians(0),np.radians(0)]) #test value alpha beta gamma T_test_vector, R_test_matrix = BuildTransformationMatrix(tx=0, ty=0, tz=0, alpha=R_test[0], beta=R_test[1], gamma=R_test[2]) instrument_to_camera_transform[0,0:3] = R_test_matrix[0,:] instrument_to_camera_transform[1,0:3] = R_test_matrix[1,:] instrument_to_camera_transform[2,0:3] = R_test_matrix[2,:] joint_values = np.asarray([list(map(float, usm_inst['articulation'][0])), list(map(float, usm_inst['articulation'][1])), list(map(float, usm_inst['articulation'][2]))], dtype=np.float64) joint_values[-1] = 2 * joint_values[-1] # print(instrument_to_camera_transform[1,2]) # [row column] #formula from http://planning.cs.uiuc.edu/node103.html # alpha and gamma were swapped, don-t know where the problem is but value are correct Extracted_theta3_rad = m.atan2(instrument_to_camera_transform[1,0],instrument_to_camera_transform[0,0]) C_2 = m.sqrt(instrument_to_camera_transform[2,1]*instrument_to_camera_transform[2,1] + instrument_to_camera_transform[2,2]*instrument_to_camera_transform[2,2]) Extracted_theta2_rad = m.atan2(-instrument_to_camera_transform[2,0], C_2 ) Extracted_theta1_rad = m.atan2(instrument_to_camera_transform[2,1],instrument_to_camera_transform[2,2]) #formula from https://d3cw3dd2w32x2b.cloudfront.net/wp-content/uploads/2012/07/euler-angles1.pdf # Extracted_theta1_rad = m.atan2(-instrument_to_camera_transform[1,2],instrument_to_camera_transform[2,2]) # C_2 = m.sqrt(instrument_to_camera_transform[0,0]*instrument_to_camera_transform[0,0] + instrument_to_camera_transform[0,1]*instrument_to_camera_transform[0,1]) # Extracted_theta2_rad = m.atan2(instrument_to_camera_transform[0,1], C_2 ) # s_1 = m.sin(Extracted_theta1_rad) # c_1 = m.cos(Extracted_theta1_rad) # Extracted_theta3_rad = m.atan2(s_1*instrument_to_camera_transform[2,0]-c_1*instrument_to_camera_transform[1,0], # c_1*instrument_to_camera_transform[1,1]-s_1*instrument_to_camera_transform[2,1]) Extracted_X = instrument_to_camera_transform[0,3] Extracted_Y = instrument_to_camera_transform[1,3] Extracted_Z = instrument_to_camera_transform[2,3] Extracted_theta1_deg = np.degrees(Extracted_theta1_rad) Extracted_theta2_deg = np.degrees(Extracted_theta2_rad) Extracted_theta3_deg = np.degrees(Extracted_theta3_rad) # # define transfomration parameter from json file # alpha =Extracted_theta1_deg # beta = Extracted_theta2_deg # gamma = Extracted_theta3_deg # x = Extracted_X # y = Extracted_Y # z = Extracted_Z # # # define transfomration parameter randomly uniform alpha =0#uniform(0, 180) beta = 0#uniform(0, 180) gamma = 0 #uniform(0, 180) x = 0 #uniform(-1.5, 1.5) y = 0 #uniform(-1.5, 1.5) z = 0.2 #uniform(5, 7) #1000t was done with value between 7 and 10, Rot and trans between 5 10 R = np.array([np.radians(alpha), np.radians(beta), np.radians(gamma)]) # angle in degree t = np.array([x, y, z]) # translation in meter Rt = np.concatenate((R, t), axis=None).astype(np.float16) # create one array of parameter in radian, this arraz will be saved in .npy file cam = camera_setttings(R=R, t=t, vert=nb_vertices, resolutionx=1280, resolutiony=1024,cx=590, cy=508, fx=1067, fy=1067) # degree angle will be converted and stored in radian images_1 = renderer(vertices_1, faces_1, textures_1, K=torch.cuda.FloatTensor(cam.K_vertices), R=torch.cuda.FloatTensor(cam.R_vertices), t=torch.cuda.FloatTensor(cam.t_vertices)) # [batch_size, RGB, image_size, image_size] image = images_1[0].detach().cpu().numpy()[0].transpose((1, 2, 0)) #float32 from 0-1 image = (image*255).astype(np.uint8) #cast from float32 255.0 to 255 uint8 sils_1 = renderer(vertices_1, faces_1, textures_1, mode='silhouettes', K=torch.cuda.FloatTensor(cam.K_vertices), R=torch.cuda.FloatTensor(cam.R_vertices), t=torch.cuda.FloatTensor(cam.t_vertices)) # [batch_size, RGB, image_size, image_size] sil = sils_1.detach().cpu().numpy().transpose((1, 2, 0)) sil = np.squeeze((sil * 255)).astype(np.uint8) # change from float 0-1 [512,512,1] to uint8 0-255 [512,512] #grow the list of cube, silhouette and parameters cubes_database.extend(image) sils_database.extend(sil) params_database.extend(Rt) im_nr = im_nr+1 # #put surgical iamge as a background # BinarySil3layermask = (np.array([sil, sil,sil])).transpose((1, 2, 0))/255 # # plt.imshow(BinarySil3layermask) # numberbackground = random.randint(1,8) # backgroundImg = mpimg.imread("3D_objects/background{}.jpg".format(numberbackground)) # # sx = backgroundImg.shape[0] # sy = backgroundImg.shape[1] # # moveX = random.randint(0,sx-512) # moveY = random.randint(0,sy-512) # # print(moveX, moveY) # cropedbackgroundImg = backgroundImg[moveX:moveX+512, moveY:moveY+512, :] # maskedbackground = np.multiply((BinarySil3layermask *-1+1), cropedbackgroundImg/255) # imWithBackground = (image + (maskedbackground*255)).astype(np.uint8) # # plt.imshow(imWithBackground) # image = imWithBackground # cubes_database.extend(image) if(im_nr%1 == 0): fig = plt.figure() fig.add_subplot(2, 1, 1) plt.imshow(image) imageio.imwrite("3D_objects/{}_ref.png".format(file_name_extension), image) fig.add_subplot(2, 1, 2) plt.imshow(sil, cmap='gray') plt.show() plt.close(fig) # save database # reshape in the form (nbr of image, x dim, y dim, layers) cubes_database = np.reshape(cubes_database, (im_nr-1, 512, 512, 3)) # 3 channel rgb sils_database = np.reshape(sils_database, (im_nr-1, 512, 512)) #binary mask monochannel params_database = np.reshape(params_database,(im_nr-1, 6)) #array of 6 params np.save('Npydatabase/cubes_{}.npy'.format(file_name_extension), cubes_database) np.save('Npydatabase/sils_{}.npy'.format(file_name_extension), sils_database) np.save('Npydatabase/params_{}.npy'.format(file_name_extension), params_database) print('images saved')
def main(): cubes_database = [] sils_database = [] params_database = [] im_nr = 1 vertices_1, faces_1, textures_1 = nr.load_obj("3D_objects/rubik_color.obj", load_texture=True)#, texture_size=4) print(vertices_1.shape) print(faces_1.shape) vertices_1 = vertices_1[None, :, :] # add dimension faces_1 = faces_1[None, :, :] #add dimension textures_1 = textures_1[None, :, :] #add dimension nb_vertices = vertices_1.shape[0] print(vertices_1.shape) print(faces_1.shape) file_name_extension = '_10' nb_im = 10 #init and create renderer object R = np.array([np.radians(0), np.radians(0), np.radians(0)]) # angle in degree t = np.array([0, 0, 0]) # translation in meter cam = camera_setttings(R=R, t=t, vert=nb_vertices) renderer = nr.Renderer(image_size=512, camera_mode='projection', dist_coeffs=None, K=cam.K_vertices, R=cam.R_vertices, t=cam.t_vertices, near=1, background_color=[1, 1, 1], #background is filled now with value 0-1 instead of 0-255 # changed from 0-255 to 0-1 far=1000, orig_size=512, light_intensity_ambient=1.0, light_intensity_directional=0, light_direction=[0, 1, 0], light_color_ambient=[1, 1, 1], light_color_directional=[1, 1, 1]) loop = tqdm.tqdm(range(0, nb_im)) for i in loop: # define transfomration parameter randomly uniform alpha = uniform(0, 180) beta = uniform(0, 180) gamma = uniform(0, 180) x = uniform(-2, 2) y = uniform(-2, 2) z = uniform(8, 14) R = np.array([np.radians(alpha), np.radians(beta), np.radians(gamma)]) # angle in degree t = np.array([x, y, z]) # translation in meter Rt = np.concatenate((R, t), axis=None).astype(np.float16) # create one array of parameter in radian, this arraz will be saved in .npy file cam = camera_setttings(R=R, t=t, vert=nb_vertices) # degree angle will be converted and stored in radian images_1 = renderer(vertices_1, faces_1, textures_1, K=torch.cuda.FloatTensor(cam.K_vertices), R=torch.cuda.FloatTensor(cam.R_vertices), t=torch.cuda.FloatTensor(cam.t_vertices)) # [batch_size, RGB, image_size, image_size] image = images_1[0].detach().cpu().numpy()[0].transpose((1, 2, 0)) #float32 from 0-1 image = (image*255).astype(np.uint8) #cast from float32 255.0 to 255 uint8 sils_1 = renderer(vertices_1, faces_1, textures_1, mode='silhouettes', K=torch.cuda.FloatTensor(cam.K_vertices), R=torch.cuda.FloatTensor(cam.R_vertices), t=torch.cuda.FloatTensor(cam.t_vertices)) # [batch_size, RGB, image_size, image_size] sil = sils_1.detach().cpu().numpy().transpose((1, 2, 0)) sil = np.squeeze((sil * 255)).astype(np.uint8) # change from float 0-1 [512,512,1] to uint8 0-255 [512,512] #grow the list of cube, silhouette and parameters cubes_database.extend(image) sils_database.extend(sil) params_database.extend(Rt) im_nr = im_nr+1 if(im_nr%1 == 0): fig = plt.figure() fig.add_subplot(1, 2, 1) plt.imshow(image) fig.add_subplot(1, 2, 2) plt.imshow(sil, cmap='gray') plt.show() plt.close(fig) # save database # reshape in the form (nbr of image, x dim, y dim, layers) cubes_database = np.reshape(cubes_database, (im_nr-1, 512, 512, 3)) # 3 channel rgb sils_database = np.reshape(sils_database, (im_nr-1, 512, 512)) #binary mask monochannel params_database = np.reshape(params_database,(im_nr-1, 6)) #array of 6 params np.save('Npydatabase/cubes_{}.npy'.format(file_name_extension), cubes_database) np.save('Npydatabase/sils_{}.npy'.format(file_name_extension), sils_database) np.save('Npydatabase/params_{}.npy'.format(file_name_extension), params_database) print('images saved')
def main(): processcount = 0 current_dir = os.path.dirname(os.path.realpath(__file__)) data_dir = os.path.join(current_dir, 'data') parser = argparse.ArgumentParser() parser.add_argument('-or', '--filename_output', type=str, default=os.path.join(current_dir, 'ResultRender_{}.gif'.format('test2'))) parser.add_argument('-mr', '--make_reference_image', type=int, default=0) parser.add_argument('-g', '--gpu', type=int, default=0) args = parser.parse_args() cubes_database = [] sils_database = [] params_database = [] im_nr = 1 vertices_1, faces_1, textures_1 = nr.load_obj("3D_objects/LongShaft3.obj", load_texture=True, normalization=False) #, texture_size=4) print(vertices_1.shape) print(faces_1.shape) vertices_1 = vertices_1[None, :, :] # add dimension faces_1 = faces_1[None, :, :] #add dimension textures_1 = textures_1[None, :, :] #add dimension nb_vertices = vertices_1.shape[0] print(vertices_1.shape) print(faces_1.shape) file_name_extension = 'shaft_1im_180_M15_15_5_7' nb_im = 100 useTransformMatrix = False instrument_to_camera_transform = np.array([(0, 0, 0, 0), (0, 0, 0, 0), (0, 0, 0, 0), (0, 0, 0, 1)]) instrument_to_camera_transform = np.zeros((4,4)) ## -------------------------------------------------------------------------------- loop = tqdm.tqdm(range(0, nb_im)) for i in loop: # # define transfomration parameter from angle and translation if useTransformMatrix : tempx = 145 tempy = 52 tempz = -114 R_test = np.array([np.radians(tempx), np.radians(tempy), np.radians(tempz)]) # test value alpha beta gamma T_test = np.array([0.017, -0.008, 0.1]) print(T_test, tempx, tempy, tempz) T_test_vector, R_test_matrix = BuildTransformationMatrix(tx=T_test[0], ty=T_test[1], tz=T_test[2], alpha=R_test[0], beta=R_test[1], gamma=R_test[2]) #return 1x3 vector and 3x3matrix instrument_to_camera_transform[0, 0:3] = R_test_matrix[0, :] instrument_to_camera_transform[1, 0:3] = R_test_matrix[1, :] instrument_to_camera_transform[2, 0:3] = R_test_matrix[2, :] instrument_to_camera_transform[0, 3] = T_test[0] instrument_to_camera_transform[1, 3] = T_test[1] instrument_to_camera_transform[2, 3] = T_test[2] instrument_to_camera_transform[3, 3] = 1 # test[0:3,0:3] = R_test_matrix # formula from http://planning.cs.uiuc.edu/node103.html # alpha and gamma were swapped, don-t know where the problem is but value are correct Extracted_theta3_rad = m.atan2(instrument_to_camera_transform[1, 0], instrument_to_camera_transform[0, 0]) C_2 = m.sqrt(instrument_to_camera_transform[2, 1] * instrument_to_camera_transform[2, 1] + instrument_to_camera_transform[2, 2] * instrument_to_camera_transform[2, 2]) Extracted_theta2_rad = m.atan2(-instrument_to_camera_transform[2, 0], C_2) Extracted_theta1_rad = m.atan2(instrument_to_camera_transform[2, 1], instrument_to_camera_transform[2, 2]) Extracted_X = instrument_to_camera_transform[0, 3] Extracted_Y = instrument_to_camera_transform[1, 3] Extracted_Z = instrument_to_camera_transform[2, 3] Extracted_theta1_deg = np.degrees(Extracted_theta1_rad) Extracted_theta2_deg = np.degrees(Extracted_theta2_rad) Extracted_theta3_deg = np.degrees(Extracted_theta3_rad) # define transfomration parameter from json fileprint(x,y,z, alpha,beta,gamma) alpha =Extracted_theta1_deg beta = Extracted_theta2_deg gamma = Extracted_theta3_deg x = Extracted_X y = Extracted_Y z = Extracted_Z print(x,y,z, alpha,beta,gamma) else: alpha =np.degrees(00) #uniform(0, 180) beta = np.degrees(00) #uniform(0, 180) gamma = np.degrees(0) #uniform(0, 180) x =0#uniform(-1.5, 1.5) y =0 #uniform(-1.5, 1.5) z = 8.5381e-02#uniform(5, 7) #1000t was done with value between 7 and 10, Rot and trans between 5 10 print(x, y, z, alpha, beta, gamma) R = np.array([np.radians(alpha), np.radians(beta), np.radians(gamma)]) # angle in degree t = np.array([x, y, z]) # translation in meter # define transformation by transformation matrix Rt = np.concatenate((R, t), axis=None).astype(np.float16) # create one array of parameter in radian, this arraz will be saved in .npy file cam = camera_setttings(R=R, t=t, vert=nb_vertices, resolutionx=1280, resolutiony=1024,cx=590, cy=508, fx=1067, fy=1067) # degree angle will be converted and stored in radian renderer = nr.Renderer(image_size=1280, camera_mode='projection', dist_coeffs=None,anti_aliasing=True, fill_back=True, perspective=False, K=cam.K_vertices, R=cam.R_vertices, t=cam.t_vertices, near=0, background_color=[1, 1, 1], # background is filled now with value 0-1 instead of 0-255 # changed from 0-255 to 0-1 far=20, orig_size=1280, light_intensity_ambient=1, light_intensity_directional=0.5, light_direction=[0, 1, 0], light_color_ambient=[1, 1, 1], light_color_directional=[1, 1, 1]) images_1 = renderer(vertices_1, faces_1, textures_1, K=torch.cuda.FloatTensor(cam.K_vertices), R=torch.cuda.FloatTensor(cam.R_vertices), t=torch.cuda.FloatTensor(cam.t_vertices)) # [batch_size, RGB, image_size, image_size] image = images_1[0].detach().cpu().numpy()[0].transpose((1, 2, 0)) #float32 from 0-1 image = (image*255).astype(np.uint8) #cast from float32 255.0 to 255 uint8 sils_1 = renderer(vertices_1, faces_1, textures_1, mode='silhouettes', K=torch.cuda.FloatTensor(cam.K_vertices), R=torch.cuda.FloatTensor(cam.R_vertices), t=torch.cuda.FloatTensor(cam.t_vertices)) # [batch_size, RGB, image_size, image_size] sil = sils_1.detach().cpu().numpy().transpose((1, 2, 0)) sil = np.squeeze((sil * 255)).astype(np.uint8) # change from float 0-1 [512,512,1] to uint8 0-255 [512,512] #grow the list of cube, silhouette and parameters cubes_database.extend(image) sils_database.extend(sil) params_database.extend(Rt) im_nr = im_nr+1 if(im_nr%1 == 0): fig = plt.figure() fig.add_subplot(2, 1, 1) plt.imshow(image) imageio.imwrite("3D_objects/{}_ref.png".format(file_name_extension), image) fig.add_subplot(2, 1, 2) plt.imshow(sil, cmap='gray') plt.show() plt.close(fig) imsave('/tmp/_tmp_%04d.png' % processcount, image) processcount = processcount + 1 make_gif(args.filename_output)