def init(Obj_Name, params):
vertices_1, faces_1, textures_1 = nr.load_obj(
"./3D_objects/{}.obj".format(Obj_Name),
load_texture=True) #, texture_size=4)
# print(vertices_1.shape)
# print(faces_1.shape)
vertices_1 = vertices_1[None, :, :] # add dimension
faces_1 = faces_1[None, :, :] #add dimension
textures_1 = textures_1[None, :, :] #add dimension
nb_vertices = vertices_1.shape[0]
# define extrinsic parameter
alpha = params[0]
beta = params[1]
gamma = params[2]
x = params[3]
y = params[4]
z = limit(params[5])
R = np.array([alpha, beta, gamma]) # angle in degree
t = np.array([x, y, z]) # translation in meter
Rt = np.concatenate(
(R, t), axis=None
) # create one array of parameter in radian, this arraz will be saved in .npy file
cam = camera_setttings(
R=R, t=t, vert=nb_vertices
) # degree angle will be converted and stored in radian
renderer = nr.Renderer(image_size=512,
camera_mode='projection',
dist_coeffs=None,
K=cam.K_vertices,
R=cam.R_vertices,
t=cam.t_vertices,
near=1,
background_color=[1, 1, 1],
far=1000,
orig_size=512,
light_intensity_ambient=1.0,
light_intensity_directional=0,
light_direction=[0, 1, 0],
light_color_ambient=[1, 1, 1],
light_color_directional=[1, 1, 1]) # [1,1,1]
return vertices_1, faces_1, textures_1, renderer
def renderingGivenTm(self):
# print('rendering the 3D cad tool')
instrument_to_camera_transform = self.T_m
Extracted_X, Extracted_Y, Extracted_Z, Extracted_theta1_deg, Extracted_theta2_deg, Extracted_theta3_deg = self.matrix2angle(
self.T_m)
# define transfomration parameter from json file
alpha = Extracted_theta1_deg + 90 #adapt from openCV to renderer axis system
beta = Extracted_theta2_deg
gamma = Extracted_theta3_deg
x = Extracted_X
y = Extracted_Y
z = Extracted_Z
# print('parameter found are: ',x, y, z, alpha, beta, gamma)
#renderer the 3D cad model
vertices_1, faces_1, textures_1 = nr.load_obj(
"3D_objects/shaftshortOnly.obj",
load_texture=True,
normalization=False) # , texture_size=4)
vertices_1 = vertices_1[None, :, :] # add dimension
faces_1 = faces_1[None, :, :] # add dimension
textures_1 = textures_1[None, :, :] # add dimension
nb_vertices = vertices_1.shape[0]
R = np.array([np.radians(alpha),
np.radians(beta),
np.radians(gamma)]) # angle in degree
t = np.array([x, y, z]) # translation in meter
# define transformation by transformation matrix
self.Rt = np.concatenate((R, t), axis=None).astype(
np.float16
) # create one array of parameter in radian, this arraz will be saved in .npy file
cam = camera_setttings(
R=R,
t=t,
PnPtm=self.T_m,
PnPtmFlag=False,
vert=nb_vertices,
resolutionx=1280,
resolutiony=1024,
cx=c_x,
cy=c_y,
fx=f_x,
fy=f_y) # degree angle will be converted and stored in radian
renderer = nr.Renderer(
image_size=1280,
camera_mode='projection',
dist_coeffs=None,
anti_aliasing=True,
fill_back=True,
perspective=False,
K=cam.K_vertices,
R=cam.R_vertices,
t=cam.t_vertices,
near=0,
background_color=[1, 1, 1],
# background is filled now with value 0-1 instead of 0-255
# changed from 0-255 to 0-1
far=1,
orig_size=1280,
light_intensity_ambient=1,
light_intensity_directional=0.5,
light_direction=[0, 1, 0],
light_color_ambient=[1, 1, 1],
light_color_directional=[1, 1, 1])
images_1 = renderer(
vertices_1,
faces_1,
textures_1,
K=torch.cuda.FloatTensor(cam.K_vertices),
R=torch.cuda.FloatTensor(cam.R_vertices),
t=torch.cuda.FloatTensor(
cam.t_vertices)) # [batch_size, RGB, image_size, image_size]
image = images_1[0].detach().cpu().numpy()[0].transpose(
(1, 2, 0)) #float32 from 0-1
image = (image * 255).astype(
np.uint8) #cast from float32 255.0 to 255 uint8
self.image = image[0:1024, 0:1280, :]
#modifification done in the rasterize.py file for the default far and near value DEFAULT_FAR = 1 , DEFAULT_EPS = 1e-4
sils_1 = renderer(
vertices_1,
faces_1,
textures_1,
mode='silhouettes',
K=torch.cuda.FloatTensor(cam.K_vertices),
R=torch.cuda.FloatTensor(cam.R_vertices),
t=torch.cuda.FloatTensor(
cam.t_vertices)) # [batch_size, RGB, image_size, image_size]
sil = sils_1.detach().cpu().numpy().transpose((1, 2, 0))
sil = np.squeeze((sil * 255)).astype(
np.uint8
) # change from float 0-1 [512,512,1] to uint8 0-255 [512,512]
self.sil = sil[0:1024, 0:1280]
#create window of the overlap of the tool and renderer
backgroundImage = Image.open("{}/frameL{}.jpg".format(
self.pathfile, self.currentFrameId))
self.backgroundIm = np.array(backgroundImage)
toolbck = backgroundImage.load()
sil3d = self.sil[:, :, np.newaxis]
renderim = np.concatenate((sil3d, sil3d, sil3d), axis=2)
toolIm = Image.fromarray(np.uint8(renderim))
alpha = 0.2
size = 10 #ellipse size
out = Image.blend(backgroundImage, toolIm, alpha)
draw = ImageDraw.Draw(out)
for i in range(len(self.pinhole_point2)):
px = self.pinhole_point2[i, 0]
py = self.pinhole_point2[i, 1]
draw.ellipse(
[px - size / 2, py - size / 2, px + size / 2, py + size / 2],
fill='blue')
# draw.show()
draw.ellipse(
[c_x - size / 2, c_y - size / 2, c_x + size / 2, c_y + size / 2],
fill='red')
self.out = np.array(out)
def __init__(self, filename_obj=None, *args, **kwargs):
super(ModelResNet50, self).__init__(Bottleneck, [3, 4, 6, 3],
num_classes=6,
**kwargs)
# resnet part
self.seq1 = nn.Sequential(self.conv1, self.bn1, self.relu,
self.maxpool, self.layer1, self.layer2)
self.seq2 = nn.Sequential(
self.layer3,
self.layer4,
self.avgpool,
)
self.fc
# self.fc1 = nn.Linear(2*6, 6)
# self.fc2 = nn.Linear(8, 6)
# render part
vertices, faces, textures = nr.load_obj(filename_obj,
load_texture=True,
normalization=False)
vertices = vertices[
None, :, :] # [num_vertices, XYZ] -> [batch_size=1, num_vertices, XYZ]
faces = faces[
None, :, :] # [num_faces, 3] -> [batch_size=1, num_faces, 3
textures = textures[None, :, :]
nb_vertices = vertices.shape[0]
self.register_buffer('vertices', vertices)
self.register_buffer('faces', faces)
self.register_buffer('textures', textures)
# ---------------------------------------------------------------------------------
# extrinsic parameter, link world/object coordinate to camera coordinate
# ---------------------------------------------------------------------------------
alpha = np.radians(0)
beta = np.radians(0)
gamma = np.radians(0)
x = 0 # uniform(-2, 2)
y = 0 # uniform(-2, 2)
z = 0.08 # uniform(5, 10) #1000t was done with value between 7 and 10, Rot and trans between 5 10
R = np.array([np.radians(alpha),
np.radians(beta),
np.radians(gamma)]) # angle in degree
t = np.array([x, y, z]) # translation in meter
batch = vertices.shape[0]
# ---------------------------------------------------------------------------------
# intrinsic parameter
# ---------------------------------------------------------------------------------
c_x = 590
c_y = 508
f_x = 1067
f_y = 1067
# camera_calibration = np.zeros((4, 4))
# camera_calibration[0, 0] = f_x
# camera_calibration[1, 1] = f_y
# camera_calibration[0, 2] = c_x
# camera_calibration[1, 2] = c_y
# camera_calibration[2, 2] = 1
cam = camera_setttings(
R=R,
t=t,
PnPtm=0,
PnPtmFlag=False,
vert=nb_vertices,
resolutionx=1280,
resolutiony=1024,
cx=c_x,
cy=c_y,
fx=f_x,
fy=f_y) # degree angle will be converted and stored in radian
# K = np.array([[f / pix_sizeX, 0, Cam_centerX],
# [0, f / pix_sizeY, Cam_centerY],
# [0, 0, 1]]) # shape of [nb_vertice, 3, 3]
#
# K = np.repeat(K[np.newaxis, :, :], batch, axis=0) # shape of [batch=1, 3, 3]
# R = np.repeat(R[np.newaxis, :, :], batch, axis=0) # shape of [batch=1, 3, 3]
# t = np.repeat(t[np.newaxis, :], 1, axis=0) # shape of [1, 3]
#
# self.K = K
# self.R = R
# # -------------------------- working block translation
# self.tx = torch.from_numpy(np.array(x, dtype=np.float32)).cuda()
# self.ty = torch.from_numpy(np.array(y, dtype=np.float32)).cuda()
# self.tz = torch.from_numpy(np.array(z, dtype=np.float32)).cuda()
# self.t =torch.from_numpy(np.array([self.tx, self.ty, self.tz], dtype=np.float32)).unsqueeze(0)
# --------------------------
# setup renderer
renderer = nr.Renderer(
image_size=1280,
camera_mode='projection',
dist_coeffs=None,
anti_aliasing=True,
fill_back=True,
perspective=False,
K=cam.K_vertices,
R=cam.R_vertices,
t=cam.t_vertices,
near=0,
background_color=[1, 1, 1],
# background is filled now with value 0-1 instead of 0-255
# changed from 0-255 to 0-1
far=1,
orig_size=1280,
light_intensity_ambient=1,
light_intensity_directional=0.5,
light_direction=[0, 1, 0],
light_color_ambient=[1, 1, 1],
light_color_directional=[1, 1, 1])
self.renderer = renderer
def main():
cubes_database = []
sils_database = []
params_database = []
im_nr = 1
vertices_1, faces_1, textures_1 = nr.load_obj(
"3D_objects/wrist.obj", load_texture=True) #, texture_size=4)
print(vertices_1.shape)
print(faces_1.shape)
vertices_1 = vertices_1[None, :, :] # add dimension
faces_1 = faces_1[None, :, :] #add dimension
textures_1 = textures_1[None, :, :] #add dimension
nb_vertices = vertices_1.shape[0]
print(vertices_1.shape)
print(faces_1.shape)
file_name_extension = 'ExampleDatabaseOf10images'
nb_im = 10
#init and create renderer object
R = np.array([np.radians(0), np.radians(0),
np.radians(0)]) # angle in degree
t = np.array([0, 0, 0]) # translation in meter
cam = camera_setttings(R=R, t=t, vert=nb_vertices)
renderer = nr.Renderer(
image_size=512,
camera_mode='projection',
dist_coeffs=None,
K=cam.K_vertices,
R=cam.R_vertices,
t=cam.t_vertices,
near=1,
background_color=[
0, 0, 0
], #background is filled now with value 0-1 instead of 0-255
# changed from 0-255 to 0-1
far=1000,
orig_size=512,
light_intensity_ambient=1.0,
light_intensity_directional=0,
light_direction=[0, 1, 0],
light_color_ambient=[1, 1, 1],
light_color_directional=[1, 1, 1])
loop = tqdm.tqdm(range(0, nb_im))
for i in loop:
# define transfomration parameter randomly uniform
alpha = uniform(0, 180)
beta = uniform(0, 180)
gamma = uniform(0, 180)
x = uniform(-1.5, 1.5)
y = uniform(-1.5, 1.5)
z = uniform(
5, 7
) #1000t was done with value between 7 and 10, Rot and trans between 5 10
R = np.array([np.radians(alpha),
np.radians(beta),
np.radians(gamma)]) # angle in degree
t = np.array([x, y, z]) # translation in meter
Rt = np.concatenate((R, t), axis=None).astype(
np.float16
) # create one array of parameter in radian, this arraz will be saved in .npy file
cam = camera_setttings(
R=R, t=t, vert=nb_vertices
) # degree angle will be converted and stored in radian
images_1 = renderer(
vertices_1,
faces_1,
textures_1,
K=torch.cuda.FloatTensor(cam.K_vertices),
R=torch.cuda.FloatTensor(cam.R_vertices),
t=torch.cuda.FloatTensor(
cam.t_vertices)) # [batch_size, RGB, image_size, image_size]
image = images_1[0].detach().cpu().numpy()[0].transpose(
(1, 2, 0)) #float32 from 0-1
image = (image * 255).astype(
np.uint8) #cast from float32 255.0 to 255 uint8
sils_1 = renderer(
vertices_1,
faces_1,
textures_1,
mode='silhouettes',
K=torch.cuda.FloatTensor(cam.K_vertices),
R=torch.cuda.FloatTensor(cam.R_vertices),
t=torch.cuda.FloatTensor(
cam.t_vertices)) # [batch_size, RGB, image_size, image_size]
sil = sils_1.detach().cpu().numpy().transpose((1, 2, 0))
sil = np.squeeze((sil * 255)).astype(
np.uint8
) # change from float 0-1 [512,512,1] to uint8 0-255 [512,512]
#grow the list of cube, silhouette and parameters
cubes_database.extend(image)
sils_database.extend(sil)
params_database.extend(Rt)
im_nr = im_nr + 1
# #put surgical iamge as a background
# BinarySil3layermask = (np.array([sil, sil,sil])).transpose((1, 2, 0))/255
# # plt.imshow(BinarySil3layermask)
# numberbackground = random.randint(1,8)
# backgroundImg = mpimg.imread("3D_objects/background{}.jpg".format(numberbackground))
#
# sx = backgroundImg.shape[0]
# sy = backgroundImg.shape[1]
#
# moveX = random.randint(0,sx-512)
# moveY = random.randint(0,sy-512)
# # print(moveX, moveY)
# cropedbackgroundImg = backgroundImg[moveX:moveX+512, moveY:moveY+512, :]
# maskedbackground = np.multiply((BinarySil3layermask *-1+1), cropedbackgroundImg/255)
# imWithBackground = (image + (maskedbackground*255)).astype(np.uint8)
# # plt.imshow(imWithBackground)
# image = imWithBackground
# cubes_database.extend(image)
# if(im_nr%1 == 0):
# fig = plt.figure()
# fig.add_subplot(2, 1, 1)
# plt.imshow(image)
# imageio.imwrite("3D_objects/{}_ref.png".format(file_name_extension), image)
#
# fig.add_subplot(2, 1, 2)
# plt.imshow(sil, cmap='gray')
# plt.show()
# plt.close(fig)
# save database
# reshape in the form (nbr of image, x dim, y dim, layers)
cubes_database = np.reshape(cubes_database,
(im_nr - 1, 512, 512, 3)) # 3 channel rgb
sils_database = np.reshape(sils_database,
(im_nr - 1, 512, 512)) #binary mask monochannel
params_database = np.reshape(params_database,
(im_nr - 1, 6)) #array of 6 params
np.save('Npydatabase/cubes_{}.npy'.format(file_name_extension),
cubes_database)
np.save('Npydatabase/sils_{}.npy'.format(file_name_extension),
sils_database)
np.save('Npydatabase/params_{}.npy'.format(file_name_extension),
params_database)
print('images saved')
def main():
cubes_database = []
sils_database = []
params_database = []
im_nr = 1
vertices_1, faces_1, textures_1 = nr.load_obj(
"3D_objects/wrist.obj", load_texture=True) #, texture_size=4)
print(vertices_1.shape)
print(faces_1.shape)
vertices_1 = vertices_1[None, :, :] # add dimension
faces_1 = faces_1[None, :, :] #add dimension
textures_1 = textures_1[None, :, :] #add dimension
nb_vertices = vertices_1.shape[0]
print(vertices_1.shape)
print(faces_1.shape)
nb_im = 1000
file_name_extension = 'exampleOfNameForTheTimelapse'.format(nb_im)
parser = argparse.ArgumentParser()
parser.add_argument('-or',
'--filename_output',
type=str,
default=os.path.join(
result_dir,
'Animation_{}.gif'.format(file_name_extension)))
args = parser.parse_args()
#init and create renderer object
R = np.array([np.radians(0), np.radians(0),
np.radians(0)]) # angle in degree
t = np.array([0, 0, 0]) # translation in meter
cam = camera_setttings(R=R, t=t, vert=nb_vertices)
renderer = nr.Renderer(
image_size=512,
camera_mode='projection',
dist_coeffs=None,
K=cam.K_vertices,
R=cam.R_vertices,
t=cam.t_vertices,
near=1,
background_color=[
0, 0, 0
], #background is filled now with value 0-1 instead of 0-255
# changed from 0-255 to 0-1
far=1000,
orig_size=512,
light_intensity_ambient=1.0,
light_intensity_directional=0,
light_direction=[0, 1, 0],
light_color_ambient=[1, 1, 1],
light_color_directional=[1, 1, 1])
prevA = 0
prevB = 0
prevG = 0
circlepoint = PointsInCircum(1.4, nb_im)
Z_sinus = np.linspace(0, 360, nb_im)
Z_sinus = 6 + np.sin(np.radians(Z_sinus))
circlepoint_alpha = np.linspace(0, 180, nb_im)
circlepoint_beta = np.linspace(180, 0, nb_im)
circlepoint_gamma = np.linspace(0, 180, nb_im)
print('z axis translation min max are {} and {}'.format(
Z_sinus.min(),
Z_sinus.max())) #5.000000015261379 and 6.999998763828597
loop = tqdm.tqdm(range(0, nb_im))
for i in loop:
# define transfomration parameter randomly uniform
#------INSERT HERE SPECIFIC VALUE OF TRANSLATION AND ROTATION -----------------------------------------------
alpha = 0 # circlepoint_alpha[i]
beta = 0 #circlepoint_beta[i]
gamma = circlepoint_gamma[i]
x = circlepoint[i][0]
y = circlepoint[i][1]
z = Z_sinus[
i] #1000t was done with value between 7 and 10, Rot and trans between 5 10
R = np.array([np.radians(alpha),
np.radians(beta),
np.radians(gamma)]) # angle in degree
t = np.array([x, y, z]) # translation in meter
Rt = np.concatenate((R, t), axis=None).astype(
np.float16
) # create one array of parameter in radian, this arraz will be saved in .npy file
cam = camera_setttings(
R=R, t=t, vert=nb_vertices
) # degree angle will be converted and stored in radian
images_1 = renderer(
vertices_1,
faces_1,
textures_1,
K=torch.cuda.FloatTensor(cam.K_vertices),
R=torch.cuda.FloatTensor(cam.R_vertices),
t=torch.cuda.FloatTensor(
cam.t_vertices)) # [batch_size, RGB, image_size, image_size]
image = images_1[0].detach().cpu().numpy()[0].transpose(
(1, 2, 0)) #float32 from 0-1
img = image
image = (image * 255).astype(
np.uint8) #cast from float32 255.0 to 255 uint8
sils_1 = renderer(
vertices_1,
faces_1,
textures_1,
mode='silhouettes',
K=torch.cuda.FloatTensor(cam.K_vertices),
R=torch.cuda.FloatTensor(cam.R_vertices),
t=torch.cuda.FloatTensor(
cam.t_vertices)) # [batch_size, RGB, image_size, image_size]
sil = sils_1.detach().cpu().numpy().transpose((1, 2, 0))
sil = np.squeeze((sil * 255)).astype(
np.uint8
) # change from float 0-1 [512,512,1] to uint8 0-255 [512,512]
#grow the list of cube, silhouette and parameters
cubes_database.extend(image)
sils_database.extend(sil)
params_database.extend(Rt)
im_nr = im_nr + 1
# #put surgical iamge as a background
# BinarySil3layermask = (np.array([sil, sil,sil])).transpose((1, 2, 0))/255
# # plt.imshow(BinarySil3layermask)
# numberbackground = random.randint(1,8)
# backgroundImg = mpimg.imread("3D_objects/background{}.jpg".format(numberbackground))
#
# sx = backgroundImg.shape[0]
# sy = backgroundImg.shape[1]
#
# moveX = random.randint(0,sx-512)
# moveY = random.randint(0,sy-512)
# # print(moveX, moveY)
# cropedbackgroundImg = backgroundImg[moveX:moveX+512, moveY:moveY+512, :]
# maskedbackground = np.multiply((BinarySil3layermask *-1+1), cropedbackgroundImg/255)
# imWithBackground = (image + (maskedbackground*255)).astype(np.uint8)
# # plt.imshow(imWithBackground)
# image = imWithBackground
# cubes_database.extend(image)
# if(im_nr%50 == 0):
# fig = plt.figure()
# fig.add_subplot(2, 1, 1)
# plt.imshow(image)
# imageio.imwrite("3D_objects/{}_ref.png".format(file_name_extension), image)
#
# fig.add_subplot(2, 1, 2)
# plt.imshow(sil, cmap='gray')
# plt.show()
# plt.close(fig)
#
imsave('/tmp/_tmp_%04d.png' % i, img)
# save database
make_gif(args.filename_output)
# reshape in the form (nbr of image, x dim, y dim, layers)
cubes_database = np.reshape(cubes_database,
(im_nr - 1, 512, 512, 3)) # 3 channel rgb
sils_database = np.reshape(sils_database,
(im_nr - 1, 512, 512)) #binary mask monochannel
params_database = np.reshape(params_database,
(im_nr - 1, 6)) #array of 6 params
np.save('Npydatabase/cubes_{}.npy'.format(file_name_extension),
cubes_database)
np.save('Npydatabase/sils_{}.npy'.format(file_name_extension),
sils_database)
np.save('Npydatabase/params_{}.npy'.format(file_name_extension),
params_database)
print('images saved')
def main():
cubes_database = []
sils_database = []
params_database = []
im_nr = 1
vertices_1, faces_1, textures_1 = nr.load_obj("3D_objects/shaft.obj", load_texture=True) #, texture_size=4)
print(vertices_1.shape)
print(faces_1.shape)
vertices_1 = vertices_1[None, :, :] # add dimension
faces_1 = faces_1[None, :, :] #add dimension
textures_1 = textures_1[None, :, :] #add dimension
nb_vertices = vertices_1.shape[0]
print(vertices_1.shape)
print(faces_1.shape)
file_name_extension = 'shaft_1im_180_M15_15_5_7'
nb_im = 1
#init and create renderer object
# R = np.array([np.radians(0), np.radians(0), np.radians(0)]) # angle in degree
# t = np.array([0, 0, 0]) # translation in meter
cam = camera_setttings()
renderer = nr.Renderer(image_size=1280, camera_mode='projection', dist_coeffs=None,
K=cam.K_vertices, R=cam.R_vertices, t=cam.t_vertices, near=1, background_color=[1, 1, 1], #background is filled now with value 0-1 instead of 0-255
# changed from 0-255 to 0-1
far=1000, orig_size=674,
light_intensity_ambient=0.7, light_intensity_directional=0.5, light_direction=[0, 1, 0],
light_color_ambient=[1, 1, 1], light_color_directional=[1, 1, 1])
## -------------------------read json file -------------------------------------------
with open('data/data.json') as json_file:
data = json.load(json_file)
data_len = len(data)
# usm_camera = data[0:data_len]['usm-1']
## --------------------------------------------------------------------------------
loop = tqdm.tqdm(range(0, nb_im))
for i in loop:
## -------------------------extract json frame matrix -------------------------------------------
usm_camera = data[i]['usm-1']
usm_inst = data[i]['usm-2']
instrument_to_camera_transform = np.asarray([list(map(float, usm_inst['pose'][0])),
list(map(float, usm_inst['pose'][1])),
list(map(float, usm_inst['pose'][2])),
list(map(float, usm_inst['pose'][3]))],
dtype=np.float64)
#to test the conversion degree to radian to transformation matrix and then back to euler angle in radian
R_test = np.array([np.radians(0),np.radians(0),np.radians(0)]) #test value alpha beta gamma
T_test_vector, R_test_matrix = BuildTransformationMatrix(tx=0, ty=0, tz=0, alpha=R_test[0], beta=R_test[1], gamma=R_test[2])
instrument_to_camera_transform[0,0:3] = R_test_matrix[0,:]
instrument_to_camera_transform[1,0:3] = R_test_matrix[1,:]
instrument_to_camera_transform[2,0:3] = R_test_matrix[2,:]
joint_values = np.asarray([list(map(float, usm_inst['articulation'][0])),
list(map(float, usm_inst['articulation'][1])),
list(map(float, usm_inst['articulation'][2]))],
dtype=np.float64)
joint_values[-1] = 2 * joint_values[-1]
# print(instrument_to_camera_transform[1,2]) # [row column]
#formula from http://planning.cs.uiuc.edu/node103.html
# alpha and gamma were swapped, don-t know where the problem is but value are correct
Extracted_theta3_rad = m.atan2(instrument_to_camera_transform[1,0],instrument_to_camera_transform[0,0])
C_2 = m.sqrt(instrument_to_camera_transform[2,1]*instrument_to_camera_transform[2,1] + instrument_to_camera_transform[2,2]*instrument_to_camera_transform[2,2])
Extracted_theta2_rad = m.atan2(-instrument_to_camera_transform[2,0], C_2 )
Extracted_theta1_rad = m.atan2(instrument_to_camera_transform[2,1],instrument_to_camera_transform[2,2])
#formula from https://d3cw3dd2w32x2b.cloudfront.net/wp-content/uploads/2012/07/euler-angles1.pdf
# Extracted_theta1_rad = m.atan2(-instrument_to_camera_transform[1,2],instrument_to_camera_transform[2,2])
# C_2 = m.sqrt(instrument_to_camera_transform[0,0]*instrument_to_camera_transform[0,0] + instrument_to_camera_transform[0,1]*instrument_to_camera_transform[0,1])
# Extracted_theta2_rad = m.atan2(instrument_to_camera_transform[0,1], C_2 )
# s_1 = m.sin(Extracted_theta1_rad)
# c_1 = m.cos(Extracted_theta1_rad)
# Extracted_theta3_rad = m.atan2(s_1*instrument_to_camera_transform[2,0]-c_1*instrument_to_camera_transform[1,0],
# c_1*instrument_to_camera_transform[1,1]-s_1*instrument_to_camera_transform[2,1])
Extracted_X = instrument_to_camera_transform[0,3]
Extracted_Y = instrument_to_camera_transform[1,3]
Extracted_Z = instrument_to_camera_transform[2,3]
Extracted_theta1_deg = np.degrees(Extracted_theta1_rad)
Extracted_theta2_deg = np.degrees(Extracted_theta2_rad)
Extracted_theta3_deg = np.degrees(Extracted_theta3_rad)
# # define transfomration parameter from json file
# alpha =Extracted_theta1_deg
# beta = Extracted_theta2_deg
# gamma = Extracted_theta3_deg
# x = Extracted_X
# y = Extracted_Y
# z = Extracted_Z
#
# # define transfomration parameter randomly uniform
alpha =0#uniform(0, 180)
beta = 0#uniform(0, 180)
gamma = 0 #uniform(0, 180)
x = 0 #uniform(-1.5, 1.5)
y = 0 #uniform(-1.5, 1.5)
z = 0.2 #uniform(5, 7) #1000t was done with value between 7 and 10, Rot and trans between 5 10
R = np.array([np.radians(alpha), np.radians(beta), np.radians(gamma)]) # angle in degree
t = np.array([x, y, z]) # translation in meter
Rt = np.concatenate((R, t), axis=None).astype(np.float16) # create one array of parameter in radian, this arraz will be saved in .npy file
cam = camera_setttings(R=R, t=t, vert=nb_vertices, resolutionx=1280, resolutiony=1024,cx=590, cy=508, fx=1067, fy=1067) # degree angle will be converted and stored in radian
images_1 = renderer(vertices_1, faces_1, textures_1,
K=torch.cuda.FloatTensor(cam.K_vertices),
R=torch.cuda.FloatTensor(cam.R_vertices),
t=torch.cuda.FloatTensor(cam.t_vertices)) # [batch_size, RGB, image_size, image_size]
image = images_1[0].detach().cpu().numpy()[0].transpose((1, 2, 0)) #float32 from 0-1
image = (image*255).astype(np.uint8) #cast from float32 255.0 to 255 uint8
sils_1 = renderer(vertices_1, faces_1, textures_1,
mode='silhouettes',
K=torch.cuda.FloatTensor(cam.K_vertices),
R=torch.cuda.FloatTensor(cam.R_vertices),
t=torch.cuda.FloatTensor(cam.t_vertices)) # [batch_size, RGB, image_size, image_size]
sil = sils_1.detach().cpu().numpy().transpose((1, 2, 0))
sil = np.squeeze((sil * 255)).astype(np.uint8) # change from float 0-1 [512,512,1] to uint8 0-255 [512,512]
#grow the list of cube, silhouette and parameters
cubes_database.extend(image)
sils_database.extend(sil)
params_database.extend(Rt)
im_nr = im_nr+1
# #put surgical iamge as a background
# BinarySil3layermask = (np.array([sil, sil,sil])).transpose((1, 2, 0))/255
# # plt.imshow(BinarySil3layermask)
# numberbackground = random.randint(1,8)
# backgroundImg = mpimg.imread("3D_objects/background{}.jpg".format(numberbackground))
#
# sx = backgroundImg.shape[0]
# sy = backgroundImg.shape[1]
#
# moveX = random.randint(0,sx-512)
# moveY = random.randint(0,sy-512)
# # print(moveX, moveY)
# cropedbackgroundImg = backgroundImg[moveX:moveX+512, moveY:moveY+512, :]
# maskedbackground = np.multiply((BinarySil3layermask *-1+1), cropedbackgroundImg/255)
# imWithBackground = (image + (maskedbackground*255)).astype(np.uint8)
# # plt.imshow(imWithBackground)
# image = imWithBackground
# cubes_database.extend(image)
if(im_nr%1 == 0):
fig = plt.figure()
fig.add_subplot(2, 1, 1)
plt.imshow(image)
imageio.imwrite("3D_objects/{}_ref.png".format(file_name_extension), image)
fig.add_subplot(2, 1, 2)
plt.imshow(sil, cmap='gray')
plt.show()
plt.close(fig)
# save database
# reshape in the form (nbr of image, x dim, y dim, layers)
cubes_database = np.reshape(cubes_database, (im_nr-1, 512, 512, 3)) # 3 channel rgb
sils_database = np.reshape(sils_database, (im_nr-1, 512, 512)) #binary mask monochannel
params_database = np.reshape(params_database,(im_nr-1, 6)) #array of 6 params
np.save('Npydatabase/cubes_{}.npy'.format(file_name_extension), cubes_database)
np.save('Npydatabase/sils_{}.npy'.format(file_name_extension), sils_database)
np.save('Npydatabase/params_{}.npy'.format(file_name_extension), params_database)
print('images saved')
def main():
cubes_database = []
sils_database = []
params_database = []
im_nr = 1
vertices_1, faces_1, textures_1 = nr.load_obj("3D_objects/rubik_color.obj", load_texture=True)#, texture_size=4)
print(vertices_1.shape)
print(faces_1.shape)
vertices_1 = vertices_1[None, :, :] # add dimension
faces_1 = faces_1[None, :, :] #add dimension
textures_1 = textures_1[None, :, :] #add dimension
nb_vertices = vertices_1.shape[0]
print(vertices_1.shape)
print(faces_1.shape)
file_name_extension = '_10'
nb_im = 10
#init and create renderer object
R = np.array([np.radians(0), np.radians(0), np.radians(0)]) # angle in degree
t = np.array([0, 0, 0]) # translation in meter
cam = camera_setttings(R=R, t=t, vert=nb_vertices)
renderer = nr.Renderer(image_size=512, camera_mode='projection', dist_coeffs=None,
K=cam.K_vertices, R=cam.R_vertices, t=cam.t_vertices, near=1, background_color=[1, 1, 1], #background is filled now with value 0-1 instead of 0-255
# changed from 0-255 to 0-1
far=1000, orig_size=512,
light_intensity_ambient=1.0, light_intensity_directional=0, light_direction=[0, 1, 0],
light_color_ambient=[1, 1, 1], light_color_directional=[1, 1, 1])
loop = tqdm.tqdm(range(0, nb_im))
for i in loop:
# define transfomration parameter randomly uniform
alpha = uniform(0, 180)
beta = uniform(0, 180)
gamma = uniform(0, 180)
x = uniform(-2, 2)
y = uniform(-2, 2)
z = uniform(8, 14)
R = np.array([np.radians(alpha), np.radians(beta), np.radians(gamma)]) # angle in degree
t = np.array([x, y, z]) # translation in meter
Rt = np.concatenate((R, t), axis=None).astype(np.float16) # create one array of parameter in radian, this arraz will be saved in .npy file
cam = camera_setttings(R=R, t=t, vert=nb_vertices) # degree angle will be converted and stored in radian
images_1 = renderer(vertices_1, faces_1, textures_1,
K=torch.cuda.FloatTensor(cam.K_vertices),
R=torch.cuda.FloatTensor(cam.R_vertices),
t=torch.cuda.FloatTensor(cam.t_vertices)) # [batch_size, RGB, image_size, image_size]
image = images_1[0].detach().cpu().numpy()[0].transpose((1, 2, 0)) #float32 from 0-1
image = (image*255).astype(np.uint8) #cast from float32 255.0 to 255 uint8
sils_1 = renderer(vertices_1, faces_1, textures_1,
mode='silhouettes',
K=torch.cuda.FloatTensor(cam.K_vertices),
R=torch.cuda.FloatTensor(cam.R_vertices),
t=torch.cuda.FloatTensor(cam.t_vertices)) # [batch_size, RGB, image_size, image_size]
sil = sils_1.detach().cpu().numpy().transpose((1, 2, 0))
sil = np.squeeze((sil * 255)).astype(np.uint8) # change from float 0-1 [512,512,1] to uint8 0-255 [512,512]
#grow the list of cube, silhouette and parameters
cubes_database.extend(image)
sils_database.extend(sil)
params_database.extend(Rt)
im_nr = im_nr+1
if(im_nr%1 == 0):
fig = plt.figure()
fig.add_subplot(1, 2, 1)
plt.imshow(image)
fig.add_subplot(1, 2, 2)
plt.imshow(sil, cmap='gray')
plt.show()
plt.close(fig)
# save database
# reshape in the form (nbr of image, x dim, y dim, layers)
cubes_database = np.reshape(cubes_database, (im_nr-1, 512, 512, 3)) # 3 channel rgb
sils_database = np.reshape(sils_database, (im_nr-1, 512, 512)) #binary mask monochannel
params_database = np.reshape(params_database,(im_nr-1, 6)) #array of 6 params
np.save('Npydatabase/cubes_{}.npy'.format(file_name_extension), cubes_database)
np.save('Npydatabase/sils_{}.npy'.format(file_name_extension), sils_database)
np.save('Npydatabase/params_{}.npy'.format(file_name_extension), params_database)
print('images saved')
def main():
processcount = 0
current_dir = os.path.dirname(os.path.realpath(__file__))
data_dir = os.path.join(current_dir, 'data')
parser = argparse.ArgumentParser()
parser.add_argument('-or', '--filename_output', type=str,
default=os.path.join(current_dir, 'ResultRender_{}.gif'.format('test2')))
parser.add_argument('-mr', '--make_reference_image', type=int, default=0)
parser.add_argument('-g', '--gpu', type=int, default=0)
args = parser.parse_args()
cubes_database = []
sils_database = []
params_database = []
im_nr = 1
vertices_1, faces_1, textures_1 = nr.load_obj("3D_objects/LongShaft3.obj", load_texture=True, normalization=False) #, texture_size=4)
print(vertices_1.shape)
print(faces_1.shape)
vertices_1 = vertices_1[None, :, :] # add dimension
faces_1 = faces_1[None, :, :] #add dimension
textures_1 = textures_1[None, :, :] #add dimension
nb_vertices = vertices_1.shape[0]
print(vertices_1.shape)
print(faces_1.shape)
file_name_extension = 'shaft_1im_180_M15_15_5_7'
nb_im = 100
useTransformMatrix = False
instrument_to_camera_transform = np.array([(0, 0, 0, 0),
(0, 0, 0, 0),
(0, 0, 0, 0),
(0, 0, 0, 1)])
instrument_to_camera_transform = np.zeros((4,4))
## --------------------------------------------------------------------------------
loop = tqdm.tqdm(range(0, nb_im))
for i in loop:
# # define transfomration parameter from angle and translation
if useTransformMatrix :
tempx = 145
tempy = 52
tempz = -114
R_test = np.array([np.radians(tempx), np.radians(tempy), np.radians(tempz)]) # test value alpha beta gamma
T_test = np.array([0.017, -0.008, 0.1])
print(T_test, tempx, tempy, tempz)
T_test_vector, R_test_matrix = BuildTransformationMatrix(tx=T_test[0], ty=T_test[1], tz=T_test[2], alpha=R_test[0], beta=R_test[1],
gamma=R_test[2]) #return 1x3 vector and 3x3matrix
instrument_to_camera_transform[0, 0:3] = R_test_matrix[0, :]
instrument_to_camera_transform[1, 0:3] = R_test_matrix[1, :]
instrument_to_camera_transform[2, 0:3] = R_test_matrix[2, :]
instrument_to_camera_transform[0, 3] = T_test[0]
instrument_to_camera_transform[1, 3] = T_test[1]
instrument_to_camera_transform[2, 3] = T_test[2]
instrument_to_camera_transform[3, 3] = 1
# test[0:3,0:3] = R_test_matrix
# formula from http://planning.cs.uiuc.edu/node103.html
# alpha and gamma were swapped, don-t know where the problem is but value are correct
Extracted_theta3_rad = m.atan2(instrument_to_camera_transform[1, 0], instrument_to_camera_transform[0, 0])
C_2 = m.sqrt(instrument_to_camera_transform[2, 1] * instrument_to_camera_transform[2, 1] +
instrument_to_camera_transform[2, 2] * instrument_to_camera_transform[2, 2])
Extracted_theta2_rad = m.atan2(-instrument_to_camera_transform[2, 0], C_2)
Extracted_theta1_rad = m.atan2(instrument_to_camera_transform[2, 1], instrument_to_camera_transform[2, 2])
Extracted_X = instrument_to_camera_transform[0, 3]
Extracted_Y = instrument_to_camera_transform[1, 3]
Extracted_Z = instrument_to_camera_transform[2, 3]
Extracted_theta1_deg = np.degrees(Extracted_theta1_rad)
Extracted_theta2_deg = np.degrees(Extracted_theta2_rad)
Extracted_theta3_deg = np.degrees(Extracted_theta3_rad)
# define transfomration parameter from json fileprint(x,y,z, alpha,beta,gamma)
alpha =Extracted_theta1_deg
beta = Extracted_theta2_deg
gamma = Extracted_theta3_deg
x = Extracted_X
y = Extracted_Y
z = Extracted_Z
print(x,y,z, alpha,beta,gamma)
else:
alpha =np.degrees(00) #uniform(0, 180)
beta = np.degrees(00) #uniform(0, 180)
gamma = np.degrees(0) #uniform(0, 180)
x =0#uniform(-1.5, 1.5)
y =0 #uniform(-1.5, 1.5)
z = 8.5381e-02#uniform(5, 7) #1000t was done with value between 7 and 10, Rot and trans between 5 10
print(x, y, z, alpha, beta, gamma)
R = np.array([np.radians(alpha), np.radians(beta), np.radians(gamma)]) # angle in degree
t = np.array([x, y, z]) # translation in meter
# define transformation by transformation matrix
Rt = np.concatenate((R, t), axis=None).astype(np.float16) # create one array of parameter in radian, this arraz will be saved in .npy file
cam = camera_setttings(R=R, t=t, vert=nb_vertices, resolutionx=1280, resolutiony=1024,cx=590, cy=508, fx=1067, fy=1067) # degree angle will be converted and stored in radian
renderer = nr.Renderer(image_size=1280, camera_mode='projection', dist_coeffs=None,anti_aliasing=True, fill_back=True, perspective=False,
K=cam.K_vertices, R=cam.R_vertices, t=cam.t_vertices, near=0, background_color=[1, 1, 1],
# background is filled now with value 0-1 instead of 0-255
# changed from 0-255 to 0-1
far=20, orig_size=1280,
light_intensity_ambient=1, light_intensity_directional=0.5, light_direction=[0, 1, 0],
light_color_ambient=[1, 1, 1], light_color_directional=[1, 1, 1])
images_1 = renderer(vertices_1, faces_1, textures_1,
K=torch.cuda.FloatTensor(cam.K_vertices),
R=torch.cuda.FloatTensor(cam.R_vertices),
t=torch.cuda.FloatTensor(cam.t_vertices)) # [batch_size, RGB, image_size, image_size]
image = images_1[0].detach().cpu().numpy()[0].transpose((1, 2, 0)) #float32 from 0-1
image = (image*255).astype(np.uint8) #cast from float32 255.0 to 255 uint8
sils_1 = renderer(vertices_1, faces_1, textures_1,
mode='silhouettes',
K=torch.cuda.FloatTensor(cam.K_vertices),
R=torch.cuda.FloatTensor(cam.R_vertices),
t=torch.cuda.FloatTensor(cam.t_vertices)) # [batch_size, RGB, image_size, image_size]
sil = sils_1.detach().cpu().numpy().transpose((1, 2, 0))
sil = np.squeeze((sil * 255)).astype(np.uint8) # change from float 0-1 [512,512,1] to uint8 0-255 [512,512]
#grow the list of cube, silhouette and parameters
cubes_database.extend(image)
sils_database.extend(sil)
params_database.extend(Rt)
im_nr = im_nr+1
if(im_nr%1 == 0):
fig = plt.figure()
fig.add_subplot(2, 1, 1)
plt.imshow(image)
imageio.imwrite("3D_objects/{}_ref.png".format(file_name_extension), image)
fig.add_subplot(2, 1, 2)
plt.imshow(sil, cmap='gray')
plt.show()
plt.close(fig)
imsave('/tmp/_tmp_%04d.png' % processcount, image)
processcount = processcount + 1
make_gif(args.filename_output)