def __init__(self,
width=512,
height=512,
gpu_id=0,
render_rotation=False,
render_marker=False):
self.render_marker = render_marker
self.VAOs = []
self.VBOs = []
self.textures = []
self.is_textured = []
self.objects = []
self.texUnitUniform = None
self.width = width
self.height = height
self.faces = []
self.poses_trans = []
self.poses_rot = []
self.instances = []
self.render_rotation = render_rotation
self.r = CppYCBRenderer.CppYCBRenderer(width, height,
get_available_devices()[gpu_id])
self.r.init()
self.glstring = GL.glGetString(GL.GL_VERSION)
from OpenGL.GL import shaders
self.shaders = shaders
self.colors = [[0.9, 0, 0], [0.6, 0, 0], [0.3, 0, 0]]
self.lightcolor = [1, 1, 1]
vertexShader = self.shaders.compileShader(
"""
#version 460
uniform mat4 V;
uniform mat4 P;
uniform mat4 pose_rot;
uniform mat4 pose_trans;
uniform vec3 instance_color;
layout (location=0) in vec3 position;
layout (location=1) in vec3 normal;
layout (location=2) in vec2 texCoords;
out vec2 theCoords;
out vec3 Normal;
out vec3 FragPos;
out vec3 Normal_cam;
out vec3 Instance_color;
out vec3 Pos_cam;
out vec3 Pos_obj;
void main() {
gl_Position = P * V * pose_trans * pose_rot * vec4(position, 1);
vec4 world_position4 = pose_trans * pose_rot * vec4(position, 1);
FragPos = vec3(world_position4.xyz / world_position4.w); // in world coordinate
Normal = normalize(mat3(pose_rot) * normal); // in world coordinate
Normal_cam = normalize(mat3(V) * mat3(pose_rot) * normal); // in camera coordinate
vec4 pos_cam4 = V * pose_trans * pose_rot * vec4(position, 1);
Pos_cam = pos_cam4.xyz / pos_cam4.w;
Pos_obj = position;
theCoords = texCoords;
Instance_color = instance_color;
}
""", GL.GL_VERTEX_SHADER)
fragmentShader = self.shaders.compileShader(
"""
#version 460
uniform sampler2D texUnit;
in vec2 theCoords;
in vec3 Normal;
in vec3 Normal_cam;
in vec3 FragPos;
in vec3 Instance_color;
in vec3 Pos_cam;
in vec3 Pos_obj;
layout (location = 0) out vec4 outputColour;
layout (location = 1) out vec4 NormalColour;
layout (location = 2) out vec4 InstanceColour;
layout (location = 3) out vec4 PCObject;
layout (location = 4) out vec4 PCColour;
uniform vec3 light_position; // in world coordinate
uniform vec3 light_color; // light color
void main() {
float ambientStrength = 0.2;
vec3 ambient = ambientStrength * light_color;
vec3 lightDir = normalize(light_position - FragPos);
float diff = max(dot(Normal, lightDir), 0.0);
vec3 diffuse = diff * light_color;
outputColour = texture(texUnit, theCoords) * vec4(diffuse + ambient, 1);
NormalColour = vec4((Normal_cam + 1) / 2,1);
InstanceColour = vec4(Instance_color,1);
PCObject = vec4(Pos_obj,1);
PCColour = vec4(Pos_cam,1);
}
""", GL.GL_FRAGMENT_SHADER)
vertexShader_textureless = self.shaders.compileShader(
"""
#version 460
uniform mat4 V;
uniform mat4 P;
uniform mat4 pose_rot;
uniform mat4 pose_trans;
uniform vec3 instance_color;
layout (location=0) in vec3 position;
layout (location=1) in vec3 normal;
layout (location=2) in vec3 color;
out vec3 theColor;
out vec3 Normal;
out vec3 FragPos;
out vec3 Normal_cam;
out vec3 Instance_color;
out vec3 Pos_cam;
out vec3 Pos_obj;
void main() {
gl_Position = P * V * pose_trans * pose_rot * vec4(position, 1);
vec4 world_position4 = pose_trans * pose_rot * vec4(position, 1);
FragPos = vec3(world_position4.xyz / world_position4.w); // in world coordinate
Normal = normalize(mat3(pose_rot) * normal); // in world coordinate
Normal_cam = normalize(mat3(V) * mat3(pose_rot) * normal); // in camera coordinate
vec4 pos_cam4 = V * pose_trans * pose_rot * vec4(position, 1);
Pos_cam = pos_cam4.xyz / pos_cam4.w;
Pos_obj = position;
theColor = color;
Instance_color = instance_color;
}
""", GL.GL_VERTEX_SHADER)
fragmentShader_textureless = self.shaders.compileShader(
"""
#version 460
in vec3 theColor;
in vec3 Normal;
in vec3 Normal_cam;
in vec3 FragPos;
in vec3 Instance_color;
in vec3 Pos_cam;
in vec3 Pos_obj;
layout (location = 0) out vec4 outputColour;
layout (location = 1) out vec4 NormalColour;
layout (location = 2) out vec4 InstanceColour;
layout (location = 3) out vec4 PCObject;
layout (location = 4) out vec4 PCColour;
uniform vec3 light_position; // in world coordinate
uniform vec3 light_color; // light color
void main() {
float ambientStrength = 0.4;
float diffuseStrength = 0.6;
vec3 ambient_color = vec3(1,1,1);
vec3 ambient = ambientStrength * ambient_color;
vec3 lightDir = normalize(light_position - FragPos);
float diff = max(dot(Normal, lightDir), 0.0);
vec3 diffuse = (diff * light_color) * diffuseStrength;
outputColour = vec4(theColor, 1) * vec4(diffuse + ambient, 1);
NormalColour = vec4((Normal_cam + 1) / 2,1);
InstanceColour = vec4(Instance_color,1);
PCObject = vec4(Pos_obj,1);
PCColour = vec4(Pos_cam,1);
}
""", GL.GL_FRAGMENT_SHADER)
vertexShader_simple = self.shaders.compileShader(
"""
#version 460
uniform mat4 V;
uniform mat4 P;
layout (location=0) in vec3 position;
layout (location=1) in vec3 normal;
layout (location=2) in vec2 texCoords;
void main() {
gl_Position = P * V * vec4(position,1);
}
""", GL.GL_VERTEX_SHADER)
fragmentShader_simple = self.shaders.compileShader(
"""
#version 460
layout (location = 0) out vec4 outputColour;
layout (location = 1) out vec4 NormalColour;
layout (location = 2) out vec4 InstanceColour;
layout (location = 3) out vec4 PCColour;
void main() {
outputColour = vec4(0.1, 0.1, 0.1, 1.0);
NormalColour = vec4(0,0,0,0);
InstanceColour = vec4(0,0,0,0);
PCColour = vec4(0,0,0,0);
}
""", GL.GL_FRAGMENT_SHADER)
vertexShader_simple2 = self.shaders.compileShader(
"""
#version 450
uniform mat4 V;
uniform mat4 P;
layout (location=0) in vec3 position;
layout (location=1) in vec3 normal;
layout (location=2) in vec2 texCoords;
out vec3 Normal_cam;
void main() {
gl_Position = P * V * vec4(position,1);
Normal_cam = normal;
}
""", GL.GL_VERTEX_SHADER)
fragmentShader_simple2 = self.shaders.compileShader(
"""
#version 450
in vec3 Normal_cam;
layout (location = 0) out vec4 outputColour;
layout (location = 1) out vec4 NormalColour;
layout (location = 2) out vec4 InstanceColour;
layout (location = 3) out vec4 PCColour;
void main() {
outputColour = vec4(Normal_cam, 1.0);
NormalColour = vec4(0,0,0,0);
InstanceColour = vec4(0,0,0,0);
PCColour = vec4(0,0,0,0);
}
""", GL.GL_FRAGMENT_SHADER)
self.shaderProgram = self.shaders.compileProgram(
vertexShader, fragmentShader)
self.shaderProgram_textureless = self.shaders.compileProgram(
vertexShader_textureless, fragmentShader_textureless)
self.shaderProgram_simple = self.shaders.compileProgram(
vertexShader_simple, fragmentShader_simple)
self.texUnitUniform = GL.glGetUniformLocation(self.shaderProgram,
'texUnit')
self.shaderProgram_simple2 = self.shaders.compileProgram(
vertexShader_simple2, fragmentShader_simple2)
self.lightpos = [0, 0, 0]
self.fbo = GL.glGenFramebuffers(1)
self.color_tex = GL.glGenTextures(1)
self.color_tex_2 = GL.glGenTextures(1)
self.color_tex_3 = GL.glGenTextures(1)
self.color_tex_4 = GL.glGenTextures(1)
self.color_tex_5 = GL.glGenTextures(1)
self.depth_tex = GL.glGenTextures(1)
GL.glBindTexture(GL.GL_TEXTURE_2D, self.color_tex)
GL.glTexImage2D(GL.GL_TEXTURE_2D, 0, GL.GL_RGBA32F, self.width,
self.height, 0, GL.GL_RGBA, GL.GL_UNSIGNED_BYTE, None)
GL.glBindTexture(GL.GL_TEXTURE_2D, self.color_tex_2)
GL.glTexImage2D(GL.GL_TEXTURE_2D, 0, GL.GL_RGBA32F, self.width,
self.height, 0, GL.GL_RGBA, GL.GL_UNSIGNED_BYTE, None)
GL.glBindTexture(GL.GL_TEXTURE_2D, self.color_tex_3)
GL.glTexImage2D(GL.GL_TEXTURE_2D, 0, GL.GL_RGBA32F, self.width,
self.height, 0, GL.GL_RGBA, GL.GL_UNSIGNED_BYTE, None)
GL.glBindTexture(GL.GL_TEXTURE_2D, self.color_tex_4)
GL.glTexImage2D(GL.GL_TEXTURE_2D, 0, GL.GL_RGBA32F, self.width,
self.height, 0, GL.GL_RGBA, GL.GL_FLOAT, None)
GL.glBindTexture(GL.GL_TEXTURE_2D, self.color_tex_5)
GL.glTexImage2D(GL.GL_TEXTURE_2D, 0, GL.GL_RGBA32F, self.width,
self.height, 0, GL.GL_RGBA, GL.GL_FLOAT, None)
GL.glBindTexture(GL.GL_TEXTURE_2D, self.depth_tex)
GL.glTexImage2D.wrappedOperation(GL.GL_TEXTURE_2D, 0,
GL.GL_DEPTH24_STENCIL8, self.width,
self.height, 0, GL.GL_DEPTH_STENCIL,
GL.GL_UNSIGNED_INT_24_8, None)
GL.glBindFramebuffer(GL.GL_FRAMEBUFFER, self.fbo)
GL.glFramebufferTexture2D(GL.GL_FRAMEBUFFER, GL.GL_COLOR_ATTACHMENT0,
GL.GL_TEXTURE_2D, self.color_tex, 0)
GL.glFramebufferTexture2D(GL.GL_FRAMEBUFFER, GL.GL_COLOR_ATTACHMENT1,
GL.GL_TEXTURE_2D, self.color_tex_2, 0)
GL.glFramebufferTexture2D(GL.GL_FRAMEBUFFER, GL.GL_COLOR_ATTACHMENT2,
GL.GL_TEXTURE_2D, self.color_tex_3, 0)
GL.glFramebufferTexture2D(GL.GL_FRAMEBUFFER, GL.GL_COLOR_ATTACHMENT3,
GL.GL_TEXTURE_2D, self.color_tex_4, 0)
GL.glFramebufferTexture2D(GL.GL_FRAMEBUFFER, GL.GL_COLOR_ATTACHMENT4,
GL.GL_TEXTURE_2D, self.color_tex_5, 0)
GL.glFramebufferTexture2D(GL.GL_FRAMEBUFFER,
GL.GL_DEPTH_STENCIL_ATTACHMENT,
GL.GL_TEXTURE_2D, self.depth_tex, 0)
GL.glViewport(0, 0, self.width, self.height)
GL.glDrawBuffers(5, [
GL.GL_COLOR_ATTACHMENT0, GL.GL_COLOR_ATTACHMENT1,
GL.GL_COLOR_ATTACHMENT2, GL.GL_COLOR_ATTACHMENT3,
GL.GL_COLOR_ATTACHMENT4
])
assert GL.glCheckFramebufferStatus(
GL.GL_FRAMEBUFFER) == GL.GL_FRAMEBUFFER_COMPLETE
self.fov = 20
self.camera = [1, 0, 0]
self.target = [0, 0, 0]
self.up = [0, 0, 1]
P = perspective(self.fov,
float(self.width) / float(self.height), 0.01, 100)
V = lookat(self.camera, self.target, up=self.up)
self.V = np.ascontiguousarray(V, np.float32)
self.P = np.ascontiguousarray(P, np.float32)
self.grid = self.generate_grid()
if render_rotation:
initial_distribution = np.ones((37 * 72, ), dtype=np.float32)
# initial_distribution /= np.sum(initial_distribution)
self.views_vis = np.load('./code/views_vis.npy')
print(self.views_vis)
self.rotation_visualization = self.generate_rotation_visualization(
initial_distribution)
def __init__(self, width=512, height=512, gpu_id=0, render_marker=False, robot=''):
self.render_marker = render_marker
self.VAOs = []
self.VBOs = []
self.materials = []
self.textures = []
self.is_textured = []
self.is_materialed = []
self.objects = []
self.texUnitUniform = None
self.width = width
self.height = height
self.vertices = []
self.faces = []
self.poses_trans = []
self.poses_rot = []
self.instances = []
self.extents = []
self.robot = robot
if len(self.robot) > 3:
self._offset_map = self.load_offset()
else:
self._offset_map = None
self.r = CppYCBRenderer.CppYCBRenderer(width, height, get_available_devices()[gpu_id])
self.r.init()
self.glstring = GL.glGetString(GL.GL_VERSION)
from OpenGL.GL import shaders
self.shaders = shaders
self.colors = [[0.9, 0, 0], [0.6, 0, 0], [0.3, 0, 0], [0.3, 0, 0], [0.3, 0, 0], [0.3, 0, 0], [0.3, 0, 0]]
self.lightcolor = [1, 1, 1]
cur_dir = os.path.dirname(os.path.abspath(__file__))
vertexShader = self.shaders.compileShader(
open(os.path.join(cur_dir, 'shaders/vert.shader')).readlines(), GL.GL_VERTEX_SHADER)
fragmentShader = self.shaders.compileShader(
open(os.path.join(cur_dir,'shaders/frag.shader')).readlines(), GL.GL_FRAGMENT_SHADER)
vertexShader_textureMat = self.shaders.compileShader(
open(os.path.join(cur_dir,'shaders/vert_blinnphong.shader')).readlines(), GL.GL_VERTEX_SHADER)
fragmentShader_textureMat = self.shaders.compileShader(
open(os.path.join(cur_dir,'shaders/frag_blinnphong.shader')).readlines(), GL.GL_FRAGMENT_SHADER)
vertexShader_textureless = self.shaders.compileShader(
open(os.path.join(cur_dir,'shaders/vert_textureless.shader')).readlines(), GL.GL_VERTEX_SHADER)
fragmentShader_textureless = self.shaders.compileShader(
open(os.path.join(cur_dir,'shaders/frag_textureless.shader')).readlines(), GL.GL_FRAGMENT_SHADER)
#try with the easiest shader first, and then look at Gl apply material
vertexShader_material = self.shaders.compileShader(
open(os.path.join(cur_dir,'shaders/vert_mat.shader')).readlines(), GL.GL_VERTEX_SHADER)
fragmentShader_material = self.shaders.compileShader(
open(os.path.join(cur_dir,'shaders/frag_mat.shader')).readlines(), GL.GL_FRAGMENT_SHADER)
vertexShader_simple = self.shaders.compileShader(
open(os.path.join(cur_dir,'shaders/vert_simple.shader')).readlines(), GL.GL_VERTEX_SHADER)
fragmentShader_simple = self.shaders.compileShader(
open(os.path.join(cur_dir,'shaders/frag_simple.shader')).readlines(), GL.GL_FRAGMENT_SHADER)
self.shaderProgram = self.shaders.compileProgram(vertexShader, fragmentShader)
self.shaderProgram_textureless = self.shaders.compileProgram(vertexShader_textureless, fragmentShader_textureless)
self.shaderProgram_simple = self.shaders.compileProgram(vertexShader_simple, fragmentShader_simple)
self.shaderProgram_material = self.shaders.compileProgram(vertexShader_material, fragmentShader_material)
self.shaderProgram_textureMat = self.shaders.compileProgram(vertexShader_textureMat, fragmentShader_textureMat)
self.texUnitUniform_textureMat = GL.glGetUniformLocation(self.shaderProgram_textureMat, 'texUnit')
self.lightpos = [0, 0, 0]
self.fbo = GL.glGenFramebuffers(1)
self.color_tex = GL.glGenTextures(1)
self.color_tex_2 = GL.glGenTextures(1)
self.color_tex_3 = GL.glGenTextures(1)
self.color_tex_4 = GL.glGenTextures(1)
self.color_tex_5 = GL.glGenTextures(1)
self.depth_tex = GL.glGenTextures(1)
GL.glBindTexture(GL.GL_TEXTURE_2D, self.color_tex)
GL.glTexImage2D(GL.GL_TEXTURE_2D, 0, GL.GL_RGBA32F, self.width, self.height, 0,
GL.GL_RGBA, GL.GL_UNSIGNED_BYTE, None)
GL.glBindTexture(GL.GL_TEXTURE_2D, self.color_tex_2)
GL.glTexImage2D(GL.GL_TEXTURE_2D, 0, GL.GL_RGBA32F, self.width, self.height, 0,
GL.GL_RGBA, GL.GL_UNSIGNED_BYTE, None)
GL.glBindTexture(GL.GL_TEXTURE_2D, self.color_tex_3)
GL.glTexImage2D(GL.GL_TEXTURE_2D, 0, GL.GL_RGBA32F, self.width, self.height, 0,
GL.GL_RGBA, GL.GL_UNSIGNED_BYTE, None)
GL.glBindTexture(GL.GL_TEXTURE_2D, self.color_tex_4)
GL.glTexImage2D(GL.GL_TEXTURE_2D, 0, GL.GL_RGBA32F, self.width, self.height, 0,
GL.GL_RGBA, GL.GL_FLOAT, None)
GL.glBindTexture(GL.GL_TEXTURE_2D, self.color_tex_5)
GL.glTexImage2D(GL.GL_TEXTURE_2D, 0, GL.GL_RGBA32F, self.width, self.height, 0,
GL.GL_RGBA, GL.GL_FLOAT, None)
GL.glBindTexture(GL.GL_TEXTURE_2D, self.depth_tex)
GL.glTexImage2D.wrappedOperation(
GL.GL_TEXTURE_2D, 0, GL.GL_DEPTH24_STENCIL8, self.width, self.height, 0,
GL.GL_DEPTH_STENCIL, GL.GL_UNSIGNED_INT_24_8, None)
GL.glBindFramebuffer(GL.GL_FRAMEBUFFER, self.fbo)
GL.glFramebufferTexture2D(GL.GL_FRAMEBUFFER, GL.GL_COLOR_ATTACHMENT0, GL.GL_TEXTURE_2D, self.color_tex, 0)
GL.glFramebufferTexture2D(GL.GL_FRAMEBUFFER, GL.GL_COLOR_ATTACHMENT1, GL.GL_TEXTURE_2D, self.color_tex_2, 0)
GL.glFramebufferTexture2D(GL.GL_FRAMEBUFFER, GL.GL_COLOR_ATTACHMENT2, GL.GL_TEXTURE_2D, self.color_tex_3, 0)
GL.glFramebufferTexture2D(GL.GL_FRAMEBUFFER, GL.GL_COLOR_ATTACHMENT3, GL.GL_TEXTURE_2D, self.color_tex_4, 0)
GL.glFramebufferTexture2D(GL.GL_FRAMEBUFFER, GL.GL_COLOR_ATTACHMENT4, GL.GL_TEXTURE_2D, self.color_tex_5, 0)
GL.glFramebufferTexture2D(GL.GL_FRAMEBUFFER, GL.GL_DEPTH_STENCIL_ATTACHMENT, GL.GL_TEXTURE_2D, self.depth_tex, 0)
GL.glViewport(0, 0, self.width, self.height)
GL.glDrawBuffers(5, [GL.GL_COLOR_ATTACHMENT0, GL.GL_COLOR_ATTACHMENT1,
GL.GL_COLOR_ATTACHMENT2, GL.GL_COLOR_ATTACHMENT3, GL.GL_COLOR_ATTACHMENT4])
assert GL.glCheckFramebufferStatus(
GL.GL_FRAMEBUFFER) == GL.GL_FRAMEBUFFER_COMPLETE
self.fov = 20
self.camera = [1, 0, 0]
self.target = [0, 0, 0]
self.up = [0, 0, 1]
P = perspective(self.fov, float(self.width) /
float(self.height), 0.01, 100)
V = lookat(
self.camera,
self.target, up=self.up)
self.V = np.ascontiguousarray(V, np.float32)
self.P = np.ascontiguousarray(P, np.float32)
self.grid = self.generate_grid()
#added mouse interaction
self.is_rotating = False
state = torch.cuda.FloatTensor(width, height, 4)
state = state.byte().contiguous()
tex, cuda_buffer, sz = loadTexture('misc/husky.jpg')
nbytes = state.numel() * state.element_size()
for _ in bar(range(20000)):
with cuda_activate(cuda_buffer) as ary:
cpy = pycuda.driver.Memcpy2D()
cpy.set_dst_device(state.data_ptr())
cpy.set_src_array(ary)
cpy.width_in_bytes = cpy.src_pitch = cpy.dst_pitch = nbytes // height
cpy.height = height
cpy(aligned=False)
torch.cuda.synchronize()
img_np = state.data.cpu().numpy().reshape(height, width, 4)
print(img_np.shape)
img_np[:, :, 3] = 255
plt.imshow(img_np)
plt.show()
gl_dev = get_available_devices()[0]
print('opengl using GPU {}'.format(gl_dev))
r = CppYCBRenderer.CppYCBRenderer(512, 512, gl_dev)
r.init()
main()
r.release()