def main():
webcam = Webcam()
webcam.start()
mnp = MarkNplay()
while True:
image = webcam.get_current_frame()
mnp.screen_thread(image)
cv2.imshow('AR Book Reader', image)
cv2.waitKey(10)
def take_imgs(chessboard_size=(11,7), kSaveImageDeltaTime=1):
sys.path.append("../")
os.makedirs("./calib_images", exist_ok=True)
camera_num = 0
if len(sys.argv) == 2:
camera_num = int(sys.argv[1])
print('opening camera: ', camera_num)
webcam = Webcam(camera_num)
webcam.start()
lastSaveTime = time.time()
while True:
# get image from webcam
image = webcam.get_current_frame()
if image is not None:
# check if pattern found
ret, corners = cv2.findChessboardCorners(cv2.cvtColor(image,cv2.COLOR_BGR2GRAY), chessboard_size, None)
if ret == True:
print('found chessboard')
# save image
filename = datetime.now().strftime('%Y%m%d_%Hh%Mm%Ss%f') + '.bmp'
image_path="./calib_images/" + filename
elapsedTimeSinceLastSave = time.time() - lastSaveTime
do_save = elapsedTimeSinceLastSave > kSaveImageDeltaTime
print(elapsedTimeSinceLastSave, kSaveImageDeltaTime)
if do_save:
lastSaveTime = time.time()
print('saving file ', image_path)
cv2.imwrite(image_path, image)
# draw the corners
image = cv2.drawChessboardCorners(image, chessboard_size, corners, ret)
cv2.imshow('camera', image)
else:
pass
#print('empty image')
if cv2.waitKey(1) & 0xFF == ord('q'):
break
class SaltwashAR:
# constants
INVERSE_MATRIX = np.array([[ 1.0, 1.0, 1.0, 1.0],
[-1.0,-1.0,-1.0,-1.0],
[-1.0,-1.0,-1.0,-1.0],
[ 1.0, 1.0, 1.0, 1.0]])
def __init__(self):
# initialise config
self.config_provider = ConfigProvider()
# initialise robots
self.rocky_robot = RockyRobot()
self.sporty_robot = SportyRobot()
# initialise webcam
self.webcam = Webcam()
# initialise glyphs
self.glyphs = Glyphs()
self.glyphs_cache = None
# initialise browser
self.browser = None
if self.config_provider.browser:
self.browser = Browser()
# initialise texture
self.texture_background = None
def _init_gl(self):
glClearColor(0.0, 0.0, 0.0, 0.0)
glClearDepth(1.0)
glDepthFunc(GL_LESS)
glEnable(GL_DEPTH_TEST)
glShadeModel(GL_SMOOTH)
glMatrixMode(GL_PROJECTION)
glLoadIdentity()
gluPerspective(33.7, 1.3, 0.1, 100.0)
glMatrixMode(GL_MODELVIEW)
# load robots frames
self.rocky_robot.load_frames(self.config_provider.animation)
self.sporty_robot.load_frames(self.config_provider.animation)
# start threads
self.webcam.start()
if self.browser:
self.browser.start()
# assign texture
glEnable(GL_TEXTURE_2D)
self.texture_background = glGenTextures(1)
def _draw_scene(self):
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
glLoadIdentity()
# reset robots
self.rocky_robot.is_detected = False
self.sporty_robot.is_detected = False
# get image from webcam
image = self.webcam.get_current_frame()
# handle background
self._handle_background(image)
# handle glyphs
self._handle_glyphs(image)
# handle browser
self._handle_browser()
glutSwapBuffers()
def _handle_background(self, image):
# convert image to OpenGL texture format
bg_image = cv2.flip(image, 0)
bg_image = Image.fromarray(bg_image)
ix = bg_image.size[0]
iy = bg_image.size[1]
bg_image = bg_image.tostring("raw", "BGRX", 0, -1)
# create background texture
glBindTexture(GL_TEXTURE_2D, self.texture_background)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST)
glTexImage2D(GL_TEXTURE_2D, 0, 3, ix, iy, 0, GL_RGBA, GL_UNSIGNED_BYTE, bg_image)
# draw background
glBindTexture(GL_TEXTURE_2D, self.texture_background)
glPushMatrix()
glTranslatef(0.0,0.0,-10.0)
glBegin(GL_QUADS)
glTexCoord2f(0.0, 1.0); glVertex3f(-4.0, -3.0, 0.0)
glTexCoord2f(1.0, 1.0); glVertex3f( 4.0, -3.0, 0.0)
glTexCoord2f(1.0, 0.0); glVertex3f( 4.0, 3.0, 0.0)
glTexCoord2f(0.0, 0.0); glVertex3f(-4.0, 3.0, 0.0)
glEnd( )
glPopMatrix()
def _handle_glyphs(self, image):
# attempt to detect glyphs
glyphs = []
try:
glyphs = self.glyphs.detect(image)
except Exception as ex:
print(ex)
# manage glyphs cache
if glyphs:
self.glyphs_cache = glyphs
elif self.glyphs_cache:
glyphs = self.glyphs_cache
self.glyphs_cache = None
else:
return
for glyph in glyphs:
rvecs, tvecs, _, glyph_name = glyph
# build view matrix
rmtx = cv2.Rodrigues(rvecs)[0]
view_matrix = np.array([[rmtx[0][0],rmtx[0][1],rmtx[0][2],tvecs[0]],
[rmtx[1][0],rmtx[1][1],rmtx[1][2],tvecs[1]],
[rmtx[2][0],rmtx[2][1],rmtx[2][2],tvecs[2]],
[0.0 ,0.0 ,0.0 ,1.0 ]])
view_matrix = view_matrix * self.INVERSE_MATRIX
view_matrix = np.transpose(view_matrix)
# load view matrix and draw cube
glPushMatrix()
glLoadMatrixd(view_matrix)
if glyph_name == ROCKY_ROBOT:
self.rocky_robot.is_detected = True
if self.browser and self.browser.is_speaking:
self.rocky_robot.next_frame(True)
else:
self.rocky_robot.next_frame(False)
elif glyph_name == SPORTY_ROBOT:
self.sporty_robot.is_detected = True
if self.browser and self.browser.is_speaking:
self.sporty_robot.next_frame(True)
else:
self.sporty_robot.next_frame(False)
glColor3f(1.0, 1.0, 1.0)
glPopMatrix()
def _handle_browser(self):
# check browser instantiated
if not self.browser: return
# handle browser
if self.rocky_robot.is_detected:
self.browser.load(ROCK)
elif self.sporty_robot.is_detected:
self.browser.load(SPORT)
else:
self.browser.halt()
def main(self):
# setup and run OpenGL
glutInit()
glutInitDisplayMode(GLUT_RGBA | GLUT_DOUBLE | GLUT_DEPTH)
glutInitWindowSize(640, 480)
glutInitWindowPosition(800, 400)
self.window_id = glutCreateWindow("SaltwashAR")
glutDisplayFunc(self._draw_scene)
glutIdleFunc(self._draw_scene)
self._init_gl()
glutMainLoop()
class CPPN:
def __init__(self,
height,
width,
n_inputs,
n_hidden,
n_outputs,
non_linearity,
device,
webcam=False):
self.height = height
self.width = width
self.n_inputs = n_inputs
self.n_hidden = n_hidden
self.n_outputs = n_outputs
self.non_linearity = non_linearity
self.device = device
self.network = Network(n_inputs, n_hidden, n_outputs, non_linearity)
self.network.to(device)
self.network_new = Network(n_inputs, n_hidden, n_outputs,
non_linearity)
self.network_new.to(device)
self.sound = Sound()
self.visualisation_input = self._create_visualisation_tensor()
if webcam:
self.webcam = Webcam()
self.webcam.start()
def _create_visualisation_tensor(self):
visualisation_input = np.zeros(
(self.height, self.width, self.n_inputs))
for i in range(self.height):
for j in range(self.width):
visualisation_input[i, j] = [
i / float(self.height), j / float(self.width)
] + [0] * (self.n_inputs - 2)
visualisation_input = torch.tensor(visualisation_input.reshape(
-1, self.n_inputs),
dtype=torch.float,
device=self.device)
return visualisation_input
# def _create_kaleidoscope_visualisation_tensor(self):
# visualisation_input = np.zeros((self.height, self.width, self.n_inputs))
#
# for i in range(self.height):
# for j in range(self.width):
# visualisation_input[i, j] = [i/float(self.height),j/float(self.width)] + [0]*(self.n_inputs-2)
#
# visualisation_input = torch.tensor(visualisation_input.reshape(-1, self.n_inputs), dtype=torch.float, device=self.device)
#
# return visualisation_input
def _visualise_np(self, bands=None):
with torch.no_grad():
if bands is not None:
o = torch.tensor(bands, dtype=torch.float).repeat(
self.visualisation_input.size(0), 1)
self.visualisation_input[:, 2:] = o
im = self.network(self.visualisation_input).reshape(
self.width, self.height, self.n_outputs)
return im.detach().cpu().numpy()
def _visualise(self, name, save_im=False):
with torch.no_grad():
im = self.network(self.visualisation_input).reshape(
self.width, self.height, 3).detach().cpu().numpy() * 255
if save_im:
cv2.imwrite(name + ".png", im)
return im
def start(self):
cv2.namedWindow("CPPN", cv2.WINDOW_NORMAL)
cv2.setWindowProperty("CPPN", cv2.WND_PROP_FULLSCREEN,
cv2.WINDOW_FULLSCREEN)
audio_generator = self.sound.read_audio()
t = time.time()
interpolating = 0
training = 0
generator = None
while True:
if time.time() - t > 20. and not training and not interpolating:
choice = np.random.choice([1])
if choice == 0:
training = 1
target_images = glob.glob("target_images/*")
choice = np.random.choice(target_images)
image = cv2.imread(choice)
image = ToTensor()(image).float().to(self.device)
generator = self.train(image,
self.height,
self.width,
0.001,
save_im=False,
image_folder="formation1",
network_name="Santa",
epochs=100,
live=True,
verbose=True)
elif choice == 1:
self._random_network()
generator = self.interpolate(60)
interpolating = 1
elif choice == 2:
print("webcam")
training = 1
image = self.webcam.read()
image = ToTensor()(image).float().to(self.device)
generator = self.train(image,
self.height,
self.width,
0.001,
save_im=False,
image_folder="formation1",
network_name="Santa",
epochs=100,
live=True,
verbose=True)
if choice == 0:
self.sound.sin = self.sound.sin == False
self.sound.amplitudes = np.abs(np.random.rand(8))
self.sound.frequencies = np.abs(np.random.rand(8))
elif choice == 1:
self.sound.calculate_bands = self.sound.calculate_bands == False
if self.sound.calculate_bands == False and self.sound.sin == False:
if np.random.rand() < 0.5:
self.sound.calculate_bands = True
else:
self.sound.sin = True
t = time.time()
if interpolating:
try:
self.network = next(generator)
except StopIteration:
interpolating = 0
if training:
try:
self.network = next(generator)
except StopIteration:
training = 0
torch.cuda.empty_cache()
frame = self._visualise_np(next(audio_generator))
cv2.imshow(
"CPPN",
cv2.copyMakeBorder(frame, 0, 0, 0, 0, cv2.BORDER_CONSTANT, 0))
if cv2.waitKey(1) & 0xFF == ord('q'):
break
cv2.destroyAllWindows()
self.sound.stream.stop_stream()
self.sound.stream.close()
self.sound.pa.terminate()
def _resize_image(self, image, width, height):
image = F.interpolate(image.unsqueeze(0), size=(width, height))
image = torch.squeeze(image, 0)
image = image.permute(1, 2, 0)
return image
def _random_network(self):
self.network_new = Network(self.n_inputs, self.n_hidden,
self.n_outputs, self.non_linearity)
self.network_new.to(self.device)
def train(self,
image,
width,
height,
loss_threshold,
save_im=False,
image_folder="",
save_network=True,
network_name="",
epochs=1,
live=False,
verbose=False):
image = self._resize_image(image, width, height)
optimiser = torch.optim.Adam(self.network.parameters())
criterion = torch.nn.MSELoss()
previous_loss = None
network = copy.copy(self.network)
for i in range(epochs):
loss = criterion(
network(self.visualisation_input).reshape(width, height, 3),
image)
optimiser.zero_grad()
loss.backward()
optimiser.step()
if verbose:
print(i, loss)
if save_im:
if previous_loss is None or previous_loss - loss_threshold > loss:
previous_loss = loss
self._visualise(
"{}/image{:06d}.png".format(image_folder, i), save_im)
if live:
yield network
if save_network:
torch.save(self.network.state_dict(),
'trained_networks/network{}.pt'.format(network_name))
torch.cuda.empty_cache()
def load(self, network_name, new=False):
if new:
self.network_new.load_state_dict(
torch.load("trained_networks/network" + network_name + ".pt"))
self.network_new.to(self.device)
else:
self.network.load_state_dict(
torch.load("trained_networks/network" + network_name + ".pt"))
self.network.to(self.device)
def interpolate(self, num_interpolation_frames, space="beta", beta=2.0):
"""
beta parameter alters the steepness of the function
"""
weights1 = []
biases1 = []
weights2 = []
biases2 = []
for idx, layer in enumerate(self.network.module_list):
weights1.append(layer.weight.data)
biases1.append(layer.bias.data)
weights2.append(self.network_new.module_list[idx].weight.data)
biases2.append(self.network_new.module_list[idx].bias.data)
frame_distribution = frame_distribution = np.linspace(
0, 1, num=num_interpolation_frames)
if space != "lin":
f = lambda x: 1 / (1 + np.power(x / (1 + np.finfo(float).eps - x),
-beta))
frame_distribution = f(frame_distribution)
weights = []
for weight_pair in zip(weights1, weights2):
difference = weight_pair[0] - weight_pair[1]
new_weights = torch.zeros(
num_interpolation_frames,
list(weight_pair[0].size())[0],
list(weight_pair[0].size())[1]).float().to(self.device)
for idx, i in enumerate(frame_distribution):
new_weights[idx] = weight_pair[0] - i * difference
weights.append(new_weights)
biases = []
for bias_pair in zip(biases1, biases2):
difference = bias_pair[0] - bias_pair[1]
new_biases = torch.zeros(num_interpolation_frames,
list(bias_pair[0].size())[0]).float().to(
self.device)
for idx, i in enumerate(frame_distribution):
new_biases[idx] = bias_pair[0] - i * difference
biases.append(new_biases)
network = copy.deepcopy(self.network)
for i in range(num_interpolation_frames):
for idx, layer in enumerate(network.module_list):
network.module_list[idx].weight.data = weights[idx][i]
network.module_list[idx].bias.data = biases[idx][i]
yield network
def random_walk(self):
pass
def animate(self):
pass
def alzheimer(self):
pass
from skimage.measure import compare_ssim as ssim
def mse(imageA, imageB):
# the 'Mean Squared Error' between the two images is the
# sum of the squared difference between the two images;
# NOTE: the two images must have the same dimension
err = np.sum((imageA.astype("float") - imageB.astype("float"))**2)
err /= float(imageA.shape[0] * imageA.shape[1])
return err
if __name__ == "__main__":
camera = Webcam()
if camera.isConnected():
try:
print("copying output")
camera.start()
while (True):
image1 = camera.getFrame()
image2 = camera.getFrame()
cor = ssim(image1, image2)
if cor < 0.78:
print("gettingframe: ", cor)
#camera.show(image1)
except (KeyboardInterrupt, SystemExit):
print("exiting")
camera.stop()
else:
print("problems")
class OpenGLGlyphs:
##############################################################초기화
def __init__(self):
# initialise webcam and start thread
self.webcam = Webcam()
self.webcam.start()
self.find = fp()
self.find.set_img('sample.jpg')
self.hei, self.wid = self.webcam.get_frame_shape()[:2]
# initialise cube
# self.d_obj = None
self.img = None
# initialise texture
self.texture_background = None
self.K = None
self.mark_kp = None
self.mark_des = None
self.set_keypoint()
self.new_kp = None
self.mat_kp = None
self.mat_des = None
self.H = None
# self.Rt=None
##############################################################카메라 세팅
def _init_gl(self, Width, Height):
glClearColor(0.0, 0.0, 0.0, 0.0) # 투명도 결정
glClearDepth(1.0) # 깊이 버퍼의 모든 픽셀에 설정될 초기값 지정
glDepthFunc(GL_LESS) # 언제나 새로 들어오는 값이 기준 GL_LESS를 설정했다고 하자. 이 경우에는 새로 들어온 값이
# 이미 저장되어 있는 값 보다 적을 경우에 depth buffer의 값을 새로 들어온 값으로 갱신하겠다,
glEnable(GL_DEPTH_TEST) # 요건 깊이 정보에 따라 이미지를 순서대로 나줌.
glShadeModel(GL_SMOOTH)
glMatrixMode(GL_PROJECTION)
glLoadIdentity()
self.K = my_calibration((Height, Width))
fx = self.K[0, 0]
fy = self.K[1, 1]
fovy = 2 * arctan(0.5 * Height / fy) * 180 / pi
aspect = (float)(Width * fy) / (Height * fx)
# define the near and far clipping planes
near = 0.1
far = 100.0
# set perspective
gluPerspective(fovy, aspect, near, far)
glMatrixMode(GL_MODELVIEW)
# self.d_obj=[OBJ('Rocket.obj')]
glEnable(GL_TEXTURE_2D)
self.texture_background = glGenTextures(1)
# gluPerspective(33.7, 1.3, 0.1, 100.0)
##############################################################marker의 kp, des저장
def set_keypoint(self):
self.find.start()
self.mark_kp, self.mark_des = self.find.get_point()
##############################################################K값 구하기
def _draw_scene(self):
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
glLoadIdentity()
# get image from webcam
image = self.webcam.get_current_frame()
Rt = self._my_cal(image)
"""
if Rt!=None:
box=ones((self.hei,self.wid),uint8)
H_box=cv2.warpPerspective(box,self.H,(self.wid, self.hei))
image=image*H_box[:,:,newaxis]
image=cv2.drawKeypoints(image,self.mat_kp,flags=cv2.DRAW_MATCHES_FLAGS_DEFAULT)
"""
# convert image to OpenGL texture format
bg_image = cv2.flip(image, 0)
bg_image = Image.fromarray(bg_image)
ix = bg_image.size[0]
iy = bg_image.size[1]
bg_image = bg_image.tobytes("raw", "BGRX", 0, -1)
# create background texture
glBindTexture(GL_TEXTURE_2D, self.texture_background)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST)
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, self.wid, self.hei, 0, GL_RGBA, GL_UNSIGNED_BYTE, bg_image)
# draw background
glBindTexture(GL_TEXTURE_2D, self.texture_background)
glPushMatrix()
# glTranslatef(0.0,0.0,0.0)
gluLookAt(0.0, 0.0, 12.5, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0)
self._draw_background()
glPopMatrix()
################Rt를 구해서 매칭되는 이미지가 있는지 판단
if Rt is not None:
self._set_modelview_from_camera(Rt)
glEnable(GL_LIGHTING)
glEnable(GL_LIGHT0)
glEnable(GL_DEPTH_TEST)
glEnable(GL_NORMALIZE)
glClear(GL_DEPTH_BUFFER_BIT)
glMaterialfv(GL_FRONT, GL_AMBIENT, [0.5, 0.5, 0.0, 1.0])
glMaterialfv(GL_FRONT, GL_DIFFUSE, [0.9, 0.9, 0.0, 1.0])
glMaterialfv(GL_FRONT, GL_SPECULAR, [1.0, 1.0, 1.0, 1.0])
glMaterialfv(GL_FRONT, GL_SHININESS, 0.25 * 128.0)
glutSolidTeapot(0.1)
glutSwapBuffers()
##############################################################OpenGL용 Rt변환
def _set_modelview_from_camera(self, Rt):
glMatrixMode(GL_MODELVIEW)
glLoadIdentity()
Rx = array([[1, 0, 0], [0, 0, 1], [0, 1, 0]])
# set rotation to best approximation
R = Rt[:, :3]
# change sign of x-axis
R[0, :] = -R[0, :]
# set translation
t = Rt[:, 3]
t[0] = -t[0]
# setup 4*4 model view matrix
M = eye(4)
M[:3, :3] = dot(R, Rx)
M[:3, 3] = t
M[3, :3] = t
# transpose and flatten to get column order
M = M.T
m = M.flatten()
# replace model view with the new matrix
glLoadMatrixf(m)
##############################################################Rt반환
def _my_cal(self, image):
find_H = fp()
find_H.set_cv_img(image)
find_H.start()
kp, des = find_H.get_point()
self.H = self.match_images(self.mark_kp, self.mark_des, kp, des)
if self.H is not None:
cam1 = camera.Camera(hstack((self.K, dot(self.K, array([[0], [0], [-1]])))))
# Rt1=dot(linalg.inv(self.K),cam1.P)
cam2 = camera.Camera(dot(self.H, cam1.P))
A = dot(linalg.inv(self.K), cam2.P[:, :3])
A = array([A[:, 0], A[:, 1], cross(A[:, 0], A[:, 1])]).T
cam2.P[:, :3] = dot(self.K, A)
Rt = dot(linalg.inv(self.K), cam2.P)
return Rt
else:
return None
##############################################################match image
def match_images(self, kp1, des1, kp2, des2):
matcher = cv2.BFMatcher()
match_des = matcher.knnMatch(des1, des2, k=2)
matches = []
matA, matB = [], []
matC = []
for m in match_des:
if m[0].distance < 0.8 * m[1].distance:
matA.append(kp1[m[0].queryIdx])
matB.append(kp2[m[0].trainIdx])
matC.append(des1[m[0].queryIdx])
if len(matA) > 50:
ptsA = float32([m.pt for m in matA])
ptsB = float32([n.pt for n in matB])
H1 = []
H1, status = cv2.findHomography(ptsA, ptsB, cv2.RANSAC, 5.0)
H1 = self.homo_check(H1)
self.mat_kp = array([matB[i] for i in range(status.shape[0]) if status[i] == 1])
self.mat_des = array([matC[i] for i in range(status.shape[0]) if status[i] == 1])
return H1
else:
return None
##############################################################homography check
def homo_check(self, H1):
if self.H is None:
return H1
else:
if cv2.norm(H1, self.H) > 1.0:
return H1
else:
return self.H
def _draw_background(self):
# draw background
glBegin(GL_QUADS)
glTexCoord2f(0.0, 1.0);
glVertex3f(-4.0, -3.0, 0.0)
glTexCoord2f(1.0, 1.0);
glVertex3f(4.0, -3.0, 0.0)
glTexCoord2f(1.0, 0.0);
glVertex3f(4.0, 3.0, 0.0)
glTexCoord2f(0.0, 0.0);
glVertex3f(-4.0, 3.0, 0.0)
glEnd()
glDeleteTextures(1)
def keyboard(self, *args):
if args[0] is GLUT_KEY_UP:
glutDestroyWindow(self.window_id)
self.webcam.finish()
sys.exit()
##############################################################OpenGL창 초기
def main(self):
# setup and run OpenGL
glutInit()
glutInitDisplayMode(GLUT_RGBA | GLUT_DOUBLE | GLUT_DEPTH)
glutInitWindowSize(self.wid, self.hei)
glutInitWindowPosition(400, 400)
self.window_id = glutCreateWindow(b"OpenGL Glyphs")
self._init_gl(self.wid, self.hei)
glutDisplayFunc(self._draw_scene)
glutIdleFunc(self._draw_scene)
glutSpecialFunc(self.keyboard)
glutMainLoop()
from webcam import Webcam
import cv2
from datetime import datetime
webcam = Webcam()
webcam.start()
while True:
# get image from webcam
image = webcam.get_current_frame()
# display image
cv2.imshow('grid', image)
cv2.waitKey(3000)
# save image to file, if pattern found
ret, corners = cv2.findChessboardCorners(
cv2.cvtColor(image, cv2.COLOR_BGR2GRAY), (7, 6), None)
if ret == True:
filename = datetime.now().strftime('%Y%m%d_%Hh%Mm%Ss%f') + '.jpg'
cv2.imwrite("pose/sample_images/" + filename, image)
class VidMag():
def __init__(self):
self.webcam = Webcam()
self.buffer_size = 40
self.fps = 0
self.times = []
self.t0 = time.time()
self.data_buffer = []
#self.vidmag_frames = []
self.frame_out = np.zeros((10, 10, 3), np.uint8)
self.webcam.start()
print("init")
#--------------COLOR MAGNIFICATIONN---------------------#
def build_gaussian_pyramid(self, src, level=3):
s = src.copy()
pyramid = [s]
for i in range(level):
s = cv2.pyrDown(s)
pyramid.append(s)
return pyramid
def gaussian_video(self, video_tensor, levels=3):
for i in range(0, video_tensor.shape[0]):
frame = video_tensor[i]
pyr = self.build_gaussian_pyramid(frame, level=levels)
gaussian_frame = pyr[-1]
if i == 0:
vid_data = np.zeros(
(video_tensor.shape[0], gaussian_frame.shape[0],
gaussian_frame.shape[1], 3))
vid_data[i] = gaussian_frame
return vid_data
def temporal_ideal_filter(self, tensor, low, high, fps, axis=0):
fft = fftpack.fft(tensor, axis=axis)
frequencies = fftpack.fftfreq(tensor.shape[0], d=1.0 / fps)
bound_low = (np.abs(frequencies - low)).argmin()
bound_high = (np.abs(frequencies - high)).argmin()
fft[:bound_low] = 0
fft[bound_high:-bound_high] = 0
fft[-bound_low:] = 0
iff = fftpack.ifft(fft, axis=axis)
return np.abs(iff)
def amplify_video(self, gaussian_vid, amplification=70):
return gaussian_vid * amplification
def reconstract_video(self, amp_video, origin_video, levels=3):
final_video = np.zeros(origin_video.shape)
for i in range(0, amp_video.shape[0]):
img = amp_video[i]
for x in range(levels):
img = cv2.pyrUp(img)
img = img + origin_video[i]
final_video[i] = img
return final_video
def magnify_color(self,
data_buffer,
fps,
low=0.4,
high=2,
levels=3,
amplification=30):
gau_video = self.gaussian_video(data_buffer, levels=levels)
filtered_tensor = self.temporal_ideal_filter(gau_video, low, high, fps)
amplified_video = self.amplify_video(filtered_tensor,
amplification=amplification)
final_video = self.reconstract_video(amplified_video,
data_buffer,
levels=levels)
#print("c")
return final_video
#-------------------------------------------------------------#
#-------------------MOTION MAGNIFICATIONN---------------------#
#build laplacian pyramid for video
def laplacian_video(self, video_tensor, levels=3):
tensor_list = []
for i in range(0, video_tensor.shape[0]):
frame = video_tensor[i]
pyr = self.build_laplacian_pyramid(frame, levels=levels)
if i == 0:
for k in range(levels):
tensor_list.append(
np.zeros((video_tensor.shape[0], pyr[k].shape[0],
pyr[k].shape[1], 3)))
for n in range(levels):
tensor_list[n][i] = pyr[n]
return tensor_list
#Build Laplacian Pyramid
def build_laplacian_pyramid(self, src, levels=3):
gaussianPyramid = self.build_gaussian_pyramid(src, levels)
pyramid = []
for i in range(levels, 0, -1):
GE = cv2.pyrUp(gaussianPyramid[i])
L = cv2.subtract(gaussianPyramid[i - 1], GE)
pyramid.append(L)
return pyramid
#reconstract video from laplacian pyramid
def reconstract_from_tensorlist(self, filter_tensor_list, levels=3):
final = np.zeros(filter_tensor_list[-1].shape)
for i in range(filter_tensor_list[0].shape[0]):
up = filter_tensor_list[0][i]
for n in range(levels - 1):
up = cv2.pyrUp(up) + filter_tensor_list[n + 1][i]
final[i] = up
return final
#butterworth bandpass filter
def butter_bandpass_filter(self, data, lowcut, highcut, fs, order=5):
omega = 0.5 * fs
low = lowcut / omega
high = highcut / omega
b, a = signal.butter(order, [low, high], btype='band')
y = signal.lfilter(b, a, data, axis=0)
return y
def magnify_motion(self,
video_tensor,
fps,
low=0.4,
high=1.5,
levels=3,
amplification=30):
lap_video_list = self.laplacian_video(video_tensor, levels=levels)
filter_tensor_list = []
for i in range(levels):
filter_tensor = self.butter_bandpass_filter(
lap_video_list[i], low, high, fps)
filter_tensor *= amplification
filter_tensor_list.append(filter_tensor)
recon = self.reconstract_from_tensorlist(filter_tensor_list)
final = video_tensor + recon
return final
#-------------------------------------------------------------#
def buffer_to_tensor(self, buffer):
tensor = np.zeros((len(buffer), 192, 256, 3), dtype="float")
i = 0
for i in range(len(buffer)):
tensor[i] = buffer[i]
return tensor
def run_color(self):
self.times.append(time.time() - self.t0)
L = len(self.data_buffer)
#print(self.data_buffer)
if L > self.buffer_size:
self.data_buffer = self.data_buffer[-self.buffer_size:]
self.times = self.times[-self.buffer_size:]
#self.vidmag_frames = self.vidmag_frames[-self.buffer_size:]
L = self.buffer_size
if len(self.data_buffer) > self.buffer_size - 1:
self.fps = float(L) / (self.times[-1] - self.times[0])
tensor = self.buffer_to_tensor(self.data_buffer)
final_vid = self.magnify_color(data_buffer=tensor, fps=self.fps)
#print(final_vid[0].shape)
#self.vidmag_frames.append(final_vid[-1])
#print(self.fps)
self.frame_out = final_vid[-1]
def run_motion(self):
self.times.append(time.time() - self.t0)
L = len(self.data_buffer)
#print(L)
if L > self.buffer_size:
self.data_buffer = self.data_buffer[-self.buffer_size:]
self.times = self.times[-self.buffer_size:]
#self.vidmag_frames = self.vidmag_frames[-self.buffer_size:]
L = self.buffer_size
if len(self.data_buffer) > self.buffer_size - 1:
self.fps = float(L) / (self.times[-1] - self.times[0])
tensor = self.buffer_to_tensor(self.data_buffer)
final_vid = self.magnify_motion(video_tensor=tensor, fps=self.fps)
#print(self.fps)
#self.vidmag_frames.append(final_vid[-1])
self.frame_out = final_vid[-1]
def key_handler(self):
"""
A plotting or camera frame window must have focus for keypresses to be
detected.
"""
self.pressed = waitKey(1) & 255 # wait for keypress for 10 ms
if self.pressed == 27: # exit program on 'esc'
print("[INFO] Exiting")
self.webcam.stop()
sys.exit()
def mainLoop(self):
frame = self.webcam.get_frame()
f1 = imutils.resize(frame, width=256)
#crop_frame = frame[100:228,200:328]
self.data_buffer.append(f1)
self.run_color()
#print(frame)
#if len(self.vidmag_frames) > 0:
#print(self.vidmag_frames[0])
cv2.putText(frame, "FPS " + str(float("{:.2f}".format(self.fps))),
(20, 420), cv2.FONT_HERSHEY_PLAIN, 1.5, (0, 255, 0), 2)
#frame[100:228,200:328] = cv2.convertScaleAbs(self.vidmag_frames[-1])
cv2.imshow("Original", frame)
#f2 = imutils.resize(cv2.convertScaleAbs(self.vidmag_frames[-1]), width = 640)
f2 = imutils.resize(cv2.convertScaleAbs(self.frame_out), width=640)
cv2.imshow("Color amplification", f2)
self.key_handler() #if not the GUI cant show anything
class OpenGLGlyphs:
# constants
INVERSE_MATRIX = np.array([[1.0, 1.0, 1.0, 1.0], [-1.0, -1.0, -1.0, -1.0],
[-1.0, -1.0, -1.0, -1.0], [1.0, 1.0, 1.0, 1.0]])
def __init__(self):
# initialise webcam and start thread
self.webcam = Webcam()
self.webcam.start()
# textures
self.texture_background = None
self.texture_cube = None
def _init_gl(self, Width, Height):
glClearColor(0.0, 0.0, 0.0, 0.0)
glClearDepth(1.0)
glDepthFunc(GL_LESS)
glEnable(GL_DEPTH_TEST)
glShadeModel(GL_SMOOTH)
glMatrixMode(GL_PROJECTION)
glLoadIdentity()
gluPerspective(33.7, 1.3, 0.1, 100.0)
glMatrixMode(GL_MODELVIEW)
# enable textures
glEnable(GL_TEXTURE_2D)
self.texture_background = glGenTextures(1)
self.texture_cube = glGenTextures(1)
# create cube texture
image = Image.open("sample2.jpg")
ix = image.size[0]
iy = image.size[1]
image = image.tostring("raw", "RGBX", 0, -1)
glBindTexture(GL_TEXTURE_2D, self.texture_cube)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST)
glTexImage2D(GL_TEXTURE_2D, 0, 3, ix, iy, 0, GL_RGBA, GL_UNSIGNED_BYTE,
image)
def _draw_scene(self):
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
glLoadIdentity()
# get image from webcam
image = self.webcam.get_current_frame()
# convert image to OpenGL texture format
bg_image = cv2.flip(image, 0)
bg_image = Image.fromarray(bg_image)
ix = bg_image.size[0]
iy = bg_image.size[1]
bg_image = bg_image.tostring("raw", "BGRX", 0, -1)
# create background texture
glBindTexture(GL_TEXTURE_2D, self.texture_background)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST)
glTexImage2D(GL_TEXTURE_2D, 0, 3, ix, iy, 0, GL_RGBA, GL_UNSIGNED_BYTE,
bg_image)
# draw background
glBindTexture(GL_TEXTURE_2D, self.texture_background)
glPushMatrix()
glTranslatef(0.0, 0.0, -10.0)
self._draw_background()
glPopMatrix()
# handle glyph
image = self._handle_glyph(image)
glutSwapBuffers()
def _handle_glyph(self, image):
# attempt to detect glyph
rvecs = None
tvecs = None
try:
rvecs, tvecs = detect_glyph(image)
except Exception as ex:
print(ex)
if rvecs == None or tvecs == None:
return
# build view matrix
rmtx = cv2.Rodrigues(rvecs)[0]
view_matrix = np.array([[rmtx[0][0], rmtx[0][1], rmtx[0][2], tvecs[0]],
[rmtx[1][0], rmtx[1][1], rmtx[1][2], tvecs[1]],
[rmtx[2][0], rmtx[2][1], rmtx[2][2], tvecs[2]],
[0.0, 0.0, 0.0, 1.0]])
view_matrix = view_matrix * self.INVERSE_MATRIX
view_matrix = np.transpose(view_matrix)
# load view matrix and draw cube
glBindTexture(GL_TEXTURE_2D, self.texture_cube)
glPushMatrix()
glLoadMatrixd(view_matrix)
self._draw_cube()
glPopMatrix()
def _draw_cube(self):
# draw cube
glBegin(GL_QUADS)
glTexCoord2f(0.0, 0.0)
glVertex3f(0.0, 0.0, 0.0)
glTexCoord2f(1.0, 0.0)
glVertex3f(1.0, 0.0, 0.0)
glTexCoord2f(1.0, 1.0)
glVertex3f(1.0, 1.0, 0.0)
glTexCoord2f(0.0, 1.0)
glVertex3f(0.0, 1.0, 0.0)
glTexCoord2f(1.0, 0.0)
glVertex3f(0.0, 0.0, -1.0)
glTexCoord2f(1.0, 1.0)
glVertex3f(0.0, 1.0, -1.0)
glTexCoord2f(0.0, 1.0)
glVertex3f(1.0, 1.0, -1.0)
glTexCoord2f(0.0, 0.0)
glVertex3f(1.0, 0.0, -1.0)
glTexCoord2f(0.0, 1.0)
glVertex3f(0.0, 1.0, -1.0)
glTexCoord2f(0.0, 0.0)
glVertex3f(0.0, 1.0, 0.0)
glTexCoord2f(1.0, 0.0)
glVertex3f(1.0, 1.0, 0.0)
glTexCoord2f(1.0, 1.0)
glVertex3f(1.0, 1.0, -1.0)
glTexCoord2f(1.0, 1.0)
glVertex3f(0.0, 0.0, -1.0)
glTexCoord2f(0.0, 1.0)
glVertex3f(1.0, 0.0, -1.0)
glTexCoord2f(0.0, 0.0)
glVertex3f(1.0, 0.0, 0.0)
glTexCoord2f(1.0, 0.0)
glVertex3f(0.0, 0.0, 0.0)
glTexCoord2f(1.0, 0.0)
glVertex3f(1.0, 0.0, -1.0)
glTexCoord2f(1.0, 1.0)
glVertex3f(1.0, 1.0, -1.0)
glTexCoord2f(0.0, 1.0)
glVertex3f(1.0, 1.0, 0.0)
glTexCoord2f(0.0, 0.0)
glVertex3f(1.0, 0.0, 0.0)
glTexCoord2f(0.0, 0.0)
glVertex3f(0.0, 0.0, -1.0)
glTexCoord2f(1.0, 0.0)
glVertex3f(0.0, 0.0, 0.0)
glTexCoord2f(1.0, 1.0)
glVertex3f(0.0, 1.0, 0.0)
glTexCoord2f(0.0, 1.0)
glVertex3f(0.0, 1.0, -1.0)
glEnd()
def _draw_background(self):
# draw background
glBegin(GL_QUADS)
glTexCoord2f(0.0, 1.0)
glVertex3f(-4.0, -3.0, 0.0)
glTexCoord2f(1.0, 1.0)
glVertex3f(4.0, -3.0, 0.0)
glTexCoord2f(1.0, 0.0)
glVertex3f(4.0, 3.0, 0.0)
glTexCoord2f(0.0, 0.0)
glVertex3f(-4.0, 3.0, 0.0)
glEnd()
def main(self):
# setup and run OpenGL
glutInit()
glutInitDisplayMode(GLUT_RGBA | GLUT_DOUBLE | GLUT_DEPTH)
glutInitWindowSize(640, 480)
glutInitWindowPosition(800, 400)
self.window_id = glutCreateWindow("OpenGL Glyphs")
glutDisplayFunc(self._draw_scene)
glutIdleFunc(self._draw_scene)
self._init_gl(640, 480)
glutMainLoop()
class SaltwashAR:
# constants
INVERSE_MATRIX = np.array([[ 1.0, 1.0, 1.0, 1.0],
[-1.0,-1.0,-1.0,-1.0],
[-1.0,-1.0,-1.0,-1.0],
[ 1.0, 1.0, 1.0, 1.0]])
def __init__(self):
# initialise config
self.config_provider = ConfigProvider()
# initialise robots
self.rocky_robot = RockyRobot()
self.sporty_robot = SportyRobot()
# initialise webcam
self.webcam = Webcam()
# initialise markers
self.markers = Markers()
self.markers_cache = None
# initialise features
self.features = Features(self.config_provider)
# initialise texture
self.texture_background = None
def _init_gl(self):
glClearColor(0.0, 0.0, 0.0, 0.0)
glClearDepth(1.0)
glDepthFunc(GL_LESS)
glEnable(GL_DEPTH_TEST)
glShadeModel(GL_SMOOTH)
glMatrixMode(GL_PROJECTION)
glLoadIdentity()
gluPerspective(33.7, 1.3, 0.1, 100.0)
glMatrixMode(GL_MODELVIEW)
# load robots frames
self.rocky_robot.load_frames(self.config_provider.animation)
self.sporty_robot.load_frames(self.config_provider.animation)
# start webcam thread
self.webcam.start()
# assign texture
glEnable(GL_TEXTURE_2D)
self.texture_background = glGenTextures(1)
def _draw_scene(self):
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
glLoadIdentity()
# reset robots
self.rocky_robot.reset()
self.sporty_robot.reset()
# get image from webcam
image = self.webcam.get_current_frame()
# handle background
self._handle_background(image.copy())
# handle markers
self._handle_markers(image.copy())
# handle features
self.features.handle(self.rocky_robot, self.sporty_robot, image.copy())
glutSwapBuffers()
def _handle_background(self, image):
# let features update background image
image = self.features.update_background_image(image)
# convert image to OpenGL texture format
bg_image = cv2.flip(image, 0)
bg_image = Image.fromarray(bg_image)
ix = bg_image.size[0]
iy = bg_image.size[1]
bg_image = bg_image.tobytes('raw', 'BGRX', 0, -1)
# create background texture
glBindTexture(GL_TEXTURE_2D, self.texture_background)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST)
glTexImage2D(GL_TEXTURE_2D, 0, 3, ix, iy, 0, GL_RGBA, GL_UNSIGNED_BYTE, bg_image)
# draw background
glBindTexture(GL_TEXTURE_2D, self.texture_background)
glPushMatrix()
glTranslatef(0.0,0.0,-10.0)
glBegin(GL_QUADS)
glTexCoord2f(0.0, 1.0); glVertex3f(-4.0, -3.0, 0.0)
glTexCoord2f(1.0, 1.0); glVertex3f( 4.0, -3.0, 0.0)
glTexCoord2f(1.0, 0.0); glVertex3f( 4.0, 3.0, 0.0)
glTexCoord2f(0.0, 0.0); glVertex3f(-4.0, 3.0, 0.0)
glEnd( )
glPopMatrix()
def _handle_markers(self, image):
# attempt to detect markers
markers = []
try:
markers = self.markers.detect(image)
except Exception as ex:
print(ex)
# manage markers cache
if markers:
self.markers_cache = markers
elif self.markers_cache:
markers = self.markers_cache
self.markers_cache = None
else:
return
for marker in markers:
rvecs, tvecs, marker_rotation, marker_name = marker
# build view matrix
rmtx = cv2.Rodrigues(rvecs)[0]
view_matrix = np.array([[rmtx[0][0],rmtx[0][1],rmtx[0][2],tvecs[0]],
[rmtx[1][0],rmtx[1][1],rmtx[1][2],tvecs[1]],
[rmtx[2][0],rmtx[2][1],rmtx[2][2],tvecs[2]],
[0.0 ,0.0 ,0.0 ,1.0 ]])
view_matrix = view_matrix * self.INVERSE_MATRIX
view_matrix = np.transpose(view_matrix)
# load view matrix and draw cube
glPushMatrix()
glLoadMatrixd(view_matrix)
if marker_name == ROCKY_ROBOT:
self.rocky_robot.next_frame(marker_rotation, self.features.is_speaking(), self.features.get_emotion())
elif marker_name == SPORTY_ROBOT:
self.sporty_robot.next_frame(marker_rotation, self.features.is_speaking(), self.features.get_emotion())
glColor3f(1.0, 1.0, 1.0)
glPopMatrix()
def main(self):
# setup and run OpenGL
glutInit()
glutInitDisplayMode(GLUT_RGBA | GLUT_DOUBLE | GLUT_DEPTH)
glutInitWindowSize(640, 480)
glutInitWindowPosition(100, 100)
glutCreateWindow('SaltwashAR')
glutDisplayFunc(self._draw_scene)
glutIdleFunc(self._draw_scene)
self._init_gl()
glutMainLoop()
class OpenGLGlyphs:
# constants
INVERSE_MATRIX = np.array([[ 1.0, 1.0, 1.0, 1.0],
[-1.0,-1.0,-1.0,-1.0],
[-1.0,-1.0,-1.0,-1.0],
[ 1.0, 1.0, 1.0, 1.0]])
def __init__(self):
# initialise webcam and start thread
self.webcam = Webcam()
self.webcam.start()
# initialise glyphs
self.glyphs = Glyphs()
# initialise shapes
self.cone = None
self.sphere = None
# initialise texture
self.texture_background = None
def _init_gl(self, Width, Height):
glClearColor(0.0, 0.0, 0.0, 0.0)
glClearDepth(1.0)
glDepthFunc(GL_LESS)
glEnable(GL_DEPTH_TEST)
glShadeModel(GL_SMOOTH)
glMatrixMode(GL_PROJECTION)
glLoadIdentity()
gluPerspective(33.7, 1.3, 0.1, 100.0)
glMatrixMode(GL_MODELVIEW)
# assign shapes
self.cone = OBJ('cone.obj')
self.sphere = OBJ('sphere.obj')
# assign texture
glEnable(GL_TEXTURE_2D)
self.texture_background = glGenTextures(1)
def _draw_scene(self):
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
glLoadIdentity()
# get image from webcam
image = self.webcam.get_current_frame()
# convert image to OpenGL texture format
bg_image = cv2.flip(image, 0)
bg_image = Image.fromarray(bg_image)
ix = bg_image.size[0]
iy = bg_image.size[1]
bg_image = bg_image.tobytes("raw", "BGRX", 0, -1)
# create background texture
glBindTexture(GL_TEXTURE_2D, self.texture_background)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST)
glTexImage2D(GL_TEXTURE_2D, 0, 3, ix, iy, 0, GL_RGBA, GL_UNSIGNED_BYTE, bg_image)
# draw background
glBindTexture(GL_TEXTURE_2D, self.texture_background)
glPushMatrix()
glTranslatef(0.0,0.0,-10.0)
self._draw_background()
glPopMatrix()
# handle glyphs
image = self._handle_glyphs(image)
glutSwapBuffers()
def _handle_glyphs(self, image):
# attempt to detect glyphs
glyphs = []
try:
glyphs = self.glyphs.detect(image)
except Exception as ex:
print(ex)
if not glyphs:
return
for glyph in glyphs:
rvecs, tvecs, glyph_name = glyph
# build view matrix
rmtx = cv2.Rodrigues(rvecs)[0]
view_matrix = np.array([[rmtx[0][0],rmtx[0][1],rmtx[0][2],tvecs[0]],
[rmtx[1][0],rmtx[1][1],rmtx[1][2],tvecs[1]],
[rmtx[2][0],rmtx[2][1],rmtx[2][2],tvecs[2]],
[0.0 ,0.0 ,0.0 ,1.0 ]])
view_matrix = view_matrix * self.INVERSE_MATRIX
view_matrix = np.transpose(view_matrix)
# load view matrix and draw shape
glPushMatrix()
glLoadMatrixd(view_matrix)
if glyph_name == SHAPE_CONE:
glCallList(self.cone.gl_list)
elif glyph_name == SHAPE_SPHERE:
glCallList(self.sphere.gl_list)
glPopMatrix()
def _draw_background(self):
# draw background
glBegin(GL_QUADS)
glTexCoord2f(0.0, 1.0); glVertex3f(-4.0, -3.0, 0.0)
glTexCoord2f(1.0, 1.0); glVertex3f( 4.0, -3.0, 0.0)
glTexCoord2f(1.0, 0.0); glVertex3f( 4.0, 3.0, 0.0)
glTexCoord2f(0.0, 0.0); glVertex3f(-4.0, 3.0, 0.0)
glEnd( )
def main(self):
# setup and run OpenGL
glutInit()
glutInitDisplayMode(GLUT_RGBA | GLUT_DOUBLE | GLUT_DEPTH)
glutInitWindowSize(640, 480)
glutInitWindowPosition(800, 400)
self.window_id = glutCreateWindow("OpenGL Glyphs")
glutDisplayFunc(self._draw_scene)
glutIdleFunc(self._draw_scene)
self._init_gl(640, 480)
glutMainLoop()
class OpenGLGlyphs:
# constants
INVERSE_MATRIX = np.array([[ 1.0, 1.0, 1.0, 1.0],
[-1.0,-1.0,-1.0,-1.0],
[-1.0,-1.0,-1.0,-1.0],
[ 1.0, 1.0, 1.0, 1.0]])
def __init__(self):
# initialise webcam and start thread
self.webcam = Webcam()
self.webcam.start()
# initialise glyphs
self.glyphs = Glyphs()
# initialise shapes
self.cone = None
self.sphere = None
# self.hp = HP()
# initialise texture
self.texture_background = None
def _init_gl(self, Width, Height):
glClearColor(0.0, 0.0, 0.0, 0.0)
glClearDepth(1.0)
glDepthFunc(GL_LESS)
glEnable(GL_DEPTH_TEST)
glShadeModel(GL_SMOOTH)
glMatrixMode(GL_PROJECTION)
glLoadIdentity()
gluPerspective(33.7, 1.3, 0.1, 100.0)
glMatrixMode(GL_MODELVIEW)
# assign shapes
self.cone = OBJ('texturedCube.obj')
self.sphere = OBJ('./sortedOut/hair.obj')
# assign texture
glEnable(GL_TEXTURE_2D)
self.texture_background = glGenTextures(1)
def _draw_scene(self):
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
glLoadIdentity()
# get image from webcam
image = self.webcam.get_current_frame()
# convert image to OpenGL texture formatz
bg_image = cv2.flip(image, 0)
bg_image = np.asarray(bg_image)
bg_image = Image.fromarray(bg_image)
ix = bg_image.size[0]
iy = bg_image.size[1]
bg_image = bg_image.tobytes("raw", "BGRX", 0, -1)
# create background texture
glBindTexture(GL_TEXTURE_2D, self.texture_background)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST)
glTexImage2D(GL_TEXTURE_2D, 0, 3, ix, iy, 0, GL_RGBA, GL_UNSIGNED_BYTE, bg_image)
# draw background
glBindTexture(GL_TEXTURE_2D, self.texture_background)
glPushMatrix()
glTranslatef(0.0,0.0,-10.0)
self._draw_background()
glPopMatrix()
# handle glyphs
image = self._handle_headpose(image)
glutSwapBuffers()
def _handle_headpose(self, image):
# attempt to detect glyphs
glyphs = []
# glyphs = self.hp.main(image)
try:
glyphs = self.glyphs.detect(image)
except Exception as ex:
print(ex)
if not glyphs:
return
for glyph in glyphs:
rvecs, tvecs, glyph_name = glyph
# build view matrix
rmtx = cv2.Rodrigues(rvecs)[0]
view_matrix = np.array([[rmtx[0][0],rmtx[0][1],rmtx[0][2],tvecs[0]],
[rmtx[1][0],rmtx[1][1],rmtx[1][2],tvecs[1]],
[rmtx[2][0],rmtx[2][1],rmtx[2][2],tvecs[2]],
[0.0 ,0.0 ,0.0 ,1.0 ]])
view_matrix = view_matrix * self.INVERSE_MATRIX
view_matrix = np.transpose(view_matrix)
# load view matrix and draw shape
glPushMatrix()
glLoadMatrixd(view_matrix)
if glyph_name == SHAPE_CONE:
glCallList(self.cone.gl_list)
elif glyph_name == SHAPE_SPHERE:
glCallList(self.sphere.gl_list)
glPopMatrix()
def _draw_background(self):
# draw background
glBegin(GL_QUADS)
glTexCoord2f(0.0, 1.0); glVertex3f(-4.0, -3.0, 0.0)
glTexCoord2f(1.0, 1.0); glVertex3f( 4.0, -3.0, 0.0)
glTexCoord2f(1.0, 0.0); glVertex3f( 4.0, 3.0, 0.0)
glTexCoord2f(0.0, 0.0); glVertex3f(-4.0, 3.0, 0.0)
glEnd( )
def main(self):
# setup and run OpenGL
glutInit()
glutInitDisplayMode(GLUT_RGBA | GLUT_DOUBLE | GLUT_DEPTH)
glutInitWindowSize(640, 480)
glutInitWindowPosition(800, 400)
self.window_id = glutCreateWindow(b"OpenGL Glyphs")
glutDisplayFunc(self._draw_scene)
glutIdleFunc(self._draw_scene)
self._init_gl(640, 480)
glutMainLoop()
class OpenGLGlyphs:
##############################################################초기화
def __init__(self):
# initialise webcam and start thread
self.webcam = Webcam()
self.webcam.start()
self.find = fp()
self.find.set_img('book.jpg')
self.hei, self.wid = self.webcam.get_frame_shape()[:2]
# initialise cube
self.d_obj = None
self.img = None
# initialise texture
self.texture_background = None
self.K = None
self.mark_kp = None
self.mark_des = None
self.set_keypoint()
self.new_kp = None
self.mat_kp = None
self.mat_des = None
self.H = None
# self.Rt=None
##############################################################카메라 세팅
def _init_gl(self, Width, Height):
glClearColor(0.0, 0.0, 0.0, 0.0)
glClearDepth(1.0)
glDepthFunc(GL_LESS)
glEnable(GL_DEPTH_TEST)
glShadeModel(GL_SMOOTH)
glMatrixMode(GL_PROJECTION)
glLoadIdentity()
self.K = my_calibration((Height, Width))
fx = self.K[0, 0]
fy = self.K[1, 1]
fovy = 2 * np.arctan(0.5 * Height / fy) * 180 / np.pi
aspect = (float)(Width * fy) / (Height * fx)
# define the near and far clipping planes
near = 0.1
far = 100.0
# set perspective
gluPerspective(fovy, aspect, near, far)
glMatrixMode(GL_MODELVIEW)
glEnable(GL_TEXTURE_2D)
self.texture_background = glGenTextures(1)
##############################################################marker의 kp, des저장
def set_keypoint(self):
self.find.start()
self.mark_kp, self.mark_des = self.find.get_point()
##############################################################K값 구하기
def _draw_scene(self):
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
glLoadIdentity()
# get image from webcam
image = self.webcam.get_current_frame()
Rt = self._my_cal(image)
# convert image to OpenGL texture format
bg_image = cv2.flip(image, 0)
bg_image = Image.fromarray(bg_image)
ix = bg_image.size[0]
iy = bg_image.size[1]
bg_image = bg_image.tobytes("raw", "BGRX", 0, -1)
# create background texture
glBindTexture(GL_TEXTURE_2D, self.texture_background)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST)
glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, self.wid, self.hei, 0, GL_RGBA,
GL_UNSIGNED_BYTE, bg_image)
# draw background
glBindTexture(GL_TEXTURE_2D, self.texture_background)
glPushMatrix()
# glTranslatef(0.0,0.0,0.0)
gluLookAt(0.0, 0.0, 12, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0)
self._draw_background()
glPopMatrix()
################Rt를 구해서 매칭되는 이미지가 있는지 판단
if Rt is not None:
self._set_modelview_from_camera(Rt)
glEnable(GL_LIGHTING)
glEnable(GL_LIGHT0)
glEnable(GL_DEPTH_TEST)
glEnable(GL_NORMALIZE)
glClear(GL_DEPTH_BUFFER_BIT)
ObjLoader("jnu.obj").render_scene()
glutSwapBuffers()
##############################################################OpenGL용 Rt변환
def _set_modelview_from_camera(self, Rt):
glMatrixMode(GL_MODELVIEW)
glLoadIdentity()
Rx = np.array([[0.2, 0, 0], [0, 0, 0.2], [0, 0.2, 0]])
# set rotation to best approximation
R = Rt[:, :3]
# change sign of x-axis
R[0, :] = -R[0, :]
# set translation
t = Rt[:, 3]
t[0] = -t[0]
# setup 4*4 model view matrix
M = np.eye(4)
M[:3, :3] = np.dot(R, Rx)
M[:3, 3] = t
M[3, :3] = t
# transpose and flatten to get column order
M = M.T
m = M.flatten()
# replace model view with the new matrix
glLoadMatrixf(m)
##############################################################Rt반환
def _my_cal(self, image):
find_H = fp()
find_H.set_cv_img(image)
find_H.start()
kp, des = find_H.get_point()
self.H = self.match_images(self.mark_kp, self.mark_des, kp, des)
if self.H is not None:
cam1 = camera.Camera(
np.hstack((self.K, np.dot(self.K, np.array([[0], [0],
[-1]])))))
# Rt1=dot(linalg.inv(self.K),cam1.P)
cam2 = camera.Camera(np.dot(self.H, cam1.P))
A = np.dot(np.linalg.inv(self.K), cam2.P[:, :3])
A = np.array([A[:, 0], A[:, 1], np.cross(A[:, 0], A[:, 1])]).T
cam2.P[:, :3] = np.dot(self.K, A)
Rt = np.dot(np.linalg.inv(self.K), cam2.P)
return Rt
else:
return None
##############################################################match image
def match_images(self, kp1, des1, kp2, des2):
matcher = cv2.BFMatcher()
match_des = matcher.knnMatch(des1, des2, k=2)
matches = []
matA, matB = [], []
matC = []
for m in match_des:
if m[0].distance < 0.8 * m[1].distance:
matA.append(kp1[m[0].queryIdx])
matB.append(kp2[m[0].trainIdx])
matC.append(des1[m[0].queryIdx])
if len(matA) > 50:
ptsA = np.float32([m.pt for m in matA])
ptsB = np.float32([n.pt for n in matB])
H1 = []
H1, status = cv2.findHomography(ptsA, ptsB, cv2.RANSAC, 5.0)
H1 = self.homo_check(H1)
self.mat_kp = np.array(
[matB[i] for i in range(status.shape[0]) if status[i] == 1])
self.mat_des = np.array(
[matC[i] for i in range(status.shape[0]) if status[i] == 1])
return H1
else:
return None
##############################################################homography check
def homo_check(self, H1):
if self.H is None:
return H1
else:
if cv2.norm(H1, self.H) > 1.0:
return H1
else:
return self.H
def _draw_background(self):
# draw background
glBegin(GL_QUADS)
glTexCoord2f(0.0, 1.0)
glVertex3f(-4.0, -3.0, 0.0)
glTexCoord2f(1.0, 1.0)
glVertex3f(4.0, -3.0, 0.0)
glTexCoord2f(1.0, 0.0)
glVertex3f(4.0, 3.0, 0.0)
glTexCoord2f(0.0, 0.0)
glVertex3f(-4.0, 3.0, 0.0)
glEnd()
glDeleteTextures(1)
def keyboard(self, *args):
if args[0] == GLUT_KEY_UP:
glutDestroyWindow(self.window_id)
self.webcam.finish()
sys.exit()
##############################################################OpenGL창 초기
def main(self):
# setup and run OpenGL
glutInit()
glutInitDisplayMode(GLUT_RGBA | GLUT_DOUBLE | GLUT_DEPTH)
glutInitWindowSize(self.wid, self.hei)
glutInitWindowPosition(200, 200)
self.window_id = glutCreateWindow(b"OpenGL Glyphs")
self._init_gl(self.wid, self.hei)
glutDisplayFunc(self._draw_scene)
glutIdleFunc(self._draw_scene)
glutMainLoop()
class MUcamera:
def __init__(self):
self.w = Webcam()
self.start = self.w.start()
self.im = self.w.grab_image()
self.w.register_callback(self.average_intensity, 1)
self.avg_intensity = []
self.images = []
self.filtered = []
self.f, self.ax = plt.subplots(1, 1)
#To find the average intensity
def average_intensity(self, image):
pix_val = list(image.getdata())
pixel_intensity = []
for x in pix_val:
avg = sum(x) / len(x)
pixel_intensity.append(avg)
self.avg_pixel = np.average(pixel_intensity)
self.avg_intensity.append(self.avg_pixel)
return self.avg_intensity
#To find the average filtered intensity using width as 3
def average_intensity_filtered(self):
width = 3
if len(self.avg_intensity) >= 5:
for x in range(len(self.avg_intensity) - 2):
self.filtered.append(
(self.avg_intensity[x] + self.avg_intensity[x + 1] +
self.avg_intensity[x + 2]) / width)
return self.filtered
else:
self.filtered = self.avg_intensity
return self.filtered
#To stop the execution of webcam.py
def stop(self):
self.w.stop()
self.average_intensity_mean_plot()
self.average_intensity_filtered_plot()
self.daytime()
self.most_common_color()
#To plot the average intensity
def average_intensity_mean_plot(self):
self.ax.plot(self.avg_intensity, 'C1', label='Average')
self.ax.legend()
self.ax.set_xlabel('Image Number')
self.ax.set_ylabel('Intensity')
self.ax.set_title('Image Intensity')
#To plot the average filtered intensity
def average_intensity_filtered_plot(self):
self.average_intensity_filtered()
self.ax.plot(self.filtered, 'C2', label='Filtered')
self.ax.legend()
#To check if it is daytime or nighttime
def daytime(self):
self.average = np.mean(np.mean(self.im, axis=1))
if self.average >= 95:
return print("True")
else:
return print("False")
#To check the most common color
def most_common_color(self):
w, h = self.im.size
pixels = self.im.getcolors(w * h)
most_frequent_pixel = pixels[0]
for count, color in pixels:
if count > most_frequent_pixel[0]:
most_frequent_pixel = (count, color)
proportion = most_frequent_pixel[0] / len(pixels)
return print(
'The most common color is {}'.format(most_frequent_pixel[1]),
'with a count of {}'.format(most_frequent_pixel[0]),
'and the proportion of pixels is {}'.format(proportion))
import cv2
from glyphfunctions import *
from glyphdatabase import *
from webcam import Webcam
webcam = Webcam()
webcam.start()
QUADRILATERAL_POINTS = 4
BLACK_THRESHOLD = 100
WHITE_THRESHOLD = 155
while True:
# Stage 1: Read an image from our webcam
image = webcam.get_current_frame()
# Stage 2: Detect edges in image
gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY)
gray = cv2.GaussianBlur(gray, (5, 5), 0)
edges = cv2.Canny(gray, 100, 200)
# Stage 3: Find contours
contours, _ = cv2.findContours(edges, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
cv2.imshow('edges',edges)
# contours = sorted(contours, key=cv2.contourArea, reverse=True)[:10]
#
# for contour in contours:
#
# # Stage 4: Shape check
# perimeter = cv2.arcLength(contour, True)
def main():
try:
camera_type = sys.argv[1]
recording = False
if len(sys.argv) == 3:
if sys.argv[2] == "record":
recording = True
else:
recording = False
if camera_type == "webcam":
collector = Webcam(video_width=640, video_height=480)
collector.start()
else:
print("No such camera {camera_type}")
collector = None
exit(-1)
if not os.path.isfile(MODEL_PATH):
print("Downloading model, please wait...")
download_file_from_google_drive(SOURCE, MODEL_PATH)
print("Done downloading the model.")
# get device
device = torch.device(
'cuda') if torch.cuda.is_available() else torch.device('cpu')
# initialise model
model = get_model_instance_segmentation(NUMBER_OF_CLASSES)
model.load_state_dict(
torch.load('./models/frcnn_hands.pth', map_location=device))
model.to(device)
model.eval()
if recording:
movie = cv2.VideoWriter(
f'./recordings/hand_frcnn_{camera_type}.avi',
cv2.VideoWriter_fourcc(*'DIVX'), 8, (640, 480))
with torch.no_grad():
while collector.started:
image, _ = collector.read()
if image is not None:
orig = image.copy()
image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
image = transforms.ToTensor()(image).to(device)
out = model([image])
boxes = get_prediction(pred=out, threshold=.7)
try:
for box in boxes:
cv2.rectangle(img=orig,
pt1=(box[0], box[1]),
pt2=(box[2], box[3]),
color=(0, 255, 255),
thickness=2)
if recording:
movie.write(orig)
cv2.imshow("mask", orig)
k = cv2.waitKey(1)
if k == ord('q'):
collector.stop()
except Exception as e:
print(e)
finally:
print("Stopping stream.")
collector.stop()
if recording:
movie.release()
cv2.destroyAllWindows()
class OpenGLGlyphs:
INVERSE_MATRIX = np.array([[1.0, 1.0, 1.0, 1.0], [-1.0, -1.0, -1.0, -1.0],
[-1.0, -1.0, -1.0, -1.0], [1.0, 1.0, 1.0, 1.0]])
def __init__(self):
self.webcam = Webcam()
self.webcam.start()
self.glyphs = Glyphs()
self.cone = None
self.sphere = None
self.texture_background = None
def _init_gl(self, Width, Height):
glClearColor(0.0, 0.0, 0.0, 0.0)
glClearDepth(1.0)
glDepthFunc(GL_LESS)
glEnable(GL_DEPTH_TEST)
glShadeModel(GL_SMOOTH)
glMatrixMode(GL_PROJECTION)
glLoadIdentity()
gluPerspective(33.7, 1.3, 0.1, 100.0)
glMatrixMode(GL_MODELVIEW)
self.cone = OBJ('cone.obj')
self.sphere = OBJ('sphere.obj')
glEnable(GL_TEXTURE_2D)
self.texture_background = glGenTextures(1)
def _draw_scene(self):
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
glLoadIdentity()
image = self.webcam.get_current_frame()
bg_image = cv2.flip(image, 0)
bg_image = Image.fromarray(bg_image)
ix = bg_image.size[0]
iy = bg_image.size[1]
bg_image = bg_image.tostring("raw", "BGRX", 0, -1)
glBindTexture(GL_TEXTURE_2D, self.texture_background)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST)
glTexImage2D(GL_TEXTURE_2D, 0, 3, ix, iy, 0, GL_RGBA, GL_UNSIGNED_BYTE,
bg_image)
glBindTexture(GL_TEXTURE_2D, self.texture_background)
glPushMatrix()
glTranslatef(0.0, 0.0, -10.0)
self._draw_background()
glPopMatrix()
image = self._handle_glyphs(image)
glutSwapBuffers()
def _handle_glyphs(self, image):
glyphs = []
try:
glyphs = self.glyphs.detect(image)
except Exception as ex:
print(ex)
if not glyphs:
return
for glyph in glyphs:
rvecs, tvecs, glyph_name = glyph
rmtx = cv2.Rodrigues(rvecs)[0]
view_matrix = np.array(
[[rmtx[0][0], rmtx[0][1], rmtx[0][2], tvecs[0]],
[rmtx[1][0], rmtx[1][1], rmtx[1][2], tvecs[1]],
[rmtx[2][0], rmtx[2][1], rmtx[2][2], tvecs[2]],
[0.0, 0.0, 0.0, 1.0]])
view_matrix = view_matrix * self.INVERSE_MATRIX
view_matrix = np.transpose(view_matrix)
glPushMatrix()
glLoadMatrixd(view_matrix)
print(view_matrix)
if glyph_name == SHAPE_CONE:
glCallList(self.cone.gl_list)
elif glyph_name == SHAPE_SPHERE:
glCallList(self.sphere.gl_list)
glPopMatrix()
def _draw_background(self):
glBegin(GL_QUADS)
glTexCoord2f(0.0, 1.0)
glVertex3f(-4.0, -3.0, 0.0)
glTexCoord2f(1.0, 1.0)
glVertex3f(4.0, -3.0, 0.0)
glTexCoord2f(1.0, 0.0)
glVertex3f(4.0, 3.0, 0.0)
glTexCoord2f(0.0, 0.0)
glVertex3f(-4.0, 3.0, 0.0)
glEnd()
def main(self):
glutInit()
glutInitDisplayMode(GLUT_RGBA | GLUT_DOUBLE | GLUT_DEPTH)
glutInitWindowSize(640, 480)
glutInitWindowPosition(800, 400)
self.window_id = glutCreateWindow("nichi Technologies")
glutDisplayFunc(self._draw_scene)
glutIdleFunc(self._draw_scene)
self._init_gl(640, 480)
glutMainLoop()
class ArkwoodAR:
def __init__(self):
# initialise webcams
self.webcam_one = Webcam(0)
self.webcam_two = Webcam(1)
# initialise config
self.config_provider = ConfigProvider()
# initialise features
self.features = Features(self.config_provider)
# initialise texture
self.texture_background = None
def _init_gl(self):
glClearColor(0.0, 0.0, 0.0, 0.0)
glClearDepth(1.0)
glDepthFunc(GL_LESS)
glEnable(GL_DEPTH_TEST)
glShadeModel(GL_SMOOTH)
glMatrixMode(GL_PROJECTION)
glLoadIdentity()
gluPerspective(33.7, 1.3, 0.1, 100.0)
glMatrixMode(GL_MODELVIEW)
# start webcam threads
self.webcam_one.start()
self.webcam_two.start()
# assign texture
glEnable(GL_TEXTURE_2D)
self.texture_background = glGenTextures(1)
def _draw_scene(self):
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
glLoadIdentity()
window_half_width = glutGet(GLUT_WINDOW_WIDTH) / 2
window_height = glutGet(GLUT_WINDOW_HEIGHT)
# get image from webcams
image_one = self.webcam_one.get_current_frame()
image_two = self.webcam_two.get_current_frame()
# detect feature in images
detection = self.features.detect(image_one, image_two)
# render first image
glViewport(0, 0, window_half_width, window_height)
if detection:
image_one = self.features.render(image_one, detection[0])
self._handle_background(image_one)
# render second image
glViewport(window_half_width, 0, window_half_width, window_height)
if detection:
image_two = self.features.render(image_two, detection[1])
self._handle_background(image_two)
# swap buffers
glutSwapBuffers()
# handle background
def _handle_background(self, image):
# convert image to OpenGL texture format
bg_image = cv2.flip(image, 0)
bg_image = Image.fromarray(bg_image)
ix = bg_image.size[0]
iy = bg_image.size[1]
bg_image = bg_image.tobytes('raw', 'BGRX', 0, -1)
# create background texture
glBindTexture(GL_TEXTURE_2D, self.texture_background)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST)
glTexImage2D(GL_TEXTURE_2D, 0, 3, ix, iy, 0, GL_RGBA, GL_UNSIGNED_BYTE,
bg_image)
# draw background
glBindTexture(GL_TEXTURE_2D, self.texture_background)
glPushMatrix()
glTranslatef(0.0, 0.0, -10.0)
glBegin(GL_QUADS)
glTexCoord2f(0.0, 1.0)
glVertex3f(-4.0, -3.0, 0.0)
glTexCoord2f(1.0, 1.0)
glVertex3f(4.0, -3.0, 0.0)
glTexCoord2f(1.0, 0.0)
glVertex3f(4.0, 3.0, 0.0)
glTexCoord2f(0.0, 0.0)
glVertex3f(-4.0, 3.0, 0.0)
glEnd()
glPopMatrix()
def main(self):
# setup and run OpenGL
glutInit()
glutInitDisplayMode(GLUT_RGBA | GLUT_DOUBLE | GLUT_DEPTH)
glutInitWindowSize(640, 360)
glutInitWindowPosition(100, 100)
glutCreateWindow('ArkwoodAR')
glutDisplayFunc(self._draw_scene)
glutIdleFunc(self._draw_scene)
self._init_gl()
glutMainLoop()
class ArucoFootball:
# constants
INVERSE_MATRIX = np.array([[1.0, 1.0, 1.0, 1.0],
[-1.0, -1.0, -1.0, -1.0],
[-1.0, -1.0, -1.0, -1.0],
[1.0, 1.0, 1.0, 1.0]])
def __init__(self, btAddr):
# init needed values
self.btAddr = btAddr
self.player = None
self.texture_background = None
self.set_ids = []
self.set_players = []
self.players = []
self.calc_values()
# initialise webcam and start thread
self.webcam = Webcam()
self.webcam.start()
# connect wiimote and create model
wiiModel = WiiModel.WiiModel(self.btAddr)
# init openGl, Qt and Wiimote
self.initOpenGL()
self.initGUI()
# run wiimote-connection-loop
thread = threading.Thread(target=wiiModel.wiimoteLoop, args=(self.mainWindow, self.cursor))
thread.start()
# run opengl and camera in a thread
thread = threading.Thread(target=glutMainLoop, args=())
thread.start()
# run Qt
self.app.exec_()
def initGUI(self):
self.app = QApplication(sys.argv)
self.mainWindow = MainWindow(self.players)
self.mainWindow.show()
self.set_player_widget = self.mainWindow.listWidgetB
self.unset_player_widget = self.mainWindow.listWidgetA
self.unset_player_widget.itemChanged.connect(self.removeID)
self.mainWindow.setFocus()
self.mainWindow.setWindowTitle("Tactic-Window")
self.mainWindow.resize(600, 800)
self.cursor = QCursor()
def initOpenGL(self):
# setup OpenGL
glutInit()
glutInitDisplayMode(GLUT_RGBA | GLUT_DOUBLE | GLUT_DEPTH)
glutInitWindowSize(1280, 960)
glutInitWindowPosition(800, 400)
self.window_id = glutCreateWindow("Footballfield")
glutDisplayFunc(self._draw_scene)
glutIdleFunc(self._draw_scene)
glClearColor(0.0, 0.0, 0.0, 0.0)
glClearDepth(1.0)
glDepthFunc(GL_LESS)
glEnable(GL_DEPTH_TEST)
glShadeModel(GL_SMOOTH)
glMatrixMode(GL_PROJECTION)
glLoadIdentity()
gluPerspective(33.7, 1.3, 0.1, 100.0)
glMatrixMode(GL_MODELVIEW)
# assign shapes
player_model = "models/football-player-new.obj"
self.player1 = OBJ(player_model, 1)
self.player2 = OBJ(player_model, 2)
self.player3 = OBJ(player_model, 3)
# self.player4 = OBJ(player_model, 4)
# add Players to list
self.players.append(Player("Dani", "1", "player_images/dani-img.jpg", self.player1))
self.players.append(Player("Maxi", "2", "player_images/maxi-img.jpg", self.player2))
self.players.append(Player("Jonas", "3", "player_images/jonas-img.jpg", self.player3))
# self.players.append(Player("Michi", "4", "player_images/michi-img.jpg", self.player4))
# assign texture
glEnable(GL_TEXTURE_2D)
self.texture_background = glGenTextures(1)
# if player removed from list -> remove set id
def removeID(self):
for index in range(self.unset_player_widget.count()):
item = self.unset_player_widget.item(index)
if item:
data = item.data(Qt.UserRole)
if data:
for player in self.players:
if player.number == data[0] and player.marker_num is not None:
self.set_ids.remove(player.marker_num)
player.marker_num = None
# add players to set_players for all players on "Field"
def setChangedListItems(self):
items = []
for index in range(self.set_player_widget.count()):
item = self.set_player_widget.item(index)
if item:
data = item.data(Qt.UserRole)
for player in self.players:
if player.number == data[0]:
items.append(player)
self.set_players = items
def _draw_scene(self):
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
glLoadIdentity()
# get image from webcam
image = self.webcam.get_current_frame()
# convert image to OpenGL texture format
bg_image = cv2.flip(image, 0)
bg_image = Image.fromarray(bg_image)
ix = bg_image.size[0]
iy = bg_image.size[1]
bg_image = bg_image.tobytes("raw", "BGRX", 0, -1)
# create background texture
glBindTexture(GL_TEXTURE_2D, self.texture_background)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST)
glTexImage2D(GL_TEXTURE_2D, 0, 3, ix, iy, 0, GL_RGBA, GL_UNSIGNED_BYTE, bg_image)
# draw background
glBindTexture(GL_TEXTURE_2D, self.texture_background)
glPushMatrix()
glTranslatef(0.0, 0.0, -10.0)
self._draw_background()
glPopMatrix()
# handle glyphs
image = self._handle_aruco(image)
glutSwapBuffers()
def calc_values(self):
path = 'calib_images/*.jpg'
self.ret, self.mtx, self.dist, self.rvecs, self.tvecs = Tracker.calculate_camera_values(path)
def _handle_aruco(self, image):
img = image
corners, ids, _ = Tracker.preprocess(img)
if np.all(ids is not None):
# check for OpenCV output in different versions
params = aruco.estimatePoseSingleMarkers(corners, 1, self.mtx, self.dist)
if len(params) == 2:
rvec, tvec = params
else:
rvec, tvec, _ = params
else:
return
# set all players to list from "Field"
self.setChangedListItems()
for i in range(len(ids)):
rvecs, tvecs, glyph_name = rvec[i], tvec[i], ids[i][0]
# build view matrix
rmtx = cv2.Rodrigues(rvecs)[0]
view_matrix = np.array([[rmtx[0][0], rmtx[0][1], rmtx[0][2], tvecs[0][0]],
[rmtx[1][0], rmtx[1][1], rmtx[1][2], tvecs[0][1]],
[rmtx[2][0], rmtx[2][1], rmtx[2][2], tvecs[0][2]],
[0.0, 0.0, 0.0, 1.0]])
view_matrix = view_matrix * self.INVERSE_MATRIX
view_matrix = np.transpose(view_matrix)
# load view matrix and draw shape
glPushMatrix()
glLoadMatrixd(view_matrix)
# check if ID is set or not and set it
if ids[i] not in self.set_ids:
for player in self.set_players:
if player.marker_num is None:
player.marker_num = ids[i]
self.set_ids.append(ids[i])
break
# if ID is set project model
for player in self.set_players:
if player.marker_num == ids[i]:
glCallList(player.model.gl_list)
glPopMatrix()
def _draw_background(self):
# draw background
glBegin(GL_QUADS)
glTexCoord2f(0.0, 1.0)
glVertex3f(-4.0, -3.0, 0.0)
glTexCoord2f(1.0, 1.0)
glVertex3f(4.0, -3.0, 0.0)
glTexCoord2f(1.0, 0.0)
glVertex3f(4.0, 3.0, 0.0)
glTexCoord2f(0.0, 0.0)
glVertex3f(-4.0, 3.0, 0.0)
glEnd()
class MUCamera:
def __init__(self):
self.MU = Webcam()
self.img_intensity = []
self.img_time = []
self.MU.start()
self.filtered_average = []
self.MU.register_callback(self._average_intensity, 1)
self.img = []
self.euclidean_dist = None
def _average_intensity(self, image):
'''
A function that does the actual calculations
:param image: the image is retrieved from the webcam.py file using the callback function
:return: the average intensity of the captured images in a list, the time of capture, the image objects
in a list
'''
self.img_intensity.append(np.mean(np.mean(image)))
self.img_time.append(time.time())
self.img.append(image)
return np.mean(np.mean(image))
def average_intensity(self):
'''
The function that should actually be called if you want to know the average intensity
:return: The average intensity of the image that was most recently retreived from the webcam
'''
while len(self.img_intensity) < 1:
pass
return self.img_intensity[-1]
def filtered_average_intensity(self):
'''
Function that takes in the average intensity list from _average_intensity() and passes the data through
a butterworth filter.
https://docs.scipy.org/doc/scipy-0.14.0/reference/generated/scipy.signal.butter.html
:return: A list of the filtered average intensities
'''
b, a = signal.butter(5, 0.025)
zi = signal.lfilter_zi(b, a)
z, _ = signal.lfilter(b,
a,
self.img_intensity,
zi=zi * self.img_intensity[0])
z2, _ = signal.lfilter(b, a, z, zi=zi * z[0])
self.filtered_average = signal.filtfilt(b, a, self.img_intensity)
def intensity_plot(self):
'''
Creates a plot of the raw average intensites and the filtered intensites
Raw is a solid black line, filtered is a dashed red line
:return: A plot
'''
t0 = self.img_time[0]
t = [(x - t0) / 60 for x in self.img_time]
y = self.img_intensity
y_filtered = self.filtered_average
plt.plot(t, y, 'k', t, y_filtered, 'r--')
plt.xlabel('Minutes')
plt.ylabel('Average Intensity')
plt.legend(
('Average Image Intensity', 'Smoothed Average Image Intensity'),
loc='best')
plt.title(
'Average Image Intensity from 5:45 PM to 8:30 AM PST, May 29th')
plt.grid()
plt.show()
def daytime(self, threshold=75):
'''
Determines whether it is night or day from retreived webcam image
:param threshold: the average intesntity value that is used to determine time of day
if calculated average intensity is less than the threshold, it is night, otherwise it is day
:return: True if daytime, false if nighttime
'''
while len(self.img_intensity) < 1:
pass
img = self.img[-1]
intensity = np.mean(np.mean(img))
if intensity < threshold:
return False
else:
return True
def common_color(self):
'''
Calculates the most common color in the retrieved webcam image
Uses statistics library
:return: color that occurs the most in a tuple (R,G,B)
'''
img = self.img[-1].getdata()
m = statistics.mode(img)
return m
def stop(self):
'''
Function that terminates the callback function that is started in __init__
Will call the filtering function and plotting function
:return:
'''
self.MU.stop()
self.filtered_average_intensity()
self.intensity_plot()
def motion(self):
'''
Determines whether or not motion took place between two images
Waits until 25 images have been retreived to make sure the two images that are being compared
are actually different. Webcam updates about once a minute so 25 images should be long enough
:return: True if motion occurred, false if motion did not occur
'''
while len(self.img) < 25:
pass
img1 = self.img[-25]
img2 = self.img[-1]
img3 = ImageChops.subtract(img1, img2)
self.euclidean_dist = mth.sqrt(np.sum(np.array(img3.getdata())**2))
if self.euclidean_dist > 8000:
return True
else:
return False
def highlight_motion(self):
'''
Creates an image that highlights the motion between 2 webcam images in red
aits until 25 images have been retreived to make sure the two images that are being compared
are actually different. Webcam updates about once a minute so 25 images should be long enough
:return: The second picture, but with the different pixels highlighted in red
'''
while len(self.img) < 25:
pass
img1 = self.img[-25]
img2 = self.img[-1]
img3 = ImageChops.subtract(img1, img2)
img2_data = np.asarray(img2)
img3_data = np.asarray(img3)
img2_data.setflags(write=1)
for i in range(len(img3_data[1, :])):
for j in range(len(img3_data[:, i])):
avg = np.mean(img3_data[j, i])
if avg > 35 and j > 250:
img2_data[j, i] = [255, 0, 0]
img_new = Image.fromarray(img2_data, 'RGB')
img_new.show()
def event(self):
'''
Determines if there is an event going on in the quad. Based on the color and euclidean distance of
two images in the quad
:return: True if there is an event, false if otherwise
Also displays the test cases.
:return The image with the grey square is the baseline from which everything is compared to. The grey was the most
common color in a cropped version of the size of the square
:return The image with the white square is the case where there is an event
'''
while len(self.img_intensity) < 1:
pass
pxl_coor = (250, 365, 500, 470)
img_grey_large = np.asarray(self.img[-1])
img_event = np.asarray(self.img[-1])
img = self.img[-1].crop(pxl_coor)
baseline = np.asarray(img)
baseline.setflags(write=1)
img_grey_large.setflags(write=1)
img_event.setflags(write=1)
for i in range(len(baseline[1, :])):
for j in range(len(baseline[:, i])):
baseline[j, i] = [170, 170, 168]
for i in range(249, 500):
for j in range(365, 470):
img_grey_large[j, i] = [170, 170, 168]
img_event[j, i] = [255, 255, 255]
img_grey = Image.fromarray(baseline, 'RGB')
img_grey_large = Image.fromarray(img_grey_large, 'RGB')
img_event = Image.fromarray(img_event, 'RGB')
img_compare = ImageChops.subtract(img, img_grey)
euclidean_dist = mth.sqrt(np.sum(np.array(img_compare.getdata())**2))
img_grey_large.show()
img_event.show()
if euclidean_dist > 8000:
return True
else:
return False
class MUcamera:
def __init__(self):
self.w = Webcam()
self.start = self.w.start()
self.im = self.w.grab_image()
self.w.register_callback(self.average_intensity, 1)
# self.grabimage = self.w.grab_image()
self.avg_intensity = []
self.images = []
self.filt_list = []
self.f, self.ax = plt.subplots(1, 1)
self.day = []
def average_intensity(self):
pix_val = list(self.im.getdata())
pixel_intensity = []
for x in pix_val:
avg = sum(x) / len(x)
pixel_intensity.append(avg)
self.avg_pixel = np.average(pixel_intensity)
self.avg_intensity.append(self.avg_pixel)
return self.avg_intensity
# avg = np.mean(np.mean(image,axis=1))
# self.avg_list.append(avg)
def average_intensity_filtered(self):
width = 3
# i=0
if len(self.avg_intensity) >= 5:
for x in range(len(self.avg_intensity) - 2):
self.filt_list.append(
(self.avg_intensity[x] + self.avg_intensity[x + 1] +
self.avg_intensity[x + 2]) / width)
return self.filt_list
else:
return self.filt_list
# while i+width <= len(self.filt_list):
# y = self.filt_list[i:i+width]
# total_sum=sum(y)/width
# self.filt_list.append(total_sum)
# i+=1
# def stop(self):
# self.w.stop()
# self.average_intensity_mean_plot()
# self.average_intensity_filtered_plot()
#
# def average_intensity_mean_plot(self):
# self.ax.plot(self.avg_intensity, 'C1')
# self.ax.set_xlabel('Image Number')
# self.ax.set_ylabel('Intensity')
# self.ax.set_title('Image Intensity')
##
# def average_intensity_filtered_plot(self):
# self.average_intensity_filtered()
# self.ax.plot(self.filt_list, 'C2')
#
def daytime(self):
self.average = np.mean(np.mean(self.im, axis=1))
if self.average >= 95:
# self.i.append(self.i)
return print("True")
else:
return print("False")
##
def most_common_color(self):
w, h = self.im.size
pixels = self.im.getcolors(w * h)
print(len(pixels))
most_frequent_pixel = pixels[0]
for count, color in pixels:
if count > most_frequent_pixel[0]:
most_frequent_pixel = (count, color)
# compare("Most Common", image, most_frequent_pixel[1])
# print(self.most_frequent_pixel)
return print(most_frequent_pixel[0] / len(pixels), most_frequent_pixel)
def stop(self):
self.w.stop()
self.daytime()
self.most_common_color()
self.average_intensity_mean_plot()
self.average_intensity_filtered_plot()
def average_intensity_mean_plot(self):
self.average_intensity()
self.ax.plot(self.avg_intensity, 'C1')
self.ax.set_xlabel('Image Number')
self.ax.set_ylabel('Intensity')
self.ax.set_title('Image Intensity')
#
def average_intensity_filtered_plot(self):
self.average_intensity_filtered()
self.ax.plot(self.filt_list, 'C2')
class HandTracker:
def __init__(self):
self.webcam = Webcam()
self.webcam.start()
self.detection = Detection()
self.x_axis = 0.0
self.z_axis = 0.0
self.show_cube = False
self.texture_background = None
self.texture_cube = None
def _init_gl(self, Width, Height):
glClearColor(0.0, 0.0, 0.0, 0.0)
glClearDepth(1.0)
glDepthFunc(GL_LESS)
glEnable(GL_DEPTH_TEST)
glShadeModel(GL_SMOOTH)
glMatrixMode(GL_PROJECTION)
glLoadIdentity()
gluPerspective(45.0, float(Width) / float(Height), 0.1, 100.0)
glMatrixMode(GL_MODELVIEW)
# enable texture
glEnable(GL_TEXTURE_2D)
self.texture_background = glGenTextures(1)
self.texture_cube = glGenTextures(1)
# create cube texture
image = Image.open("/home/annus/Pictures/image.jpeg")
ix = image.size[0]
iy = image.size[1]
image = image.tobytes("raw", "RGBX", 0, -1)
glBindTexture(GL_TEXTURE_2D, self.texture_cube)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST)
glTexImage2D(GL_TEXTURE_2D, 0, 3, ix, iy, 0, GL_RGBA, GL_UNSIGNED_BYTE,
image)
def _draw_scene(self):
# handle any hand gesture
self._handle_gesture()
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
glLoadIdentity()
# draw background
glBindTexture(GL_TEXTURE_2D, self.texture_background)
glPushMatrix()
glTranslatef(0.0, 0.0, -11.2)
self._draw_background()
glPopMatrix()
# draw cube if enabled
if self.show_cube:
glColor4f(1.0, 1.0, 1.0, 1.0)
glBlendFunc(GL_SRC_ALPHA, GL_ONE)
glEnable(GL_BLEND)
glDisable(GL_DEPTH_TEST)
glBindTexture(GL_TEXTURE_2D, self.texture_cube)
glPushMatrix()
glTranslatef(0.0, 0.0, -7.0)
glRotatef(self.x_axis, 1.0, 0.0, 0.0)
glRotatef(0.0, 0.0, 1.0, 0.0)
glRotatef(self.z_axis, 0.0, 0.0, 1.0)
self._draw_cube()
glPopMatrix()
glDisable(GL_BLEND)
glEnable(GL_DEPTH_TEST)
# update rotation values
self.x_axis = self.x_axis - 10
self.z_axis = self.z_axis - 10
glutSwapBuffers()
def _handle_gesture(self):
# get image from webcam
image = self.webcam.get_current_frame()
# detect hand gesture in image
is_okay = self.detection.is_item_detected_in_image(
'haarcascade_okaygesture.xml', image.copy())
is_vicky = self.detection.is_item_detected_in_image(
'haarcascade_vickygesture.xml', image.copy())
if is_okay:
# okay gesture shows cube
self.show_cube = True
elif is_vicky:
# vicky gesture hides cube
self.show_cube = False
# convert image to OpenGL texture format
image = cv2.flip(image, 0)
gl_image = Image.fromarray(image)
ix = gl_image.size[0]
iy = gl_image.size[1]
gl_image = gl_image.tostring("raw", "BGRX", 0, -1)
# create background texture
glBindTexture(GL_TEXTURE_2D, self.texture_background)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST)
glTexImage2D(GL_TEXTURE_2D, 0, 3, ix, iy, 0, GL_RGBA, GL_UNSIGNED_BYTE,
gl_image)
def _draw_background(self):
# draw background
glBegin(GL_QUADS)
glTexCoord2f(0.0, 1.0)
glVertex3f(-4.0, -3.0, 4.0)
glTexCoord2f(1.0, 1.0)
glVertex3f(4.0, -3.0, 4.0)
glTexCoord2f(1.0, 0.0)
glVertex3f(4.0, 3.0, 4.0)
glTexCoord2f(0.0, 0.0)
glVertex3f(-4.0, 3.0, 4.0)
glEnd()
def _draw_cube(self):
# draw cube
glBegin(GL_QUADS)
glTexCoord2f(0.0, 0.0)
glVertex3f(-1.0, -1.0, 1.0)
glTexCoord2f(1.0, 0.0)
glVertex3f(1.0, -1.0, 1.0)
glTexCoord2f(1.0, 1.0)
glVertex3f(1.0, 1.0, 1.0)
glTexCoord2f(0.0, 1.0)
glVertex3f(-1.0, 1.0, 1.0)
glTexCoord2f(1.0, 0.0)
glVertex3f(-1.0, -1.0, -1.0)
glTexCoord2f(1.0, 1.0)
glVertex3f(-1.0, 1.0, -1.0)
glTexCoord2f(0.0, 1.0)
glVertex3f(1.0, 1.0, -1.0)
glTexCoord2f(0.0, 0.0)
glVertex3f(1.0, -1.0, -1.0)
glTexCoord2f(0.0, 1.0)
glVertex3f(-1.0, 1.0, -1.0)
glTexCoord2f(0.0, 0.0)
glVertex3f(-1.0, 1.0, 1.0)
glTexCoord2f(1.0, 0.0)
glVertex3f(1.0, 1.0, 1.0)
glTexCoord2f(1.0, 1.0)
glVertex3f(1.0, 1.0, -1.0)
glTexCoord2f(1.0, 1.0)
glVertex3f(-1.0, -1.0, -1.0)
glTexCoord2f(0.0, 1.0)
glVertex3f(1.0, -1.0, -1.0)
glTexCoord2f(0.0, 0.0)
glVertex3f(1.0, -1.0, 1.0)
glTexCoord2f(1.0, 0.0)
glVertex3f(-1.0, -1.0, 1.0)
glTexCoord2f(1.0, 0.0)
glVertex3f(1.0, -1.0, -1.0)
glTexCoord2f(1.0, 1.0)
glVertex3f(1.0, 1.0, -1.0)
glTexCoord2f(0.0, 1.0)
glVertex3f(1.0, 1.0, 1.0)
glTexCoord2f(0.0, 0.0)
glVertex3f(1.0, -1.0, 1.0)
glTexCoord2f(0.0, 0.0)
glVertex3f(-1.0, -1.0, -1.0)
glTexCoord2f(1.0, 0.0)
glVertex3f(-1.0, -1.0, 1.0)
glTexCoord2f(1.0, 1.0)
glVertex3f(-1.0, 1.0, 1.0)
glTexCoord2f(0.0, 1.0)
glVertex3f(-1.0, 1.0, -1.0)
glEnd()
def main(self):
# setup and run OpenGL
glutInit()
glutInitDisplayMode(GLUT_RGBA | GLUT_DOUBLE | GLUT_DEPTH)
glutInitWindowSize(640, 480)
glutInitWindowPosition(800, 400)
glutCreateWindow("OpenGL Hand Tracker")
glutDisplayFunc(self._draw_scene)
glutIdleFunc(self._draw_scene)
self._init_gl(640, 480)
glutMainLoop()
class OpenGLGlyphs:
# constants
INVERSE_MATRIX = np.array([[ 1.0, 1.0, 1.0, 1.0],
[-1.0,-1.0,-1.0,-1.0],
[-1.0,-1.0,-1.0,-1.0],
[ 1.0, 1.0, 1.0, 1.0]])
def __init__(self):
# initialise webcam and start thread
self.webcam = Webcam()
self.webcam.start()
#initialise
self.hBox = causalBox(winSize = 10)
self.vBox = causalBox(winSize = 10)
plyerName = ["John","Doe","Tommy","Emmanuel"]
self.game = GameCtrler(plyerName)
# initialise shapes
self.dragon = None
self.fly = None
self.ele = None
self.boat = None
self.horse = None
self.house = None
self.juk = None
# textures
self.texture_background = None
def _init_gl(self, Width, Height):
# initialPosition = (0,0,0)
glClearColor(0.0, 0.0, 0.0, 0.0)
glClearDepth(1.0)
glDepthFunc(GL_LESS)
glEnable(GL_DEPTH_TEST)
glShadeModel(GL_SMOOTH)
# Projection matrix
glMatrixMode(GL_PROJECTION)
glLoadIdentity()
fovy = 2*np.arctan(Height/1375.0)*180.0/np.pi
gluPerspective(fovy, float(Width)/float(Height), 0.1, 1375.1)
glViewport(0,0,Width,Height)
glMatrixMode(GL_MODELVIEW)
# assign shapes
print "loading model 1/7"
self.dragon = OBJ('Drgn9-6.obj')
print "loading model 2/7"
self.fly = OBJ('plane.obj')
print "loading model 3/7"
self.ele = OBJ('minion.obj')
print "loading model 4/7"
self.boat = OBJ('VikingShip.mtl.obj')
print "loading model 5/7"
self.horse = OBJ('Wooden_Toy_Truck.obj')
print "loading model 6/7"
self.house = OBJ('house_001.obj')
print "loading model 7/7"
self.juk = OBJ('Barrel_variation.obj')
print "loading model done"
# enable textures
glEnable(GL_TEXTURE_2D)
self.texture_background = glGenTextures(1)
def _draw_scene(self):
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
glLoadIdentity()
# get image from webcam
image = self.webcam.get_current_frame()
self._draw_background(image)
# handle glyphs
image = self._handle_glyphs(image)
glutSwapBuffers()
def _handle_glyphs(self, image):
# attempt to detect glyphs
glyphs = []
try:
glyphs = detect_glyph(image, self.hBox, self.vBox, self.game)
except Exception as ex:
print(ex)
if not glyphs:
return
for glyph in glyphs:
rvecs, tvecs, glyph_name = glyph
# build view matrix
rmtx = cv2.Rodrigues(rvecs)[0]
view_matrix = np.array([[rmtx[0][0],rmtx[0][1],rmtx[0][2],tvecs[0]],
[rmtx[1][0],rmtx[1][1],rmtx[1][2],tvecs[1]],
[rmtx[2][0],rmtx[2][1],rmtx[2][2],tvecs[2]],
[0.0 ,0.0 ,0.0 ,1.0 ]])
view_matrix = view_matrix * self.INVERSE_MATRIX
view_matrix = np.transpose(view_matrix)
# load view matrix and draw cube
glPushMatrix()
glLoadIdentity()
glLoadMatrixd(view_matrix)
if glyph_name == "B juk":
glCallList(self.juk.gl_list)
elif glyph_name == "R juk":
glCallList(self.juk.gl_list)
elif glyph_name == "B Phao":
glCallList(self.house.gl_list)
elif glyph_name == "R Phao":
glCallList(self.house.gl_list)
elif glyph_name == "B Horse":
glCallList(self.horse.gl_list)
elif glyph_name == "R Horse":
glCallList(self.horse.gl_list)
elif glyph_name == "B Boat":
glCallList(self.boat.gl_list)
elif glyph_name == "R Boat":
glCallList(self.boat.gl_list)
elif glyph_name == "B Ele":
glCallList(self.ele.gl_list)
elif glyph_name == "R Ele":
glCallList(self.ele.gl_list)
elif glyph_name == "B Fly":
glCallList(self.fly.gl_list)
elif glyph_name == "R Fly":
glCallList(self.fly.gl_list)
elif glyph_name == "B T":
glCallList(self.dragon.gl_list)
elif glyph_name == "R T":
glCallList(self.dragon.gl_list)
else:
glCallList(self.dragon.gl_list)
glPopMatrix()
def _draw_background(self, image):
# convert image to OpenGL texture format
bg_image = cv2.flip(image, 0)
bg_image = Image.fromarray(bg_image)
ix = bg_image.size[0]
iy = bg_image.size[1]
bg_image = bg_image.tobytes("raw", "BGRX", 0, -1)
# create background texture
glBindTexture(GL_TEXTURE_2D, self.texture_background)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST)
glTexImage2D(GL_TEXTURE_2D, 0, 3, ix, iy, 0, GL_RGBA, GL_UNSIGNED_BYTE, bg_image)
# draw background
glBindTexture(GL_TEXTURE_2D, self.texture_background)
glPushMatrix()
glLoadIdentity()
glTranslatef(-100,100,-1375)
glBegin(GL_QUADS)
i, j = 1520/2, 820/2
glTexCoord2f(0.0, 1.0); glVertex3f(-i, -j, 0.0)
glTexCoord2f(1.0, 1.0); glVertex3f( i, -j, 0.0)
glTexCoord2f(1.0, 0.0); glVertex3f( i, j, 0.0)
glTexCoord2f(0.0, 0.0); glVertex3f(-i, j, 0.0)
glEnd()
glPopMatrix()
def main(self):
width = 1520
heigh = 820
# setup and run OpenGL
glutInit()
glutInitDisplayMode(GLUT_RGBA | GLUT_DOUBLE | GLUT_DEPTH)
glutInitWindowSize(760, 410)
glutInitWindowPosition(100, 100)
self.window_id = glutCreateWindow("OpenGL Glyphs")
glutDisplayFunc(self._draw_scene)
glutIdleFunc(self._draw_scene)
self._init_gl(width, heigh)
glutMainLoop()
class HandTracker:
def __init__(self):
self.webcam = Webcam()
self.webcam.start()
self.detection = Detection()
self.x_axis = 0.0
self.y_axis = 0.0
self.z_axis = 0.0
self.z_pos = -7.0
def _handle_gesture(self):
# get image from webcam
image = self.webcam.get_current_frame()
# detect hand gesture in image
is_okay = self.detection.is_item_detected_in_image(
'haarcascade_okaygesture.xml', image.copy())
is_vicky = self.detection.is_item_detected_in_image(
'haarcascade_vickygesture.xml', image.copy())
if is_okay:
# okay gesture moves cube towards us
self.z_pos = self.z_pos + 1.0
elif is_vicky:
# vicky gesture moves cube away from us
self.z_pos = self.z_pos - 1.0
def _draw_cube(self):
# draw cube
glBegin(GL_QUADS)
glTexCoord2f(0.0, 0.0)
glVertex3f(-1.0, -1.0, 1.0)
glTexCoord2f(1.0, 0.0)
glVertex3f(1.0, -1.0, 1.0)
glTexCoord2f(1.0, 1.0)
glVertex3f(1.0, 1.0, 1.0)
glTexCoord2f(0.0, 1.0)
glVertex3f(-1.0, 1.0, 1.0)
glTexCoord2f(1.0, 0.0)
glVertex3f(-1.0, -1.0, -1.0)
glTexCoord2f(1.0, 1.0)
glVertex3f(-1.0, 1.0, -1.0)
glTexCoord2f(0.0, 1.0)
glVertex3f(1.0, 1.0, -1.0)
glTexCoord2f(0.0, 0.0)
glVertex3f(1.0, -1.0, -1.0)
glTexCoord2f(0.0, 1.0)
glVertex3f(-1.0, 1.0, -1.0)
glTexCoord2f(0.0, 0.0)
glVertex3f(-1.0, 1.0, 1.0)
glTexCoord2f(1.0, 0.0)
glVertex3f(1.0, 1.0, 1.0)
glTexCoord2f(1.0, 1.0)
glVertex3f(1.0, 1.0, -1.0)
glTexCoord2f(1.0, 1.0)
glVertex3f(-1.0, -1.0, -1.0)
glTexCoord2f(0.0, 1.0)
glVertex3f(1.0, -1.0, -1.0)
glTexCoord2f(0.0, 0.0)
glVertex3f(1.0, -1.0, 1.0)
glTexCoord2f(1.0, 0.0)
glVertex3f(-1.0, -1.0, 1.0)
glTexCoord2f(1.0, 0.0)
glVertex3f(1.0, -1.0, -1.0)
glTexCoord2f(1.0, 1.0)
glVertex3f(1.0, 1.0, -1.0)
glTexCoord2f(0.0, 1.0)
glVertex3f(1.0, 1.0, 1.0)
glTexCoord2f(0.0, 0.0)
glVertex3f(1.0, -1.0, 1.0)
glTexCoord2f(0.0, 0.0)
glVertex3f(-1.0, -1.0, -1.0)
glTexCoord2f(1.0, 0.0)
glVertex3f(-1.0, -1.0, 1.0)
glTexCoord2f(1.0, 1.0)
glVertex3f(-1.0, 1.0, 1.0)
glTexCoord2f(0.0, 1.0)
glVertex3f(-1.0, 1.0, -1.0)
glEnd()
def _init_gl(self, Width, Height):
glClearColor(0.0, 0.0, 0.0, 0.0)
glClearDepth(1.0)
glDepthFunc(GL_LESS)
glEnable(GL_DEPTH_TEST)
glShadeModel(GL_SMOOTH)
glMatrixMode(GL_PROJECTION)
glLoadIdentity()
gluPerspective(45.0, float(Width) / float(Height), 0.1, 100.0)
glMatrixMode(GL_MODELVIEW)
# initialize lighting
glLightfv(GL_LIGHT0, GL_AMBIENT, (0.5, 0.5, 0.5, 1.0))
glLightfv(GL_LIGHT0, GL_DIFFUSE, (1.0, 0.8, 0.0, 1.0))
glEnable(GL_LIGHT0)
glEnable(GL_LIGHTING)
# initialize blending
glColor4f(0.2, 0.2, 0.2, 0.5)
glBlendFunc(GL_SRC_ALPHA, GL_ONE)
glEnable(GL_BLEND)
# initialize texture
image = open("devil.jpg")
ix = image.size[0]
iy = image.size[1]
image = image.tostring("raw", "RGBX", 0, -1)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST)
glTexImage2D(GL_TEXTURE_2D, 0, 3, ix, iy, 0, GL_RGBA, GL_UNSIGNED_BYTE,
image)
glEnable(GL_TEXTURE_2D)
def _draw_scene(self):
# handle any hand gesture
self._handle_gesture()
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
glLoadIdentity()
# position and rotate cube
glTranslatef(0.0, 0.0, self.z_pos)
glRotatef(self.x_axis, 1.0, 0.0, 0.0)
glRotatef(self.y_axis, 0.0, 1.0, 0.0)
glRotatef(self.z_axis, 0.0, 0.0, 1.0)
# position lighting
glLightfv(GL_LIGHT0, GL_POSITION, (0.0, 0.0, 2.0, 1.0))
# draw cube
self._draw_cube()
# update rotation values
self.x_axis = self.x_axis - 10
self.z_axis = self.z_axis - 10
glutSwapBuffers()
def main(self):
# setup and run OpenGL
glutInit()
glutInitDisplayMode(GLUT_RGBA | GLUT_DOUBLE | GLUT_DEPTH)
glutInitWindowSize(640, 480)
glutInitWindowPosition(800, 400)
glutCreateWindow("OpenGL Hand Tracker")
glutDisplayFunc(self._draw_scene)
glutIdleFunc(self._draw_scene)
self._init_gl(640, 480)
glutMainLoop()
class SaltwashAR:
# constants
INVERSE_MATRIX = np.array([[ 1.0, 1.0, 1.0, 1.0],
[-1.0,-1.0,-1.0,-1.0],
[-1.0,-1.0,-1.0,-1.0],
[ 1.0, 1.0, 1.0, 1.0]])
def __init__(self):
# initialise config
self.config_provider = ConfigProvider()
# initialise robots
self.rocky_robot = RockyRobot()
self.sporty_robot = SportyRobot()
# initialise webcam
self.webcam = Webcam()
# initialise markers
self.markers = Markers()
self.markers_cache = None
# initialise features
self.features = Features(self.config_provider)
# initialise texture
self.texture_background = None
def _init_gl(self):
glClearColor(0.0, 0.0, 0.0, 0.0)
glClearDepth(1.0)
glDepthFunc(GL_LESS)
glEnable(GL_DEPTH_TEST)
glShadeModel(GL_SMOOTH)
glMatrixMode(GL_PROJECTION)
glLoadIdentity()
gluPerspective(33.7, 1.3, 0.1, 100.0)
glMatrixMode(GL_MODELVIEW)
# load robots frames
self.rocky_robot.load_frames(self.config_provider.animation)
self.sporty_robot.load_frames(self.config_provider.animation)
# start webcam thread
self.webcam.start()
# assign texture
glEnable(GL_TEXTURE_2D)
self.texture_background = glGenTextures(1)
def _draw_scene(self):
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT)
glLoadIdentity()
# reset robots
self.rocky_robot.reset()
self.sporty_robot.reset()
# get image from webcam
image = self.webcam.get_current_frame()
# handle background
self._handle_background(image.copy())
# handle markers
self._handle_markers(image.copy())
# handle features
self.features.handle(self.rocky_robot, self.sporty_robot, image.copy())
glutSwapBuffers()
def _handle_background(self, image):
# let features update background image
image = self.features.update_background_image(image)
# convert image to OpenGL texture format
bg_image = cv2.flip(image, 0)
bg_image = Image.fromarray(bg_image)
ix = bg_image.size[0]
iy = bg_image.size[1]
bg_image = bg_image.tobytes('raw', 'BGRX', 0, -1)
# create background texture
glBindTexture(GL_TEXTURE_2D, self.texture_background)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST)
glTexImage2D(GL_TEXTURE_2D, 0, 3, ix, iy, 0, GL_RGBA, GL_UNSIGNED_BYTE, bg_image)
# draw background
glBindTexture(GL_TEXTURE_2D, self.texture_background)
glPushMatrix()
glTranslatef(0.0,0.0,-10.0)
glBegin(GL_QUADS)
glTexCoord2f(0.0, 1.0); glVertex3f(-4.0, -3.0, 0.0)
glTexCoord2f(1.0, 1.0); glVertex3f( 4.0, -3.0, 0.0)
glTexCoord2f(1.0, 0.0); glVertex3f( 4.0, 3.0, 0.0)
glTexCoord2f(0.0, 0.0); glVertex3f(-4.0, 3.0, 0.0)
glEnd( )
glPopMatrix()
def _handle_markers(self, image):
# attempt to detect markers
markers = []
try:
markers = self.markers.detect(image)
except Exception as ex:
print(ex)
# manage markers cache
if markers:
self.markers_cache = markers
elif self.markers_cache:
markers = self.markers_cache
self.markers_cache = None
else:
return
for marker in markers:
rvecs, tvecs, marker_rotation, marker_name = marker
# build view matrix
rmtx = cv2.Rodrigues(rvecs)[0]
view_matrix = np.array([[rmtx[0][0],rmtx[0][1],rmtx[0][2],tvecs[0]],
[rmtx[1][0],rmtx[1][1],rmtx[1][2],tvecs[1]],
[rmtx[2][0],rmtx[2][1],rmtx[2][2],tvecs[2]],
[0.0 ,0.0 ,0.0 ,1.0 ]])
view_matrix = view_matrix * self.INVERSE_MATRIX
view_matrix = np.transpose(view_matrix)
# load view matrix and draw cube
glPushMatrix()
glLoadMatrixd(view_matrix)
if marker_name == ROCKY_ROBOT:
self.rocky_robot.next_frame(marker_rotation, self.features.is_speaking(), self.features.get_emotion())
elif marker_name == SPORTY_ROBOT:
self.sporty_robot.next_frame(marker_rotation, self.features.is_speaking(), self.features.get_emotion())
glColor3f(1.0, 1.0, 1.0)
glPopMatrix()
def main(self):
# setup and run OpenGL
glutInit()
glutInitDisplayMode(GLUT_RGBA | GLUT_DOUBLE | GLUT_DEPTH)
glutInitWindowSize(640, 480)
glutInitWindowPosition(100, 100)
self.window_id = glutCreateWindow('SaltwashAR')
glutDisplayFunc(self._draw_scene)
glutIdleFunc(self._draw_scene)
self._init_gl()
glutMainLoop()
class AR_Project:
def __init__(self):
# sigint interrupt initialize
signal.signal(signal.SIGINT, self.signal_handler)
# initialize webcam
self.webcam = Webcam()
self.webcam.start()
self.x_axis = 0.0
self.y_axis = 0.0
self.z_axis = 0.0
self.z_pos = -7.0
self.win = 0
self.texture_background = None
self.texture_teapot = None
def signal_handler(self, signal, frame):
print('\nYou pressed Ctrl+C!')
self.webcam.close()
sys.exit()
def _get_background(self):
# get image from webcam
image = self.webcam.get_current_frame()
# convert image to OpenGL texture format
image = cv2.flip(image, 0)
image = cv2.flip(image, 1)
gl_image = Image.fromarray(image)
ix = gl_image.size[0]
iy = gl_image.size[1]
gl_image = gl_image.tobytes("raw", "BGRX", 0, -1)
# create background texture
glBindTexture(GL_TEXTURE_2D, self.texture_background)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST)
glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST)
glTexImage2D(GL_TEXTURE_2D, 0, 3, ix, iy, 0, GL_RGBA, GL_UNSIGNED_BYTE, gl_image)
def _draw_background(self):
# draw background
glBegin(GL_QUADS)
glTexCoord2f(0.0, 1.0); glVertex3f(-4.0, -3.0, 4.0)
glTexCoord2f(1.0, 1.0); glVertex3f( 4.0, -3.0, 4.0)
glTexCoord2f(1.0, 0.0); glVertex3f( 4.0, 3.0, 4.0)
glTexCoord2f(0.0, 0.0); glVertex3f(-4.0, 3.0, 4.0)
glEnd()
def _init_gl(self, Width, Height):
glClearColor(0.0, 0.0, 0.0, 0.0)
glClearDepth(1.0)
glDepthFunc(GL_LESS)
glEnable(GL_DEPTH_TEST)
glShadeModel(GL_SMOOTH)
glMatrixMode(GL_PROJECTION)
glLoadIdentity()
gluPerspective(45.0, float(Width)/float(Height)-.2, 0.1, 500.0)
glMatrixMode(GL_MODELVIEW)
# enable texture
glEnable(GL_TEXTURE_2D)
self.texture_background = glGenTextures(1)
# initialize lighting
#glLightfv(GL_LIGHT0, GL_AMBIENT, (0.5, 0.5, 0.5, 1.0))
#glLightfv(GL_LIGHT0, GL_DIFFUSE, (1.0, 0.8, 0.0, 1.0))
glEnable(GL_LIGHT0)
glEnable(GL_LIGHTING)
# initialize blending
glColor4f(0.2, 0.2, 0.2, 0.5)
glBlendFunc(GL_SRC_ALPHA, GL_ONE)
glEnable(GL_BLEND)
#glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST)
#glTexParameterf(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST)
#glTexImage2D(GL_TEXTURE_2D, 0, 3, ix, iy, 0, GL_RGBA, GL_UNSIGNED_BYTE, image)
#glEnable(GL_TEXTURE_2D)
def _draw_scene(self):
# handle any hand gesture
self._get_background()
glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
glLoadIdentity();
# draw background
#glBindTexture(GL_TEXTURE_2D, self.texture_background)
glPushMatrix()
glTranslatef(0.0,0.0,-11.2)
self._draw_background()
glPopMatrix()
# position teapot
glTranslatef(0.0,0.0,self.z_pos);
glRotatef(self.x_axis,1.0,0.0,0.0)
glRotatef(self.y_axis,0.0,1.0,0.0)
glRotatef(self.z_axis,0.0,0.0,1.0)
# draw teapot
glutSolidTeapot(1.2)
# rotate teapot
self.x_axis = self.x_axis - 2
self.z_axis = self.z_axis - 2
glutSwapBuffers()
def main(self):
# setup and run OpenGL
glutInit()
glutInitDisplayMode(GLUT_RGBA | GLUT_DOUBLE | GLUT_DEPTH)
glutInitWindowSize(640, 480)
glutInitWindowPosition(800, 400)
self.win = glutCreateWindow("COS 429 AR Project")
glutDisplayFunc(self._draw_scene)
glutIdleFunc(self._draw_scene)
self._init_gl(640, 480)
glutMainLoop()
