class World(DirectObject):
def __init__(self):
# Preliminary capabilities check.
if not base.win.getGsg().getSupportsBasicShaders():
self.t = addTitle(
"Shadow Demo: Video driver reports that shaders are not supported.")
return
if not base.win.getGsg().getSupportsDepthTexture():
self.t = addTitle(
"Shadow Demo: Video driver reports that depth textures are not supported.")
return
#SHADOW DETECTOR AND LANDMARK DETECTOR
self.shadowfunc = ShadowDetector()
self.lmfunc = LM_Detector()
#base.backfaceCullingOff()
base.setBackgroundColor(0, 0, 0, 1)
base.camLens.setNearFar(0.1, 10000)
base.camLens.setFov(60)
# examine the state space
self.width = 19
self.height = 9
self.state_size = (7,64,64)
print('Size of state:', self.state_size)
self.action_low = np.array([0,0,0,0,0])
print('Action low:', self.action_low)
self.action_high = np.array([self.width,self.height,self.width,self.height,3.14/2])
print('Action high: ', self.action_high)
#number of steps taken
self.step_count = 0
#initialize the scene
self.init_scene()
self.incrementCameraPosition(0)
self.addControls()
self.reset(manual_pose=1)
self.visorpos = self.getVisorData()
#ADD TASKS THAT RUN IF THE ENVIRONMENT IS IN A LOOP
taskMgr.doMethodLater(.1,self.viewReward,"drawReward")
taskMgr.doMethodLater(.1,self.spinLightTask,"spinLight")
##########################################################
#CONVENIENCE FUNCTIONS
##########################################################
def getInfo(self):
self.lpos = self.getLightData()
self.geompos = self.getVertexData()
def getVisorData(self):
data = [[None] * len(self.hexes[i]) for i in range(len(self.hexes))]
for i in range(len(self.hexes)):
for j in range(len(self.hexes[i])):
data[i][j] = self.hexes[i][j].getPos(self.render1)
return np.array(data)
def getLightData(self):
lpos = self.light.getPos(self.render1)
return np.array([lpos[0],lpos[1],lpos[2]])
def getVertexData(self):
self.dennis2.flattenLight()
def processPrim(prim,vertex,data):
for p in range(prim.getNumPrimitives()):
s = prim.getPrimitiveStart(p)
e = prim.getPrimitiveEnd(p)
for i in range(s,e):
vi = prim.getVertex(i)
vertex.setRow(vi)
v = vertex.getData3f()
data[vi] = [v[0],v[1],v[2],1]
geomNode = self.dennis2.getChild(0).getChild(0).node()
geoms = []
for i in range(geomNode.getNumGeoms()):
data = [None] * 100000
g = geomNode.getGeom(i)
s = geomNode.getGeomState(i)
m = s.getAttrib(MaterialAttrib).getMaterial()
vdata = GeomVertexReader(g.getAnimatedVertexData(True,Thread.getCurrentThread()),'vertex')
for j in range(g.getNumPrimitives()):
p = g.getPrimitive(j)
processPrim(p,vdata,data)
data = np.array(data[:data.index(None)])
#T = LMatrix4()
#a = geomNode.getTransform(render)
#print(a)
#quit()
#T = self.dennis2.getJoints(jointName='head')[-1].getTransform()
#T = np.array(T.getRows())
#data = np.matmul(data,T.transpose())
#print(data)
#print(T)
#quit()
geoms.append((data[:,:3],m))
return geoms
def calcVisor(self,task):
self.getInfo()
lvec = self.lpos
for geom,material in self.geompos:
color = material.getDiffuse()
if color[0] == 1 and color[1] == 0 and color[2] == 0: break
bl = self.visorpos[0,0]
br = self.visorpos[0,self.width-1]
tl = self.visorpos[self.height-1,0]
tr = self.visorpos[self.height-1,self.width-1]
v1 = tr - tl
v2 = bl - tl
#GET AFFINE TRANSFORMATION MATRIX FROM WORLD COORDINATE TO VISOR MASK
#https://stackoverflow.com/questions/22954239/given-three-points-compute-affine-transformation
#ins = np.stack((tl[1:],bl[1:],tr[1:]))
#out = np.array([[self.width-1,self.height-1],[self.width-1,0],[0,self.height-1]])
#out = np.array([[0,self.height-1],[self.width-1,self.height-1],[0,0]])
#out = np.array([[0,0],[self.width-1,0],[0,self.height-1]])
#l = len(ins)
#B = np.vstack([ins.transpose(),np.ones(l)])
#D = 1.0 / np.linalg.det(B)
#entry = lambda r,d: np.linalg.det(np.delete(np.vstack([r,B]),(d+1),axis=0))
#M = [[(-1)**i * D * entry(R,i) for i in range(l)] for R in np.transpose(out)]
#A, t = np.hsplit(np.array(M),[l-1])
#t = np.transpose(t)[0]
tl = np.hstack((tl,[1]))
tr = np.hstack((tr,[1]))
bl = np.hstack((bl,[1]))
br = np.hstack((br,[1]))
ins = np.stack((tl[1:],bl[1:],tr[1:]))
out = np.array([[self.width-1,self.height-1],[self.width-1,0],[0,self.height-1]])
A = np.matmul(np.linalg.inv(ins),out)
#get intersection of eye verticies with visor plane along the light ray
nvec = np.cross(v1,v2) #normal to the visor
d = np.dot(tl[:-1] + -1 * geom,nvec) / np.dot(lvec,nvec)
#d = np.dot(tl + -1 * geom,nvec) / np.dot(lvec,nvec)
intersection = geom + d[:,np.newaxis] * lvec
t1 = intersection[:,1:]
#(OPTIONAL VISUALIZATION)
img = np.zeros((500,500,3))
b1 = tl * 100 + 100
b2 = tr * 100 + 100
b3 = bl * 100 + 100
b4 = br * 100 + 100
vbox = np.array([b2[1:-1],b1[1:-1],b3[1:-1],b4[1:-1]])
#vbox = np.array([b2[1:],b1[1:],b3[1:],b4[1:]])
vbox = np.int0(vbox)
cv2.fillConvexPoly(img,vbox,(255,255,255))
for p in t1:
pnt = (p * 100 + 100).astype(np.uint16)
cv2.circle(img,(pnt[0],pnt[1]),1,(0,255,0),-1)
cv2.imshow('img',img)
cv2.waitKey(1)
#get bounding box of the intersection points in 2d space collapsing the x axis
pnts = t1[:,np.newaxis,:] * 100000 + 100000 #why 100? cuz opencv can't handle floats
rect = cv2.minAreaRect(pnts.astype(np.int32))
pnt,dim,r = rect
x,y = pnt
w,h = dim
box = cv2.boxPoints(rect) #tr,tl,bl,br
#TRANSFORM INTERSECTION WORLD COORD TO VISOR MASK
box = (box - 100000) / 100000
#box = np.matmul(A,box.transpose()).transpose() + t
box = np.hstack((box,np.ones((box.shape[0],1))))
box = np.matmul(box,A)
xpad = 2
ypad = 2
box = box[:,:2] + np.array([[xpad,-ypad],[-xpad,-ypad],[-xpad,ypad],[xpad,ypad]])
box = np.rint(box)
box[:,0] = -box[:,0] + self.width
box[:,0] -= 3
#box[:,1] += 4
box = np.int0(box)
self.visormask *= 0
cv2.fillConvexPoly(self.visormask,box,(1))
self.shadowon()
cv2.imshow('visormask',cv2.resize((self.visormask[:,::-1] * 255).astype(np.uint8), (self.width*10,self.height*10), interpolation = cv2.INTER_LINEAR))
cv2.waitKey(1)
return task.again
def drawReward(self,params,eye_mask,shadow,rgb,lm,IOU,EYE,reward):
h,w,d = rgb.shape
img = np.zeros((h,w+100,d))
rgb[eye_mask] = rgb[eye_mask] * [0,0,1]
rgb[shadow] = rgb[shadow] * [1,0,0]
IOU = np.sum(np.logical_and(eye_mask,shadow)) / np.sum(np.logical_or(eye_mask,shadow))
EYE = np.sum(np.logical_and(eye_mask,shadow)) / np.sum(eye_mask)
#show landmarks
for x,y in lm:
cv2.circle(rgb,(x,y),2,(255,0,255),-1)
#IMG IS 224 X 112
cv2.putText(img,"IOU %.3f" % IOU,(2,h-54),2,0.4,(0,255,0),1,cv2.LINE_AA)
cv2.putText(img,"EYE %.3f" % EYE,(2,h-34),2,0.4,(0,255,0),1,cv2.LINE_AA)
cv2.putText(img,"reward %.3f" % reward ,(2,h-14),2,0.4,(0,255,0),1,cv2.LINE_AA)
img[:,100:,:] = rgb
cv2.imshow('semantic mask',img)
cv2.waitKey(10)
#REWARD MAP GENERATION GIVEN STATE S
def genRewardGT2(self):
shadow_img = self.getFrame_notex()
noshadow = (self.noshadow_img * 255).astype(np.uint8)
shadow = (shadow_img * 255).astype(np.uint8)
eye_mask = self.lmfunc.get_eyesGT(noshadow) #((cx,cy),(h,w),rotation)
shadow_mask = self.shadowfunc.get_shadowgt(shadow)
lm = np.zeros((5,2)).astype(np.uint8)
IOU = np.sum(np.logical_and(eye_mask,shadow_mask)) / np.sum(np.logical_or(eye_mask,shadow_mask))
EYE = np.sum(np.logical_and(eye_mask,shadow_mask)) / np.sum(eye_mask)
threshold = IOU + EYE
#reward
if threshold > 1.2:
reward,flag = threshold, True
else:
reward,flag = -0.1, self.step_count == 10
#DRAW THE SEMANTIC MASKS "OPTIONAL"
self.drawReward(self.visorparam,eye_mask,shadow_mask,shadow,lm,IOU,EYE,reward)
return reward, flag, threshold
#REWARD MAP GENERATION GIVEN STATE S
def genRewardGT(self):
shadow_img = self.getFrame_notex()
noshadow = (self.noshadow_img * 255).astype(np.uint8)
shadow = (shadow_img * 255).astype(np.uint8)
eye_mask = self.lmfunc.get_eyesGT(noshadow) #((cx,cy),(h,w),rotation)
shadow_mask = self.shadowfunc.get_shadowgt(shadow)
eye_and_shadow = np.logical_and(eye_mask,shadow_mask)
eye_or_shadow = np.logical_or(eye_mask,shadow_mask)
lm = np.zeros((5,2)).astype(np.uint8)
IOU = np.sum(eye_and_shadow) / np.sum(eye_or_shadow)
EYE = np.sum(eye_and_shadow) / np.sum(eye_mask)
thresh = IOU
#DRAW THE SEMANTIC MASKS "OPTIONAL"
self.drawReward(self.visorparam,eye_mask,shadow_mask,shadow,lm,IOU,EYE,thresh)
return thresh,eye_mask,shadow_mask, eye_and_shadow
#GET THE CURRENT FRAME AS NUMPY ARRAY
def getFrame(self):
base.graphicsEngine.renderFrame()
dr = self.cam.node().getDisplayRegion(0)
tex = dr.getScreenshot()
data = tex.getRamImage()
v = memoryview(data).tolist()
img = np.array(v,dtype=np.uint8)
img = img.reshape((tex.getYSize(),tex.getXSize(),4))
img = img[:,:,:3]
img = img[::-1]
img = img[159:601,150:503,:] #boundry based on range of head motion
img = cv2.resize(img,(64,64))
return img / 255.0
#GET THE CURRENT FRAME AS NUMPY ARRAY
def getFrame_notex(self):
base.graphicsEngine.renderFrame()
dr = self.cam2.node().getDisplayRegion(0)
tex = dr.getScreenshot()
data = tex.getRamImage()
v = memoryview(data).tolist()
img = np.array(v,dtype=np.uint8)
img = img.reshape((tex.getYSize(),tex.getXSize(),4))
img = img[:,:,:3]
img = img[::-1]
img = img[159:601,150:503,:] #boundry based on range of head motion
img = cv2.resize(img,(64,64))
return img / 255.0
#RESET THE ENVIRONMENT
def reset(self,manual_pose=False):
self.step_count = 1
if manual_pose: poseid = manual_pose
else: poseid = random.randint(1,200)
self.dennis.pose('head_movement',poseid) #CHOOSE A RANDOM POSE
self.dennis2.pose('head_movement',poseid) #CHOOSE A RANDOM POSE
self.light_angle = 0
self.incLightPos(speed=0) #PUT LIGHT IN RANDOM POSITION
#get image without shadow
self.shadowoff()
self.noshadow_img = self.getFrame_notex()
#init the visor to the same position always
self.visorparam = [self.width//2,self.height//2,self.width//2,self.height//2,0] #x,y,w,h,r #INITIAL VISOR POSITION IS ALWAYS THE SAME
rot_rect = ((self.visorparam[0],self.visorparam[1]),(self.visorparam[2],self.visorparam[3]),self.visorparam[4])
box = cv2.boxPoints(rot_rect)
box = np.int0(box)
self.visormask *= 0
cv2.fillConvexPoly(self.visormask,box,(1))
#APPLY VISOR MASK
self.shadowon()
#get the initial state as two copies of the first image
base.graphicsEngine.renderFrame()
self.prv_frame = self.getFrame()
cur_frame = self.prv_frame
h,w = cur_frame.shape[:2]
frame = np.zeros((h,w,6))
frame[:,:,:3] = self.prv_frame
frame[:,:,3:] = cur_frame
state = frame
return state
#RESET THE ENVIRONMENT
def reset2_4(self,manual_pose=False):
self.step_count = 1
if manual_pose: poseid = manual_pose
else: poseid = random.randint(1,200)
self.dennis.pose('head_movement',poseid) #CHOOSE A RANDOM POSE
self.dennis2.pose('head_movement',poseid) #CHOOSE A RANDOM POSE
#self.light_angle = random.randint(-135,135)
self.incLightPos(speed=2) #PUT LIGHT IN RANDOM POSITION
#get current situation
_,eye_mask,shadow_mask, eye_and_shadow = self.genRewardGT()
EYE = np.sum(np.logical_and(eye_mask,shadow_mask)) / np.sum(eye_mask)
if EYE <= 0.5:
self.visorparam = [self.width //2, self.height //2, 19,9,0]
rot_rect = ((self.visorparam[0],self.visorparam[1]),(self.visorparam[2],self.visorparam[3]),self.visorparam[4])
box = cv2.boxPoints(rot_rect)
box = np.int0(box)
self.visormask *= 0
cv2.fillConvexPoly(self.visormask,box,(1))
#get image without shadow
self.shadowoff()
self.noshadow_img = self.getFrame_notex()
#APPLY VISOR MASK
self.shadowon()
#get the initial state as two copies of the first image
base.graphicsEngine.renderFrame()
frame = self.getFrame()
self.imgstates = deque(maxlen=2)
mask = np.dstack((eye_mask,shadow_mask)).astype(np.float32)
for i in range(2): self.imgstates.append(frame.copy().astype(np.float32))
frame = self.getstate3(mask)
return np.array(self.visorparam) / 19.0, frame
#INITIALIZE THE 3D ENVIRONMENT
def init_scene(self):
#GENERATE SECOND WINDOW TO SHOW MATERIAL REGIONS
self.win2 = base.openWindow()
displayRegion = self.win2.makeDisplayRegion()
self.cam2 = NodePath(Camera('cam'))
displayRegion.setCamera(self.cam2)
self.render2 = NodePath('render2')
self.cam2.reparentTo(self.render2)
self.cam2.node().getLens().setNearFar(0.1, 10000)
self.cam2.node().getLens().setFov(60)
self.cam2.setPos(-3.8,0.0,2.25)
self.cam2.lookAt(-3,-0.2,2.69)
self.dennis2 = Actor('../assets/dennis.egg',{"head_movement": "../assets/dennis-head_movement.egg"})
self.dennis2.reparentTo(self.render2)
self.dennis2.setPlayRate(0.5,'head_movement')
self.dennis2.loop("head_movement")
#SET MAIN DISPLAY
display = base.win.makeDisplayRegion()
self.cam = NodePath(Camera('cam1'))
display.setCamera(self.cam)
self.render1 = NodePath('render1')
self.cam.reparentTo(self.render1)
self.cam.node().getLens().setNearFar(0.1, 10000)
self.cam.node().getLens().setFov(60)
self.cam.setPos(-10.8,0.0,5.25)
self.cam.lookAt(-3,-0.2,2.69)
# Load the scene
floorTex = loader.loadTexture('maps/envir-ground.jpg')
cm = CardMaker('')
cm.setFrame(-2, 2, -2, 2)
floor = self.render1.attachNewNode(PandaNode("floor"))
for y in range(12):
for x in range(12):
nn = floor.attachNewNode(cm.generate())
nn.setP(-90)
nn.setPos((x - 6) * 4, (y - 6) * 4, 0)
floor.setTexture(floorTex)
floor.flattenStrong()
self.car = loader.loadModel("../assets/cartex.egg")
self.car.reparentTo(self.render1)
self.car.set_two_sided(True) #BEST IF I CAN SOLVE THE BIAS PROBLEM ON THE SHADER
self.dennis = Actor('../assets/dennistex.egg',{"head_movement": "../assets/dennistex-head_movement.egg"})
self.dennis.reparentTo(self.render1)
self.dennis.setPlayRate(0.5,'head_movement')
self.dennis.loop("head_movement")
#CURRENTLY SHADOW QUALITY IS REDUCED DUE TO SHADOW ACNE
self.visor, self.hexes = self.genVisor2()
self.visor.reparentTo(self.dennis)
self.visor.set_two_sided(True)
self.visor.setPos(-3.75,.5,2.5)
self.visor.setH(-90)
self.visor.setP(90)
self.visor.setScale(0.027,0.027,0.027)
self.sun = DirectionalLight("Dlight")
self.sun.color = self.sun.color * 5
self.light = self.render1.attachNewNode(self.sun)
self.light.node().setScene(self.render1)
self.light.node().setShadowCaster(True)
self.light.node().setCameraMask(BitMask32.bit(1))
self.light.node().showFrustum()
self.light.node().getLens().set_film_size(20)
self.light.node().getLens().setFov(20)
self.light.node().getLens().setNearFar(10, 50)
self.render1.setLight(self.light)
self.render2.setLight(self.light)
self.alight = render.attachNewNode(AmbientLight("Ambient"))
self.alight.node().setColor(LVector4(0.2, 0.2, 0.2, 1))
self.render1.setLight(self.alight)
self.render2.setLight(self.alight)
#Important! Enable the shader generator.
self.render1.setShaderAuto()
self.render2.setShaderAuto()
self.render1.show(BitMask32.bit(1))
self.render2.show(BitMask32.bit(1))
# default values
self.light_angle = 0.0
self.car_x = 0.0
self.cameraSelection = 0
self.lightSelection = 0
self.visorMode = 0
self.visormask = np.zeros((self.height,self.width))
self.visorparam = [self.width//2,self.height//2,self.width//3,self.height//3,0] #x,y,w,h,r
self.light.node().hideFrustum()
##########################################################
#Controls
##########################################################
#SOME CONTROL SEQUENCES
def addControls(self):
#self.inst_p = addInstructions(0.06, 'up/down arrow: zoom in/out')
#self.inst_x = addInstructions(0.12, 'Left/Right Arrow : switch camera angles')
#addInstructions(0.18, 'a : put sun on face')
#addInstructions(0.24, 's : toggleSun')
#addInstructions(0.30, 'd : toggle visor')
#addInstructions(0.36, 'v: View the Depth-Texture results')
#addInstructions(0.42, 'tab : view buffer')
self.anim_flag = False
self.accept('escape', self.exit)
#self.accept("a", self.putSunOnFace)
self.accept("d", self.toggleVisor,[1])
self.accept("r", self.recordScreen)
self.accept("arrow_left", self.incrementCameraPosition, [-1])
self.accept("arrow_right", self.incrementCameraPosition, [1])
self.accept("f12",base.screenshot,['recording/snapshot'])
self.accept("tab", base.bufferViewer.toggleEnable)
self.accept("p", self.addAnimation)
def exit(self):
quit()
def addAnimation(self):
self.anim_flag = not self.anim_flag
if self.anim_flag:
self.dennis.setPlayRate(0.5,'head_movement')
self.dennis.loop('head_movement')
self.dennis2.setPlayRate(0.5,'head_movement')
self.dennis2.loop('head_movement')
else:
self.dennis.stop('head_movement')
self.dennis2.stop('head_movement')
def putSunOnFace(self):
self.lightSelection = 0
self.light.setPos(-15,0,3)
self.light.lookAt(0,0,0)
def toggleVisor(self,n):
self.visorMode = (self.visorMode + n) % 3
if (self.visorMode == 0):
taskMgr.remove('random controller')
if (self.visorMode == 1):
taskMgr.remove("random controller")
taskMgr.doMethodLater(0.1,self.calcVisor,"calcVisor")
for row in self.hexes:
for h in row:
h.show()
render.show()
if (self.visorMode == 2):
taskMgr.remove('calcVisor')
taskMgr.doMethodLater(0.5,self.randomVisor,'random controller')
def recordScreen(self):
base.movie(namePrefix='recording/frame',duration=5,fps=15)
##########################################################
#TASKS
##########################################################
def viewReward(self,task):
self.genRewardGT()
return Task.again
def randomVisor(self,task):
for _ in range(300):
i = random.randint(0,len(self.hexes)-1)
j = random.randint(0,len(self.hexes[-1]) - 1)
self.hexes[i][j].hide()
for _ in range(300):
i = random.randint(0,len(self.hexes)-1)
j = random.randint(0,len(self.hexes[-1]) - 1)
self.hexes[i][j].show()
render.show()
return Task.again
def shadowoff(self):
#APPLY VISOR MASK
for i in range(len(self.hexes)):
for j in range(len(self.hexes[-1])):
self.hexes[i][j].hide()
def shadowon(self):
#APPLY VISOR MASK
for i in range(len(self.hexes)):
for j in range(len(self.hexes[-1])):
if self.visormask[i,j] == 1:
self.hexes[i][j].show()
else:
self.hexes[i][j].hide()
def getstate(self,prv_frame,cur_frame):
h,w = cur_frame.shape[:2]
d = 6
frame = np.zeros((h,w,d))
frame[:,:,:3] = prv_frame
frame[:,:,3:] = cur_frame
state = frame
return state
#get state
def getstate2(self,mask):
h,w = self.imgstates[0].shape[:2]
d = len(self.imgstates) * 3 + 3
frame = np.zeros((h,w,d))
frame[:,:,:3] = self.imgstates[0]
frame[:,:,3:6] = self.imgstates[1]
frame[:,:,6:9] = mask
return frame
def getstate3(self,mask):
h,w = self.imgstates[0].shape[:2]
d = len(self.imgstates) * 3 + 2
frame = np.zeros((h,w,d))
frame[:,:,:3] = self.imgstates[0]
frame[:,:,3:6] = self.imgstates[1]
frame[:,:,6:8] = mask
return frame
#take a possible of 10 actions to move x,y,w,h,r up or down
#and update the visor mask accordingly
def step2_4(self,actions,speed=1):
self.step_count += 1
visor = np.array(self.visorparam)
visor = visor / 19.0
for i,a in enumerate(actions):
if i == 4:
if a == 0: self.visorparam[4] += 5 * pi / 180
elif a == 1: self.visorparam[4] -= 5 * pi / 180
elif a == 0: self.visorparam[i] += speed
elif a == 1: self.visorparam[i] -= speed
#get image with shadow after action
self.visorparam[0] = min(max(0,self.visorparam[0]),self.width-1)
self.visorparam[1] = min(max(0,self.visorparam[1]),self.height-1)
self.visorparam[2] = min(max(0,self.visorparam[2]),self.width-1)
self.visorparam[3] = min(max(0,self.visorparam[3]),self.height-1)
self.visorparam[4] = min(max(-pi,self.visorparam[4]),pi)
rot_rect = ((self.visorparam[0],self.visorparam[1]),(self.visorparam[2],self.visorparam[3]),self.visorparam[4]) #PADDED HEIGHT AND WIDTH OF 15PIXELS
box = cv2.boxPoints(rot_rect)
box = np.int0(box)
self.visormask *= 0
cv2.fillConvexPoly(self.visormask,box,(1))
self.shadowon()
cur_frame = self.getFrame()
cv2.imshow('visormask',cv2.resize((self.visormask[:,::-1]* 255).astype(np.uint8), (self.width*10,self.height*10), interpolation = cv2.INTER_LINEAR))
cv2.waitKey(1)
#get next state
reward,eye_mask,shadow_mask, eye_and_shadow = self.genRewardGT()
mask = np.dstack((eye_mask,shadow_mask))
self.imgstates.append(cur_frame.astype(np.float32))
#get the reward
EYE = np.sum(np.logical_and(eye_mask,shadow_mask)) / np.sum(eye_mask)
done = self.step_count >= 10 or reward > 0.25 or self.visorparam[2] <= 2 or self.visorparam[3] <= 2 or EYE < 0.5
if self.visorparam[2] <= 2 or self.visorparam[3] <=2 or EYE < 0.5:
r1 = -1
r2 = -1
elif reward < 0.25:
r1 = reward - 1
r2 = -0.1
else:
r1 = reward - 1
r2 = reward + 1
#set the next state
next_state = self.getstate3(mask.astype(np.float32))
return visor,next_state,r1,r2,done
def spinLightTask(self,task):
angleDegrees = (self.light_angle - 80)
angleRadians = angleDegrees * (pi / 180.0)
self.light.setPos(-15.0,2+3.0 * cos(angleRadians),2.2 + 0.1 * cos(angleRadians * 4.0))
self.light.lookAt(0,0,0)
self.light_angle += 5
return task.again
def incLightPos(self,speed=2):
angleRadians = self.light_angle * (pi / 180.0)
self.light.setPos(-15.0,2 + 3.0 * cos(angleRadians),2.2 + 0.1 * cos(angleRadians * 4.0))
self.light.lookAt(0,0,0)
self.light_angle += speed
def incrementCameraPosition(self,n):
self.cameraSelection = (self.cameraSelection + n) % 3
if (self.cameraSelection == 1):
self.cam.setPos(-7,0,3)
self.cam.lookAt(0,0,0)
if (self.cameraSelection == 0):
#self.camLens.setNearFar(0,10)
self.cam.setPos(-3.8,0.0,2.25)
self.cam.lookAt(-3,-0.2,2.69)
if (self.cameraSelection == 2):
self.cam.setPos(self.light.getPos())
self.cam.lookAt(0,0,0)
def genVisor2(self):
visor = self.render1.attach_new_node("visor")
objects = [[None] * self.width for i in range(self.height)]
r = 1.0
y = 0.866025
x = 0.5
offsetx = (1.55) / 2.00
offsety = (sqrt(1 - (offsetx * offsetx)) + 1) / 2.00
#x,y = 0,0
for i in range(0,self.height):
for j in range(0,self.width):
cx = offsety + (2*y*j) + (y*(i%2))
cy = offsetx + ((r + x) * i)
objects[i][j] = loader.loadModel("../assets/hex/hexagon.egg")
objects[i][j].reparentTo(visor)
objects[i][j].setPos(cx, cy,5)
objects[i][j].setScale(1.0,1.0,1.0)
objects[i][j].setAlphaScale(0.01)
return visor,objects
class World(DirectObject):
def __init__(self):
# Preliminary capabilities check.
if not base.win.getGsg().getSupportsBasicShaders():
self.t = addTitle(
"Shadow Demo: Video driver reports that shaders are not supported."
)
return
if not base.win.getGsg().getSupportsDepthTexture():
self.t = addTitle(
"Shadow Demo: Video driver reports that depth textures are not supported."
)
return
#SHADOW DETECTOR AND LANDMARK DETECTOR
self.shadowfunc = ShadowDetector()
self.lmfunc = LM_Detector()
#base.backfaceCullingOff()
base.setBackgroundColor(0, 0, 0, 1)
base.camLens.setNearFar(0.1, 10000)
base.camLens.setFov(60)
# examine the state space
self.width = 19
self.height = 9
self.state_size = (7, 64, 64)
print('Size of state:', self.state_size)
self.action_low = np.array([0, 0, 0, 0, 0])
print('Action low:', self.action_low)
self.action_high = np.array(
[self.width, self.height, self.width, self.height, 3.14 / 2])
print('Action high: ', self.action_high)
#initialize the scene
self.init_scene()
#number of steps taken
self.step_count = 0
self.light_noise = 0.00
self.geom_noise = 0.00
self.light_angle = 0
self.visorparam = [
self.width // 2, self.height // 2, self.width, self.height, 0
] #x,y,w,h,r
self.shadowoff()
self.noshadow_img = self.getFrame_notex()
self.incrementCameraPosition(0)
self.addControls()
self.reset(manual_pose=1)
self.visorpos = self.getVisorData()
self.visorpos[:, :, 1] -= 0.15
self.visorpos[:, :, 2] -= 0.05
#ADD TASKS THAT RUN IF THE ENVIRONMENT IS IN A LOOP
taskMgr.doMethodLater(.1, self.viewReward, "drawReward")
taskMgr.doMethodLater(.1, self.spinLightTask, "spinLight")
##########################################################
#CONVENIENCE FUNCTIONS
##########################################################
def getInfo(self):
self.lpos = self.getLightData()
self.geompos = self.getVertexData()
#GET 3D VISOR COORDINATES IN WORLD COORDINATE SPACE
def getVisorData(self):
data = [[None] * len(self.hexes[i]) for i in range(len(self.hexes))]
for i in range(len(self.hexes)):
for j in range(len(self.hexes[i])):
data[i][j] = self.hexes[i][j].getPos(self.render1)
return np.array(data)
#GET 3D LIGHT COORDINATES IN WORLD COORDINATE SPACE
def getLightData(self):
lpos = self.light.getPos(self.render1)
return np.array([lpos[0], lpos[1], lpos[2]])
#GET 3D EYE REGION COORDINATES IN WORLD COORDINATE SPACE BASED ON COLOR
def getVertexData(self):
self.dennis2.flattenLight()
def processPrim(prim, vertex, data):
for p in range(prim.getNumPrimitives()):
s = prim.getPrimitiveStart(p)
e = prim.getPrimitiveEnd(p)
for i in range(s, e):
vi = prim.getVertex(i)
vertex.setRow(vi)
v = vertex.getData3f()
data[vi] = [v[0], v[1], v[2], 1]
geomNode = self.dennis2.getChild(0).getChild(0).node()
geoms = []
for i in range(geomNode.getNumGeoms()):
data = [None] * 100000
g = geomNode.getGeom(i)
s = geomNode.getGeomState(i)
m = s.getAttrib(MaterialAttrib).getMaterial()
vdata = GeomVertexReader(
g.getAnimatedVertexData(True, Thread.getCurrentThread()),
'vertex')
for j in range(g.getNumPrimitives()):
p = g.getPrimitive(j)
processPrim(p, vdata, data)
data = np.array(data[:data.index(None)])
geoms.append((data[:, :3], m))
return geoms
#TASK FOR CALCULATING VISOR ACTIVATION MAP BASED ON CLOSED FORM SOLUTION
def calcVisorTask(self, task):
self.getInfo()
lvec = self.lpos + np.random.normal(
0, self.light_noise, 3) - np.random.normal(0, self.light_noise, 3)
for geom, material in self.geompos:
color = material.getDiffuse()
if color[0] == 1 and color[1] == 0 and color[2] == 0: break
geom += np.random.normal(0, self.geom_noise, geom.shape)
bl = self.visorpos[0, 0]
br = self.visorpos[0, self.width - 1]
tl = self.visorpos[self.height - 1, 0]
tr = self.visorpos[self.height - 1, self.width - 1]
v1 = tr - tl
v2 = bl - tl
#GET AFFINE TRANSFORMATION MATRIX FROM WORLD COORDINATE TO VISOR MASK
#https://stackoverflow.com/questions/22954239/given-three-points-compute-affine-transformation
tl = np.hstack((tl, [1]))
tr = np.hstack((tr, [1]))
bl = np.hstack((bl, [1]))
br = np.hstack((br, [1]))
ins = np.stack((tl[1:], bl[1:], tr[1:]))
out = np.array([[self.width - 1, self.height - 1], [self.width - 1, 0],
[0, self.height - 1]])
A = np.matmul(np.linalg.inv(ins), out)
#get intersection of eye verticies with visor plane along the light ray
nvec = np.cross(v1, v2) #normal to the visor
d = np.dot(tl[:-1] + -1 * geom, nvec) / np.dot(lvec, nvec)
intersection = geom + d[:, np.newaxis] * lvec
t1 = intersection[:, 1:]
#get bounding box of the intersection points in 2d space collapsing the x axis
pnts = t1[:, np.
newaxis, :] * 100000 + 100000 #why scale? cuz opencv can't handle floats
rect = cv2.minAreaRect(pnts.astype(np.int32))
pnt, dim, r = rect
x, y = pnt
w, h = dim
box = cv2.boxPoints(rect) #br,bl,tl,tr
box = (box - 100000) / 100000
#get which visor hexagons activated
self.visormask *= 0
p1, p2, p3, p4 = box
m1 = (p1[1] - p2[1]) / (p1[0] - p2[0])
m2 = (p2[1] - p3[1]) / (p2[0] - p3[0])
m3 = (p3[1] - p4[1]) / (p3[0] - p4[0])
m4 = (p4[1] - p1[1]) / (p4[0] - p1[0])
l1 = lambda x, y: y <= m1 * (x - p1[0]) + p1[1] + 0.04
l2 = lambda x, y: y >= m2 * (x - p2[0] + 0.06) + p2[1]
l3 = lambda x, y: y >= m3 * (x - p3[0]) + p3[1] - 0.07
l4 = lambda x, y: y <= m4 * (x - p4[0] - 0.06) + p4[1]
for i in range(self.height):
for j in range(self.width):
hexagon = self.visorpos[i, j][1:]
c1 = l1(hexagon[0], hexagon[1])
c2 = l2(hexagon[0], hexagon[1])
c3 = l3(hexagon[0], hexagon[1])
c4 = l4(hexagon[0], hexagon[1])
if c1 and c2 and c3 and c4:
self.visormask[i, j] = 1
self.shadowon()
cv2.imshow(
'visormask',
cv2.resize((self.visormask[:, ::-1] * 255).astype(np.uint8),
(self.width * 10, self.height * 10),
interpolation=cv2.INTER_LINEAR))
cv2.waitKey(1)
return task.again
#MANUAL CALCULATION FOR THE VISOR ACTIVATION
def calcVisor(self, light_noise=0.0, geom_noise=0.0):
self.getInfo()
lvec = self.lpos + np.random.normal(0, light_noise, 3)
'''
lpos_norm = self.lpos / np.linalg.norm(self.lpos)
for light_noise in [0.2,0.4,0.6,0.8,1.0,1.2,1.4,1.6,1.8]:
lvec_norm = (self.lpos + light_noise)/ np.linalg.norm(self.lpos + light_noise)
angle = np.arccos(np.dot(lpos_norm,lvec_norm))
angle = (angle * 180) / 3.14159
print(angle)
'''
for geom, material in self.geompos:
color = material.getDiffuse()
if color[0] == 1 and color[1] == 0 and color[2] == 0: break
geom += np.random.normal(0, geom_noise, geom.shape)
bl = self.visorpos[0, 0]
br = self.visorpos[0, self.width - 1]
tl = self.visorpos[self.height - 1, 0]
tr = self.visorpos[self.height - 1, self.width - 1]
v1 = tr - tl
v2 = bl - tl
#GET AFFINE TRANSFORMATION MATRIX FROM WORLD COORDINATE TO VISOR MASK
#https://stackoverflow.com/questions/22954239/given-three-points-compute-affine-transformation
tl = np.hstack((tl, [1]))
tr = np.hstack((tr, [1]))
bl = np.hstack((bl, [1]))
br = np.hstack((br, [1]))
ins = np.stack((tl[1:], bl[1:], tr[1:]))
out = np.array([[self.width - 1, self.height - 1], [self.width - 1, 0],
[0, self.height - 1]])
A = np.matmul(np.linalg.inv(ins), out)
#get intersection of eye verticies with visor plane along the light ray
nvec = np.cross(v1, v2) #normal to the visor
d = np.dot(tl[:-1] + -1 * geom, nvec) / np.dot(lvec, nvec)
intersection = geom + d[:, np.newaxis] * lvec
t1 = intersection[:, 1:]
#get bounding box of the intersection points in 2d space collapsing the x axis
pnts = t1[:, np.
newaxis, :] * 100000 + 100000 #why scale? cuz opencv can't handle floats
rect = cv2.minAreaRect(pnts.astype(np.int32))
pnt, dim, r = rect
x, y = pnt
w, h = dim
box = cv2.boxPoints(rect) #br,bl,tl,tr
box = (box - 100000) / 100000
#get which visor hexagons activated
self.visormask *= 0
p1, p2, p3, p4 = box
m1 = (p1[1] - p2[1]) / (p1[0] - p2[0])
m2 = (p2[1] - p3[1]) / (p2[0] - p3[0])
m3 = (p3[1] - p4[1]) / (p3[0] - p4[0])
m4 = (p4[1] - p1[1]) / (p4[0] - p1[0])
l1 = lambda x, y: y <= m1 * (x - p1[0]) + p1[1] + 0.04
l2 = lambda x, y: y >= m2 * (x - p2[0] + 0.06) + p2[1]
l3 = lambda x, y: y >= m3 * (x - p3[0]) + p3[1] - 0.07
l4 = lambda x, y: y <= m4 * (x - p4[0] - 0.06) + p4[1]
for i in range(self.height):
for j in range(self.width):
hexagon = self.visorpos[i, j][1:]
c1 = l1(hexagon[0], hexagon[1])
c2 = l2(hexagon[0], hexagon[1])
c3 = l3(hexagon[0], hexagon[1])
c4 = l4(hexagon[0], hexagon[1])
if c1 and c2 and c3 and c4:
self.visormask[i, j] = 1
self.shadowon()
base.graphicsEngine.renderFrame()
cv2.imshow(
'visormask',
cv2.resize((self.visormask[:, ::-1] * 255).astype(np.uint8),
(self.width * 10, self.height * 10),
interpolation=cv2.INTER_LINEAR))
cv2.waitKey(1)
reward, eye_mask, shadow_mask = self.genRewardGT()
EYE = np.sum(np.logical_and(eye_mask, shadow_mask)) / np.sum(eye_mask)
done = self.step_count >= 10 or reward > 0.25 or self.visorparam[
2] <= 2 or self.visorparam[3] <= 2 or EYE < 0.5
if reward >= 0.25:
reward = 1 + reward
elif reward < 0.25:
reward = 0
if self.visorparam[2] <= 2 or self.visorparam[3] <= 2 or EYE < 0.5:
reward = -1
return reward, done
#VISUALIZE REWARD FUNCTION
def drawReward(self, params, eye_mask, shadow, rgb, lm, IOU, EYE, reward):
h, w, d = rgb.shape
img = np.zeros((h, w + 100, d))
rgb[eye_mask] = rgb[eye_mask] * [0, 0, 1]
rgb[shadow] = rgb[shadow] * [1, 0, 0]
IOU = np.sum(np.logical_and(eye_mask, shadow)) / np.sum(
np.logical_or(eye_mask, shadow))
EYE = np.sum(np.logical_and(eye_mask, shadow)) / np.sum(eye_mask)
#show landmarks
for x, y in lm:
cv2.circle(rgb, (x, y), 2, (255, 0, 255), -1)
#IMG IS 224 X 112
cv2.putText(img, "IOU %.3f" % IOU, (2, h - 54), 2, 0.4, (0, 255, 0), 1,
cv2.LINE_AA)
cv2.putText(img, "EYE %.3f" % EYE, (2, h - 34), 2, 0.4, (0, 255, 0), 1,
cv2.LINE_AA)
cv2.putText(img, "reward %.3f" % reward, (2, h - 14), 2, 0.4,
(0, 255, 0), 1, cv2.LINE_AA)
img[:, 100:, :] = rgb
cv2.imshow('semantic mask', img)
cv2.waitKey(10)
#REWARD MAP GENERATION GIVEN STATE S
def genRewardGT(self):
shadow_img = self.getFrame_notex()
noshadow = (self.noshadow_img * 255).astype(np.uint8)
shadow = (shadow_img * 255).astype(np.uint8)
eye_mask = self.lmfunc.get_eyesGT(noshadow) #((cx,cy),(h,w),rotation)
shadow_mask = self.shadowfunc.get_shadowgt(shadow)
lm = np.zeros((5, 2)).astype(np.uint8)
IOU = np.sum(np.logical_and(eye_mask, shadow_mask)) / np.sum(
np.logical_or(eye_mask, shadow_mask))
EYE = np.sum(np.logical_and(eye_mask, shadow_mask)) / np.sum(eye_mask)
thresh = IOU
#DRAW THE SEMANTIC MASKS "OPTIONAL"
self.drawReward(self.visorparam, eye_mask, shadow_mask, shadow, lm,
IOU, EYE, thresh)
return thresh, eye_mask, shadow_mask
#GET THE CURRENT FRAME AS NUMPY ARRAY
def getFrame(self):
base.graphicsEngine.renderFrame()
dr = self.cam.node().getDisplayRegion(0)
tex = dr.getScreenshot()
data = tex.getRamImage()
v = memoryview(data).tolist()
img = np.array(v, dtype=np.uint8)
img = img.reshape((tex.getYSize(), tex.getXSize(), 4))
img = img[:, :, :3]
img = img[::-1]
img = img[159:601, 150:503, :] #boundry based on range of head motion
img = cv2.resize(img, (64, 64))
return img / 255.0
#GET THE CURRENT FRAME AS NUMPY ARRAY
def getFrame_notex(self):
base.graphicsEngine.renderFrame()
dr = self.cam2.node().getDisplayRegion(0)
tex = dr.getScreenshot()
data = tex.getRamImage()
v = memoryview(data).tolist()
img = np.array(v, dtype=np.uint8)
img = img.reshape((tex.getYSize(), tex.getXSize(), 4))
img = img[:, :, :3]
img = img[::-1]
img = img[159:601, 150:503, :] #boundry based on range of head motion
img = cv2.resize(img, (64, 64))
return img / 255.0
#RESET THE ENVIRONMENT
def reset(self, manual_pose=False):
self.step_count = 1
if manual_pose: poseid = manual_pose
else: poseid = random.randint(1, 200)
self.dennis.pose('head_movement', poseid) #CHOOSE A RANDOM POSE
self.dennis2.pose('head_movement', poseid) #CHOOSE A RANDOM POSE
self.incLightPos(speed=2) #PUT LIGHT IN RANDOM POSITION
#get current situation
_, eye_mask, shadow_mask = self.genRewardGT()
EYE = np.sum(np.logical_and(eye_mask, shadow_mask)) / np.sum(eye_mask)
if EYE <= 0.5:
self.visorparam = [self.width // 2, self.height // 2, 19, 9, 0]
#get image without shadow
self.shadowoff()
self.noshadow_img = self.getFrame_notex()
#APPLY VISOR MASK
self.shadowon()
#get the initial state as two copies of the first image
base.graphicsEngine.renderFrame()
self.prv_frame = self.getFrame()
cur_frame = self.prv_frame
h, w = cur_frame.shape[:2]
frame = np.zeros((h, w, 8))
frame[:, :, :3] = self.prv_frame
frame[:, :, 3:6] = cur_frame
frame[:, :, 6] = eye_mask.astype(np.float32)
frame[:, :, 7] = shadow_mask.astype(np.float32)
state = frame
return np.array(self.visorparam) / 19.0, state
#INITIALIZE THE 3D ENVIRONMENT
def init_scene(self):
#GENERATE SECOND WINDOW TO SHOW MATERIAL REGIONS
self.win2 = base.openWindow()
displayRegion = self.win2.makeDisplayRegion()
self.cam2 = NodePath(Camera('cam'))
displayRegion.setCamera(self.cam2)
self.render2 = NodePath('render2')
self.cam2.reparentTo(self.render2)
self.cam2.node().getLens().setNearFar(0.1, 10000)
self.cam2.node().getLens().setFov(60)
self.cam2.setPos(-3.8, 0.0, 2.25)
self.cam2.lookAt(-3, -0.2, 2.69)
self.dennis2 = Actor(
'../assets/dennis.egg',
{"head_movement": "../assets/dennis-head_movement.egg"})
self.dennis2.reparentTo(self.render2)
self.dennis2.setPlayRate(0.5, 'head_movement')
self.dennis2.loop("head_movement")
#SET MAIN DISPLAY
display = base.win.makeDisplayRegion()
self.cam = NodePath(Camera('cam1'))
display.setCamera(self.cam)
self.render1 = NodePath('render1')
self.cam.reparentTo(self.render1)
self.cam.node().getLens().setNearFar(0.1, 10000)
self.cam.node().getLens().setFov(60)
self.cam.setPos(-10.8, 0.0, 5.25)
self.cam.lookAt(-3, -0.2, 2.69)
# Load the scene
floorTex = loader.loadTexture('maps/envir-ground.jpg')
cm = CardMaker('')
cm.setFrame(-2, 2, -2, 2)
floor = self.render1.attachNewNode(PandaNode("floor"))
for y in range(12):
for x in range(12):
nn = floor.attachNewNode(cm.generate())
nn.setP(-90)
nn.setPos((x - 6) * 4, (y - 6) * 4, 0)
floor.setTexture(floorTex)
floor.flattenStrong()
self.car = loader.loadModel("../assets/cartex.egg")
self.car.reparentTo(self.render1)
self.car.set_two_sided(
True) #BEST IF I CAN SOLVE THE BIAS PROBLEM ON THE SHADER
self.dennis = Actor(
'../assets/dennistex.egg',
{"head_movement": "../assets/dennistex-head_movement.egg"})
self.dennis.reparentTo(self.render1)
self.dennis.setPlayRate(0.5, 'head_movement')
self.dennis.loop("head_movement")
#CURRENTLY SHADOW QUALITY IS REDUCED DUE TO SHADOW ACNE
self.visor, self.hexes = self.genVisor2()
self.visor.reparentTo(self.dennis)
self.visor.set_two_sided(True)
self.visor.setPos(-3.75, .5, 2.5)
self.visor.setH(-90)
self.visor.setP(90)
self.visor.setScale(0.027, 0.027, 0.027)
self.sun = DirectionalLight("Dlight")
self.sun.color = self.sun.color * 5
self.light = self.render1.attachNewNode(self.sun)
self.light.node().setScene(self.render1)
self.light.node().setShadowCaster(True)
self.light.node().setCameraMask(BitMask32.bit(1))
self.light.node().showFrustum()
self.light.node().getLens().set_film_size(20)
self.light.node().getLens().setFov(20)
self.light.node().getLens().setNearFar(10, 50)
self.render1.setLight(self.light)
self.render2.setLight(self.light)
self.alight = render.attachNewNode(AmbientLight("Ambient"))
self.alight.node().setColor(LVector4(0.2, 0.2, 0.2, 1))
self.render1.setLight(self.alight)
self.render2.setLight(self.alight)
#Important! Enable the shader generator.
self.render1.setShaderAuto()
self.render2.setShaderAuto()
self.render1.show(BitMask32.bit(1))
self.render2.show(BitMask32.bit(1))
# default values
self.light_angle = 0.0
self.car_x = 0.0
self.cameraSelection = 0
self.lightSelection = 0
self.visorMode = 0
self.visormask = np.zeros((self.height, self.width))
self.visorparam = [
self.width // 2, self.height // 2, self.width // 3,
self.height // 3, 0
] #x,y,w,h,r
self.light.node().hideFrustum()
##########################################################
#Controls
##########################################################
#SOME CONTROL SEQUENCES
def addControls(self):
self.info1 = TextNode('info')
self.info1.setText("Light Noise: " + str(self.light_noise))
self.info2 = TextNode('info2')
self.info2.setText("Geom Noise: " + str(self.geom_noise))
textNodePath1 = aspect2d.attachNewNode(self.info1)
textNodePath2 = aspect2d.attachNewNode(self.info2)
textNodePath1.setScale(0.07)
textNodePath1.setPos(0.6, 0, 0.9)
textNodePath2.setScale(0.07)
textNodePath2.setPos(0.6, 0, 0.8)
#self.inst_p = addInstructions(0.06, 'up/down arrow: zoom in/out')
#self.inst_x = addInstructions(0.12, 'Left/Right Arrow : switch camera angles')
addInstructions(0.18, 'a : put sun on face')
addInstructions(0.24, 's : toggleSun')
addInstructions(0.30, 'd : toggle visor mode')
addInstructions(0.36, 'p: toggle face pose')
addInstructions(0.42, '1: inc light noise up')
addInstructions(0.48, '2: inc light noise down')
self.anim_flag = False
self.accept('escape', self.exit)
self.accept("a", self.putSunOnFace)
self.accept("d", self.toggleVisor, [1])
self.accept("r", self.recordScreen)
self.accept("arrow_left", self.incrementCameraPosition, [-1])
self.accept("arrow_right", self.incrementCameraPosition, [1])
self.accept("f12", base.screenshot, ['recording/snapshot'])
self.accept("tab", base.bufferViewer.toggleEnable)
self.accept("p", self.addAnimation)
self.accept("1", self.incLightNoise, [0.1])
self.accept("2", self.incLightNoise, [-0.1])
self.accept("3", self.incGeomNoise, [0.001])
self.accept("4", self.incGeomNoise, [-0.001])
self.accept("r", self.resetNoise)
def exit(self):
quit()
def addAnimation(self):
self.anim_flag = not self.anim_flag
if self.anim_flag:
self.dennis.setPlayRate(0.5, 'head_movement')
self.dennis.loop('head_movement')
self.dennis2.setPlayRate(0.5, 'head_movement')
self.dennis2.loop('head_movement')
else:
self.dennis.stop('head_movement')
self.dennis2.stop('head_movement')
def putSunOnFace(self):
self.lightSelection = 0
self.light.setPos(-15, 0, 3)
self.light.lookAt(0, 0, 0)
def toggleVisor(self, n):
self.visorMode = (self.visorMode + n) % 3
if (self.visorMode == 0):
taskMgr.remove('random controller')
if (self.visorMode == 1):
taskMgr.remove("random controller")
taskMgr.doMethodLater(0.1, self.calcVisorTask, "calcVisorTask")
for row in self.hexes:
for h in row:
h.show()
render.show()
if (self.visorMode == 2):
taskMgr.remove('calcVisorTask')
taskMgr.doMethodLater(0.5, self.randomVisor, 'random controller')
def recordScreen(self):
base.movie(namePrefix='recording/frame', duration=5, fps=15)
##########################################################
#TASKS
##########################################################
def viewReward(self, task):
self.genRewardGT()
return Task.again
def randomVisor(self, task):
for _ in range(300):
i = random.randint(0, len(self.hexes) - 1)
j = random.randint(0, len(self.hexes[-1]) - 1)
self.hexes[i][j].hide()
for _ in range(300):
i = random.randint(0, len(self.hexes) - 1)
j = random.randint(0, len(self.hexes[-1]) - 1)
self.hexes[i][j].show()
render.show()
return Task.again
def shadowoff(self):
#APPLY VISOR MASK
for i in range(len(self.hexes)):
for j in range(len(self.hexes[-1])):
self.hexes[i][j].hide()
def shadowon(self):
#APPLY VISOR MASK
for i in range(len(self.hexes)):
for j in range(len(self.hexes[-1])):
if self.visormask[i, j] == 1:
self.hexes[i][j].show()
else:
self.hexes[i][j].hide()
def getstate(self, prv_frame, cur_frame):
h, w = cur_frame.shape[:2]
d = 6
frame = np.zeros((h, w, d))
frame[:, :, :3] = prv_frame
frame[:, :, 3:] = cur_frame
state = frame
return state
#get state
def getstate2(self, prv_frame, cur_frame, eye_mask, shadow_mask):
h, w = cur_frame.shape[:2]
d = 8
frame = np.zeros((h, w, d))
frame[:, :, :3] = prv_frame
frame[:, :, 3:6] = cur_frame
frame[:, :, 6] = eye_mask.astype(np.float32)
frame[:, :, 7] = shadow_mask.astype(np.float32)
state = frame
return state
#take a possible of 10 actions to move x,y,w,h,r up or down
#and update the visor mask accordingly
def step2_4(self, actions, speed=1):
self.step_count += 1
visor = np.array(self.visorparam)
visor = visor / 19.0
rewardpre, _, _ = self.genRewardGT()
for i, a in enumerate(actions):
if i == 4:
if a == 0: self.visorparam[4] += 5 * pi / 180
elif a == 1: self.visorparam[4] -= 5 * pi / 180
elif a == 0: self.visorparam[i] += speed
elif a == 1: self.visorparam[i] -= speed
#get image with shadow after action
self.visorparam[0] = min(max(0, self.visorparam[0]), self.width - 1)
self.visorparam[1] = min(max(0, self.visorparam[1]), self.height - 1)
self.visorparam[2] = min(max(0, self.visorparam[2]), self.width - 1)
self.visorparam[3] = min(max(0, self.visorparam[3]), self.height - 1)
self.visorparam[4] = min(max(-pi, self.visorparam[4]), pi)
rot_rect = ((self.visorparam[0], self.visorparam[1]),
(self.visorparam[2], self.visorparam[3]),
self.visorparam[4]) #PADDED HEIGHT AND WIDTH OF 15PIXELS
box = cv2.boxPoints(rot_rect)
box = np.int0(box)
self.visormask *= 0
cv2.fillConvexPoly(self.visormask, box, (1))
self.shadowon()
cur_frame = self.getFrame()
#cv2.imshow('prv',(self.prv_frame * 255).astype(np.uint8))
#cv2.imshow('cur',(cur_frame*255).astype(np.uint8))
cv2.imshow(
'visormask',
cv2.resize((self.visormask[:, ::-1] * 255).astype(np.uint8),
(self.width * 10, self.height * 10),
interpolation=cv2.INTER_LINEAR))
cv2.waitKey(1)
#get next state and reward
reward, eye_mask, shadow_mask = self.genRewardGT()
EYE = np.sum(np.logical_and(eye_mask, shadow_mask)) / np.sum(eye_mask)
#done = (self.step_count >= 10) or (reward == 0.0)
done = self.step_count >= 10 or reward > 0.25 or self.visorparam[
2] <= 2 or self.visorparam[3] <= 2 or EYE < 0.5
if reward > 0.25:
reward = 1 + 2 * reward
if self.visorparam[2] <= 2 or self.visorparam[3] <= 2 or EYE < 0.5:
reward = -1
#set the next state
next_state = self.getstate2(self.prv_frame, cur_frame, eye_mask,
shadow_mask)
self.prv_frame = cur_frame.copy()
return visor, next_state, reward, done
def spinLightTask(self, task):
angleRadians = self.light_angle * (pi / 180.0)
self.light.setPos(-15.0, 2 + 3.0 * cos(angleRadians),
2.20 + 0.1 * cos(angleRadians * 4.0))
self.light.lookAt(0, 0, 0)
self.light_angle += 2
self.light.node().showFrustum()
return task.again
###########################################################################################################
#MANUAL CONTROL FUNCTIONS
###########################################################################################################
def incLightPos(self, speed=2):
angleRadians = self.light_angle * (pi / 180.0)
self.light.setPos(-15.0, 2 + 3.0 * cos(angleRadians),
2.20 + 0.1 * cos(angleRadians * 4.0))
self.light.lookAt(0, 0, 0)
self.light_angle += speed
def incrementCameraPosition(self, n):
self.cameraSelection = (self.cameraSelection + n) % 3
if (self.cameraSelection == 1):
self.cam.setPos(-7, 0, 3)
self.cam.lookAt(0, 0, 0)
if (self.cameraSelection == 0):
#self.camLens.setNearFar(0,10)
self.cam.setPos(-3.8, 0.0, 2.25)
self.cam.lookAt(-3, -0.2, 2.69)
if (self.cameraSelection == 2):
self.cam.setPos(-20, 0, 7)
self.cam.lookAt(0, 0, 0)
#INCREMENT NOISE VALUE
def incLightNoise(self, n):
self.light_noise += n
self.info1.setText("Light Noise: " + str(self.light_noise))
#INCREMENT NOISE VALUE
def incGeomNoise(self, n):
self.geom_noise += n
self.geom_noise = max(0.00, self.geom_noise)
self.info2.setText("Geom Noise: " + str(self.geom_noise))
#RESET NOISE
def resetNoise(self):
self.light_noise = 0.00
self.geom_noise = 0.00
self.info1.setText("Light Noise: " + str(self.light_noise))
self.info2.setText("Geom Noise: " + str(self.geom_noise))
def genVisor2(self):
visor = self.render1.attach_new_node("visor")
objects = [[None] * self.width for i in range(self.height)]
r = 1.0
y = 0.866025
x = 0.5
offsetx = (1.55) / 2.00
offsety = (sqrt(1 - (offsetx * offsetx)) + 1) / 2.00
for i in range(0, self.height):
for j in range(0, self.width):
cx = offsety + (2 * y * j) + (y * (i % 2))
cy = offsetx + ((r + x) * i)
objects[i][j] = loader.loadModel("../assets/hex/hexagon.egg")
objects[i][j].reparentTo(visor)
objects[i][j].setPos(cx, cy, 5)
objects[i][j].setScale(1.0, 1.0, 1.0)
objects[i][j].setAlphaScale(0.01)
return visor, objects