def test_GaborFilters(self):
ilog = None # pv.ImageLog(name="GaborTest1")
#bank = FilterBank(tile_size=(128,128))
kernels = createGaborKernels()
filters = GaborFilters(kernels)
gim = filters.convolve(self.test_images[0])
template = gim.extractJet(pv.Point(64,64))
table = pv.Table()
for i in range(0,128):
table[i,'disp'] = i - 64
novel = gim.extractJet(pv.Point(i,64))
table[i,'simMag'] = template.simMag(novel)
table[i,'simPhase'] = template.simPhase(novel)
table[i,'simDisplace'] = template.simDisplace(novel)
dx,dy = template.displace(novel,method="Simple")
table[i,'displace_dx'] = dx
table[i,'displace_dy'] = dy
dx,dy = template.displace(novel,method="Blocked")
table[i,'blocked_dx'] = dx
table[i,'blocked_dy'] = dy
dx,dy = template.displace(novel,method="Iter")
table[i,'iter_dx'] = dx
table[i,'iter_dy'] = dy
def __init__(self,
face_detector=CascadeDetector(),
tile_size=(128, 128),
subtile_size=(32, 32),
left_center=pv.Point(39.325481787836871, 50.756936769089975),
right_center=pv.Point(91.461135538006289, 50.845357457309881),
validate=None,
n_iter=1,
annotate=False,
**kwargs):
'''
Create an eye locator. This default implentation uses a
cascade classifier for face detection and then SVR for eye
location.
'''
#TODO: Learn the mean eye locations durring training.
self.face_detector = face_detector
self.left_center = left_center
self.right_center = right_center
self.tile_size = tile_size
self.subtile_size = subtile_size
self.validate = validate
self.n_iter = n_iter
self.annotate = annotate
self.perturbations = True
# Number of training images where the face detection did not work.
self.detection_failures = 0
# point locators that learn to find the eyes.
self.createLocators(**kwargs)
def _selectTrackingPoints(self,frame):
'''
This uses the OpenCV get good features to track to initialize a set of tracking points_b.
'''
quality = 0.01
min_distance = 15
w,h = self.tile_size
tw = w//self.grid
th = h//self.grid
for i in range(self.grid):
for j in range(self.grid):
ul = pv.Point(i*tw,j*th)
rect = pv.Rect(i*tw,j*th,tw,th)
count = 0
for pt in self.tracks:
if rect.containsPoint(pt):
count += 1
if count < self.min_points:
gray = cv.CreateImage ((tw,th), 8, 1)
faceim = cv.GetSubRect(frame, rect.asOpenCV())
cv.Resize(faceim,gray)
eig = cv.CreateImage ((tw,th), 32, 1)
temp = cv.CreateImage ((tw,th), 32, 1)
# search the good points_b
points_b = cv.GoodFeaturesToTrack (gray, eig, temp, 2*self.min_points, quality, min_distance, None, 3, 0, 0.04)
for pt in points_b:
self.tracks.append(ul+pv.Point(pt))
def readEyeData(self):
f = open(self.metadata_path,'rb')
xml = et.parse(f)
self.metadata = {}
for recording in xml.findall('Recording'):
md = FRGCMetadata()
md.rec_id = recording.get('recording_id')
md.sub_id = recording.get('subject_id')
md.capture_date = recording.get('capturedate')
md.left_eye = None
md.right_eye = None
md.nose = None
md.mouth = None
for leye_center in recording.findall('LeftEyeCenter'):
x,y = float(leye_center.get('x')),float(leye_center.get('y'))
md.left_eye = pv.Point(x,y)
for leye_center in recording.findall('RightEyeCenter'):
x,y = float(leye_center.get('x')),float(leye_center.get('y'))
md.right_eye = pv.Point(x,y)
for leye_center in recording.findall('Nose'):
x,y = float(leye_center.get('x')),float(leye_center.get('y'))
md.nose = pv.Point(x,y)
for leye_center in recording.findall('Mouth'):
x,y = float(leye_center.get('x')),float(leye_center.get('y'))
md.mouth = pv.Point(x,y)
self.metadata[md.rec_id] = md
def _onNewFrame(self, img, fn, key=None, buffer=None, prevModule=None):
pt = pv.Point(10, 10)
img.annotateLabel(label="Frame: %d" % (fn + 1),
point=pt,
color="white")
img.annotateLabel(label="Key: %s" % key,
point=pv.Point(10, 20),
color="white")
if self._windowName != None: img.show(window=self._windowName)
def locatePoint(self,jet,start_pt=None,method="Simple"):
'''
If start_pt == None perform a grid search with a spacing of one half
the longest Gabor wavelength. Otherwize start at start_pt and follow
the Jacobian to the local minimum.
@param jet: the an example jet.
@param start_pt The point to start the search from.
'''
if start_pt == None:
# Compute the gate to use in the search
kx = self.k[:,0]
ky = self.k[:,1]
kt = np.sqrt(kx*kx + ky*ky).min()
gate = int(round(0.5*np.pi/kt))
# search for best similarity
best_sim = -1.0
best_pt = pv.Point(0,0)
w,h,c = self.data.shape[:]
for y in range(gate,h,gate):
for x in range(gate,w,gate):
pt = pv.Point(x,y)
novel = self.extractJet(pt,subpixel=False)
sim = novel.simDisplace(jet)
if sim > best_sim:
best_sim = sim
best_pt = pt
start_pt = best_pt
pt = start_pt
#print pt
novel = self.extractJet(pt,subpixel=False)
d = novel.displace(jet,method=method)
pt = pv.Point(novel.x + d[0], novel.y+d[1])
#print pt
novel = self.extractJet(pt,subpixel=False)
d = novel.displace(jet,method=method)
pt = pv.Point(novel.x + d[0], novel.y+d[1])
#print pt
novel = self.extractJet(pt,subpixel=False)
d = novel.displace(jet,method=method)
pt = pv.Point(novel.x + d[0], novel.y+d[1])
#print pt
novel = self.extractJet(pt,subpixel=False)
sim = novel.simPhase(jet)
#pt = pv.Point(novel.x + d[0], novel.y+d[1])
#print pt
return pt,sim,novel
def reset(self):
'''
Clear the points and start over.
'''
self.im = self.im.copy()
self.im.annotateLabel(pv.Point(10,10), "Click anywhere in the image to select a point.",color='yellow')
self.im.annotateLabel(pv.Point(10,20), "Press the 'r' to reset.",color='yellow')
self.im.annotateLabel(pv.Point(10,30), "Press the space bar when finished.",color='yellow')
self.points = []
def _readEyesFile(self):
'''
Private: Do not call directly. Reads the eye file.
'''
if self.filename[-4:] == '.csv':
f = open(self.filename, 'r')
for line in f:
#print line
line = line.split(',')
if len(line) < 5:
continue
for i in range(1, len(line), 4):
fname = self._parseName(line[0])
if len(line) < i + 4:
print(
"Warning in %s image %s: Count of numbers is not a multiple of four."
% (self.filename, fname))
break
eye1 = pv.Point(float(line[i + 0]), float(line[i + 1]))
eye2 = pv.Point(float(line[i + 2]), float(line[i + 3]))
truth_rect = pv.BoundingRect(eye1, eye2)
truth_rect.w = 2.0 * truth_rect.w
truth_rect.h = truth_rect.w
truth_rect.x = truth_rect.x - 0.25 * truth_rect.w
truth_rect.y = truth_rect.y - 0.3 * truth_rect.w
#print fname,eye1,eye2,truth_rect
if fname not in self.images:
self.images[fname] = []
self.images[fname].append([fname, eye1, eye2, truth_rect])
else:
f = open(self.filename, 'r')
for line in f:
#print line,
line = line.split()
fname = self._parseName(line[0])
eye1 = pv.Point(float(line[1]), float(line[2]))
eye2 = pv.Point(float(line[3]), float(line[4]))
truth_rect = pv.BoundingRect(eye1, eye2)
truth_rect.w = 2.0 * truth_rect.w
truth_rect.h = truth_rect.w
truth_rect.x = truth_rect.x - 0.25 * truth_rect.w
truth_rect.y = truth_rect.y - 0.3 * truth_rect.w
#print fname,eye1,eye2,truth_rect
if fname not in self.images:
self.images[fname] = []
self.images[fname].append([fname, eye1, eye2, truth_rect])
def asCorners(self):
'''
Returns the four corners. Can be used to transform this rect
through an affine or perspective transformation.
'''
x, y, w, h = self.asTuple()
pt1 = pv.Point(x, y)
pt2 = pv.Point(x + w, y)
pt3 = pv.Point(x + w, y + h)
pt4 = pv.Point(x, y + h)
return [pt1, pt2, pt3, pt4]
def test_translate(self):
transform = pv.AffineTranslate(10., 15., (640, 480))
_ = transform.transformImage(self.test_image)
#im_a.show()
pt = transform.transformPoint(pv.Point(320, 240))
self.assertAlmostEqual(pt.X(), 330.)
self.assertAlmostEqual(pt.Y(), 255.)
pt = transform.invertPoint(pv.Point(320, 240))
self.assertAlmostEqual(pt.X(), 310.)
self.assertAlmostEqual(pt.Y(), 225.)
def __init__(self,image_path, image_ext=".jpg", coord_file=None):
''' Create an object that manages a FERET face database. '''
self.image_path = image_path
self.image_ext = image_ext
coord_name = os.path.join(pv.__path__[0], 'analysis', 'FaceAnalysis', 'data', 'pie_illum_c27.txt')
pie_months =['oct_2000-nov_2000','nov_2000-dec_2000']
#pie_pose = [27,5,29,9,7]
#pie_illum = [19,20,21,5,6,11,12,10,7,8,9]
#pie_illum = [19,20,21,6,11,12,7,8,9]
# Read in PIE Eyes File
eyes = {}
for line in open(coord_name):
#print line
data = line.split()
if len(data) != 5:
print "Warning: expected 5 values: <", data, ">"
print " in file:",coord_name
continue
sub,lx,ly,rx,ry = data
lx = float(lx)
ly = float(ly)
rx = float(rx)
ry = float(ry)
key = sub
#label = "%s %s %s"%key
eyes[key] = (pv.Point(lx,ly),pv.Point(rx,ry))
self.eyes = eyes
keys = []
# Generate PIE keys
for month in pie_months:
month_dir = os.path.join(self.image_path,month)
for sub in os.listdir(month_dir):
if sub[0] != '0' or len(sub) != 5:
continue
illum_dir = os.path.join(month_dir,sub,"ILLUM")
for filename in os.listdir(illum_dir):
pose,illum = filename.split('.')[0].split('_')
pose = int(pose)
illum = int(illum)
if pose != 27:
continue
key = (month,sub,pose,illum)
keys.append(key)
self.key_list = keys
def test_scale(self):
transform = pv.AffineScale(1.5, (640, 480))
_ = transform.transformImage(self.test_image)
#im_a.show()
pt = transform.transformPoint(pv.Point(320, 240))
self.assertAlmostEqual(pt.X(), 480.)
self.assertAlmostEqual(pt.Y(), 360.)
pt = transform.invertPoint(pv.Point(320, 240))
self.assertAlmostEqual(pt.X(), 213.33333333333331)
self.assertAlmostEqual(pt.Y(), 160.)
def test_rotation(self):
transform = pv.AffineRotate(3.14 / 8, (640, 480))
_ = transform.transformImage(self.test_image)
# im_a.show()
pt = transform.transformPoint(pv.Point(320, 240))
self.assertAlmostEqual(pt.X(), 203.86594448424472)
self.assertAlmostEqual(pt.Y(), 344.14920700118842)
pt = transform.invertPoint(pv.Point(320, 240))
self.assertAlmostEqual(pt.X(), 387.46570317672939)
self.assertAlmostEqual(pt.Y(), 99.349528744542198)
def asPolygon(self):
'''
Returns the four corners with the upper left corner repeated twice.
Can be used to transform this rect through an affine or perspective
transformations. It can also be plotted with annotatePolygon.
'''
x, y, w, h = self.asTuple()
pt1 = pv.Point(x, y)
pt2 = pv.Point(x + w, y)
pt3 = pv.Point(x + w, y + h)
pt4 = pv.Point(x, y + h)
return [pt1, pt2, pt3, pt4, pt1]
def __init__(self):
self.tile_size = (128, 160)
self.leye = pv.Point(26.0, 40.0)
self.reye = pv.Point(102.0, 40.0)
self.norm_sigma = 8.0
self.svm = None
self.n_faces = 0
self.n_labels = 0
self.training_data = {}
def setUp(self):
SCRAPS_FACE_DATA = os.path.join(pv.__path__[0],"data","csuScrapShots")
self.test_images = []
self.eyes = EyesFile(os.path.join(SCRAPS_FACE_DATA,"coords.txt"))
for filename in self.eyes.files()[0:10]:
im = pv.Image(os.path.join(SCRAPS_FACE_DATA, filename + ".pgm"))
eyes = self.eyes.getEyes(filename)
#print eyes
affine = pv.AffineFromPoints(eyes[0][0],eyes[0][1],pv.Point(40,40),pv.Point(88,40),(128,128))
im = affine.transformImage(im)
self.test_images.append(im)
def test_from_rect(self):
transform = pv.AffineFromRect(pv.Rect(100, 100, 300, 300), (100, 100))
_ = transform.transformImage(self.test_image)
#im_a.show()
pt = transform.transformPoint(pv.Point(320, 240))
self.assertAlmostEqual(pt.X(), 73.333333333333329)
self.assertAlmostEqual(pt.Y(), 46.666666666666671)
pt = transform.invertPoint(pv.Point(50., 50.))
self.assertAlmostEqual(pt.X(), 250.0)
self.assertAlmostEqual(pt.Y(), 250.0)
def callback(population):
plot = pv.Plot(x_range=[0, 10], y_range=[0, 10], title='Search Space')
plot.point(pv.Point(3, 4), size=20, shape=16, color='gray')
plot.point(pv.Point(8, 2), size=10, shape=16, color='gray')
plot.point(pv.Point(5, 9), size=40, shape=16, color='gray')
pts = [[each[1][0].value, each[1][1].value] for each in population]
#pts = [ pv.Point(each[1][0],each[1][1]) for each in population ]
#print pts
plot.points(pts, color='red')
plot.show(delay=10, window='Search Space')
def test():
'''
'''
p1 = pv.Point(1, 1)
p2 = pv.Point(4, 4)
p3 = pv.Point(5, 4)
p4 = pv.Point(6, 8)
r1 = BoundingRect(p1, p2)
r2 = BoundingRect(p3, p4)
r3 = Rect(3, 3, 3, 3)
print r1
print r2
print r1.intersect(r2)
print r3.intersect(r2)
def locate(self, tile, corr=None, subpixel_est=True, bbox=None, ilog=None):
''''''
if corr == None:
corr = self.correlate(tile, ilog=ilog)
if bbox == None:
bbox = self.bbox
if bbox:
#print "CorrShape:",corr.shape
idx = corr[bbox[0]:bbox[1], bbox[2]:bbox[3]].argmax()
_, h = corr[bbox[0]:bbox[1], bbox[2]:bbox[3]].shape
x = bbox[0] + idx / h
y = bbox[2] + idx % h
else:
idx = corr.argmax()
_, h = corr.shape
x = idx / h
y = idx % h
if subpixel_est:
dx, dy = subpixel(corr[x - 3:x + 4, y - 3:y + 4])
x += dx
y += dy
return pv.Point(x, y)
def test():
'''
Run some simple rectangle tests.
'''
p1 = pv.Point(1,1)
p2 = pv.Point(4,4)
p3 = pv.Point(5,4)
p4 = pv.Point(6,8)
r1 = BoundingRect(p1,p2)
r2 = BoundingRect(p3,p4)
r3 = Rect(3,3,3,3)
print(r1)
print(r2)
print(r1.intersect(r2))
print(r3.intersect(r2))
def _opticalFlow(self):
'''
Compute the optical flow between frames using cv.CalcOpticalFlow
@returns: a list of tracks for the new image
@rtype: list of pv.Point()
'''
flags = 0
grey = self.frame
prev_grey = self.prev_frame
pyramid = cv.CreateImage (cv.GetSize (grey), 8, 1)
prev_pyramid = cv.CreateImage (cv.GetSize (grey), 8, 1)
cv_points = []
for each in self.tracks:
cv_points.append((each.X(),each.Y()))
points_b, _, _,= cv.CalcOpticalFlowPyrLK (
prev_grey,
grey,
prev_pyramid,
pyramid,
cv_points,
(5,5),
3,#pyr number
(cv.CV_TERMCRIT_ITER | cv.CV_TERMCRIT_EPS, 10, 0.01),
flags)
result = []
for pt in points_b:
result.append(pv.Point(pt))
return result
def center(self):
'''
Compute and return a point at the center of the rectangle
@returns: a pv.Point at the center.
'''
return pv.Point(self.x + 0.5 * self.w, self.y + 0.5 * self.h)
def _onNewFrame(self, img, fn, **kwargs):
pt = pv.Point(10, 10)
img.annotateLabel(label="Frame: %d" % (fn + 1),
point=pt,
color="white",
background="black")
if self._windowName != None: img.show(window=self._windowName, delay=1)
def _readEyesFile(self):
'''
Private: Do not call directly. Reads the eye file.
'''
if self.filename[-4:] == '.csv':
f = open(self.filename, 'r')
for line in f:
#print line,
line = line.split(',')
fname = self._parseName(line[0])
eye1 = pv.Point(float(line[1]), float(line[2]))
eye2 = pv.Point(float(line[3]), float(line[4]))
truth_rect = pv.BoundingRect(eye1, eye2)
truth_rect.w = 2.0 * truth_rect.w
truth_rect.h = truth_rect.w
truth_rect.x = truth_rect.x - 0.25 * truth_rect.w
truth_rect.y = truth_rect.y - 0.3 * truth_rect.w
#print fname,eye1,eye2,truth_rect
if fname not in self.images:
self.images[fname] = []
self.images[fname].append([fname, eye1, eye2, truth_rect])
else:
f = open(self.filename, 'r')
for line in f:
#print line,
line = line.split()
fname = self._parseName(line[0])
eye1 = pv.Point(float(line[1]), float(line[2]))
eye2 = pv.Point(float(line[3]), float(line[4]))
truth_rect = pv.BoundingRect(eye1, eye2)
truth_rect.w = 2.0 * truth_rect.w
truth_rect.h = truth_rect.w
truth_rect.x = truth_rect.x - 0.25 * truth_rect.w
truth_rect.y = truth_rect.y - 0.3 * truth_rect.w
#print fname,eye1,eye2,truth_rect
if fname not in self.images:
self.images[fname] = []
self.images[fname].append([fname, eye1, eye2, truth_rect])
def mouseCallback(self, event, x, y, flags, param):
'''
Call back function for mouse events.
'''
if event in [cv2.EVENT_LBUTTONDOWN]:
point = pv.Point(x, y)
self.im.annotateLabel(point, str(len(self.points)), mark='below')
self.points.append(point)
def render(self):
'''
Clear the points and start over.
'''
im = self.buffer[self.buffer_index]
w,h = im.size
nim = pil.new('RGB',(w,h+100))
nim.paste(im.asPIL(),(0,0))
self.im = pv.Image(nim)
if self.callback != None:
self.callback(self.im,self.frame)
self.im.annotateLabel(pv.Point(10,h+10), "Frame: %d"%self.frame,color='yellow')
self.im.annotateLabel(pv.Point(10,h+20), "Click anywhere in the image to select a point.",color='yellow')
self.im.annotateLabel(pv.Point(10,h+30), "Press 'r' to reset.",color='yellow')
self.im.annotateLabel(pv.Point(10,h+40), "Press the space bar or 'n' for the next frame.",color='yellow')
self.im.annotateLabel(pv.Point(10,h+50), "Press 'p' for the previous frame.",color='yellow')
self.im.annotateLabel(pv.Point(10,h+60), "Press 'N' or 'P' to skip 10 frames.",color='yellow')
self.im.annotateLabel(pv.Point(10,h+70), "Press 'q' when finished.",color='yellow')
if self.frame in self.points:
points = self.points[self.frame]
for i in range(len(points)):
pt = points[i]
self.im.annotateLabel(pt,'%d'% i,mark='below')
def test_GaborImage(self):
kernels = createGaborKernels()
filters = GaborFilters(kernels)
gim = filters.convolve(self.test_images[0])
test_point = pv.Point(62.6, 64.8)
template = gim.extractJet(test_point)
new_point, _, _ = gim.locatePoint(template, pv.Point(60, 70))
self.assertAlmostEqual(new_point.l2(test_point), 0.0)
new_point, _, _ = gim.locatePoint(template, pv.Point(30, 49))
self.assertTrue(new_point.l2(test_point) > 1.0)
new_point, _, _ = gim.locatePoint(template)
self.assertAlmostEqual(new_point.l2(test_point), 0.0)
def locateEyes(self, im, face_rects, ilog=None):
'''
Finds the eyes in the image.
@param im: full sized image
@param face_rects: list of rectangle which are the output from the cascade face detector.
'''
cvim = im.asOpenCVBW()
faces = []
for rect in face_rects:
faceim = cv.GetSubRect(cvim, rect.asOpenCV())
cv.Resize(faceim, self.bwtile)
affine = pv.AffineFromRect(rect, (128, 128))
#cv.cvCvtColor( self.cvtile, self.bwtile, cv.CV_BGR2GRAY )
leye, reye, lcp, rcp = self.fel.locateEyes(self.bwtile)
le = pv.Point(leye)
re = pv.Point(reye)
leye = affine.invertPoint(le)
reye = affine.invertPoint(re)
faces.append([rect, leye, reye])
if ilog != None:
ilog.log(pv.Image(self.bwtile), label="FaceDetection")
lcp = pv.OpenCVToNumpy(lcp).transpose()
lcp = lcp * (lcp > 0.0)
rcp = pv.OpenCVToNumpy(rcp).transpose()
rcp = rcp * (rcp > 0.0)
ilog.log(pv.Image(lcp), label="Left_Corr")
ilog.log(pv.Image(rcp), label="Right_Corr")
tmp = pv.Image(self.bwtile)
tmp.annotatePoint(le)
tmp.annotatePoint(re)
ilog.log(tmp, "EyeLocations")
return faces
def mouseCallback(self, event, x, y, flags, param):
'''
Call back function for mouse events.
'''
if event in [cv2.CV_EVENT_LBUTTONDOWN]:
if self.frame not in self.points:
self.points[self.frame] = []
points = self.points[self.frame]
point = pv.Point(x, y)
self.im.annotateLabel(point, str(len(points)), mark='below')
points.append(point)
