def __init__(self,
prev,
curr,
templates,
callback=None,
match_threshold=0.8):
"""
State machine for hand gesture recognition.
Callback is called with the parameters:
query : array_like
The set of (x,y) points that represent the hand path
template : array_like
The best matching template gesture
score : float
The matching score (in range [0,1] with 1 being perfect match)
theta : float
The rotation of the gesture that gave the best match
clsid : int
The ID of the template class that matched best
Parameters
----------
imshape : tuple
The dimensions tuple of the image stream
templates : string, h5py.File
h5py.File instance or path to the h5 dataset of template gestures to use
callback : function
Callback function called when a gesture match is made
"""
super(self.__class__, self).__init__(self.Wait)
self.callback = callback
self.match_threshold = match_threshold
if isinstance(templates, basestring):
self.template_ds = h5py.File(templates, 'r')
else:
self.template_ds = templates
self.segmenter = SkinMotionSegmenter(prev, curr)
self.detector = ConvexityHandDetector()
self.tracker = CrCbMeanShiftTracker()
self.imshape = prev.shape
self.counter = 0
self.waypts = []
def __init__(self,prev,curr,templates,callback=None,match_threshold=0.8):
"""
State machine for hand gesture recognition.
Callback is called with the parameters:
query : array_like
The set of (x,y) points that represent the hand path
template : array_like
The best matching template gesture
score : float
The matching score (in range [0,1] with 1 being perfect match)
theta : float
The rotation of the gesture that gave the best match
clsid : int
The ID of the template class that matched best
Parameters
----------
imshape : tuple
The dimensions tuple of the image stream
templates : string, h5py.File
h5py.File instance or path to the h5 dataset of template gestures to use
callback : function
Callback function called when a gesture match is made
"""
super(self.__class__,self).__init__(self.Wait)
self.callback = callback
self.match_threshold = match_threshold
if isinstance(templates,basestring):
self.template_ds = h5py.File(templates,'r')
else:
self.template_ds = templates
self.segmenter = SkinMotionSegmenter(prev,curr)
self.detector = ConvexityHandDetector()
self.tracker = CrCbMeanShiftTracker()
self.imshape = prev.shape
self.counter = 0
self.waypts = []
self.SELECTING = False
def on_release(self, event):
try:
self._bbox[2:] = int(event.xdata) - self._bbox[0], int(
event.ydata) - self._bbox[1]
assert (self._bbox[2] > 1 and self._bbox[3] > 1)
except:
self._bbox = None
else:
self.bbox = tuple(self._bbox)
self.callback(self.bbox)
self.SELECTING = False
mstrk = CrCbMeanShiftTracker()
app = MeanShiftApp(lambda bbox: mstrk.init(curr, bbox))
app.show()
while app:
curr = blur(app.cap.read())
artists = []
if not app.SELECTING and app.bbox is not None:
bbox = mstrk.track(curr)
artists.append((get_imdisp(app.axes['raw']), curr[:, :, ::-1]))
if mstrk.backprojection is not None:
artists.append((get_imdisp(app.axes['backprojection']),
mstrk.backprojection * 255))
app.update_artists(*artists)
class HandGestureRecognizer(StateMachineBase):
def __init__(self,
prev,
curr,
templates,
callback=None,
match_threshold=0.8):
"""
State machine for hand gesture recognition.
Callback is called with the parameters:
query : array_like
The set of (x,y) points that represent the hand path
template : array_like
The best matching template gesture
score : float
The matching score (in range [0,1] with 1 being perfect match)
theta : float
The rotation of the gesture that gave the best match
clsid : int
The ID of the template class that matched best
Parameters
----------
imshape : tuple
The dimensions tuple of the image stream
templates : string, h5py.File
h5py.File instance or path to the h5 dataset of template gestures to use
callback : function
Callback function called when a gesture match is made
"""
super(self.__class__, self).__init__(self.Wait)
self.callback = callback
self.match_threshold = match_threshold
if isinstance(templates, basestring):
self.template_ds = h5py.File(templates, 'r')
else:
self.template_ds = templates
self.segmenter = SkinMotionSegmenter(prev, curr)
self.detector = ConvexityHandDetector()
self.tracker = CrCbMeanShiftTracker()
self.imshape = prev.shape
self.counter = 0
self.waypts = []
@property
def backprojection(self):
if self.state == 'Track' and self.tracker.backprojection is not None:
return self.tracker.backprojection
return self.segmenter.backprojection
def Wait(self, img):
self.counter = (self.counter + 1) % WAIT_PERIOD
if self.counter == 0:
return self.Search
def Search(self, img):
mask = self.segmenter.segment(img)
if not self.detector(mask):
return
bbox, com = findBBoxCoM_contour(self.detector.contour)
# use the hand contour to draw out a more precise mask of the crcb
# image
mask = np.zeros_like(mask)
cv2.drawContours(mask.view(np.uint8), [self.detector.contour],
0,
1,
thickness=-1)
self.tracker.init(self.segmenter.coseg.converted_image,
bbox,
mask=mask,
update=True)
self.waypts = [com]
self.tracker.backprojection = mask # this is a dirty little trick >:)
self.counter = (self.counter + 1) % VAL_PERIOD
if self.counter == 0:
return self.Track
def Track(self, img):
mask = self.segmenter.segment(img)
if self.segmenter.moseg.bbox is not None:
x, y, w, h = self.segmenter.moseg.bbox
mask.fill(0)
mask[y:y + h, x:x + w] = True
x, y, w, h = self.tracker.track(
self.segmenter.coseg.converted_image, mask)
# it's possible that there is still motion but that tracking failed
# so make sure backprojection is not all zeros
if self.tracker.backprojection.any():
bbox, (xc, yc) = findBBoxCoM(self.tracker.backprojection)
self.waypts.append((int(xc), int(yc)))
return # success! keep tracking...
# if we got to this point then tracking has failed
if len(self.waypts) > MINWAYPTS and self.callback is not None:
# Find best gesture match
x, y = zip(*[(self.imshape[1] - x, self.imshape[0] - y)
for x, y in self.waypts])
matches = dollar.query(x, y, scale, samplesize, self.template_ds)
score, theta, clsid = matches[0]
if score > self.match_threshold:
ds = self.template_ds[clsid][0]
self.callback((x, y), (ds['x'], ds['y']), score, theta, clsid)
self.waypts = []
return self.Wait
except:
self.SELECTING = False
def on_release(self,event):
try:
self._bbox[2:] = int(event.xdata) - self._bbox[0], int(event.ydata) - self._bbox[1]
assert(self._bbox[2] > 1 and self._bbox[3] > 1)
except:
self._bbox = None
else:
self.bbox = tuple(self._bbox)
self.callback(self.bbox)
self.SELECTING = False
mstrk = CrCbMeanShiftTracker()
app = MeanShiftApp(lambda bbox: mstrk.init(curr,bbox))
app.show()
while app:
curr = blur(app.cap.read())
artists = []
if not app.SELECTING and app.bbox is not None:
bbox = mstrk.track(curr)
artists.append((get_imdisp(app.axes['raw']), curr[:,:,::-1]))
if mstrk.backprojection is not None:
artists.append((get_imdisp(app.axes['backprojection']), mstrk.backprojection*255))
app.update_artists(*artists)
class HandGestureRecognizer(StateMachineBase):
def __init__(self,prev,curr,templates,callback=None,match_threshold=0.8):
"""
State machine for hand gesture recognition.
Callback is called with the parameters:
query : array_like
The set of (x,y) points that represent the hand path
template : array_like
The best matching template gesture
score : float
The matching score (in range [0,1] with 1 being perfect match)
theta : float
The rotation of the gesture that gave the best match
clsid : int
The ID of the template class that matched best
Parameters
----------
imshape : tuple
The dimensions tuple of the image stream
templates : string, h5py.File
h5py.File instance or path to the h5 dataset of template gestures to use
callback : function
Callback function called when a gesture match is made
"""
super(self.__class__,self).__init__(self.Wait)
self.callback = callback
self.match_threshold = match_threshold
if isinstance(templates,basestring):
self.template_ds = h5py.File(templates,'r')
else:
self.template_ds = templates
self.segmenter = SkinMotionSegmenter(prev,curr)
self.detector = ConvexityHandDetector()
self.tracker = CrCbMeanShiftTracker()
self.imshape = prev.shape
self.counter = 0
self.waypts = []
@property
def backprojection(self):
if self.state == 'Track' and self.tracker.backprojection is not None:
return self.tracker.backprojection
return self.segmenter.backprojection
def Wait(self,img):
self.counter = (self.counter+1) % WAIT_PERIOD
if self.counter == 0:
return self.Search
def Search(self,img):
mask = self.segmenter.segment(img)
if not self.detector(mask):
return
bbox,com = findBBoxCoM_contour(self.detector.contour)
# use the hand contour to draw out a more precise mask of the crcb
# image
mask = np.zeros_like(mask)
cv2.drawContours(mask.view(np.uint8),[self.detector.contour],0,1,thickness=-1)
self.tracker.init(self.segmenter.coseg.converted_image,bbox,mask=mask,update=True)
self.waypts = [com]
self.tracker.backprojection = mask # this is a dirty little trick >:)
self.counter = (self.counter+1) % VAL_PERIOD
if self.counter == 0:
return self.Track
def Track(self,img):
mask = self.segmenter.segment(img)
if self.segmenter.moseg.bbox is not None:
x,y,w,h = self.segmenter.moseg.bbox
mask.fill(0)
mask[y:y+h,x:x+w] = True
x,y,w,h = self.tracker.track(self.segmenter.coseg.converted_image,mask)
# it's possible that there is still motion but that tracking failed
# so make sure backprojection is not all zeros
if self.tracker.backprojection.any():
bbox,(xc,yc) = findBBoxCoM(self.tracker.backprojection)
self.waypts.append((int(xc),int(yc)))
return # success! keep tracking...
# if we got to this point then tracking has failed
if len(self.waypts) > MINWAYPTS and self.callback is not None:
# Find best gesture match
x,y = zip(*[(self.imshape[1]-x,self.imshape[0]-y) for x,y in self.waypts])
matches = dollar.query(x,y,scale,samplesize,self.template_ds)
score,theta,clsid = matches[0]
if score > self.match_threshold:
ds = self.template_ds[clsid][0]
self.callback((x,y),(ds['x'],ds['y']),score,theta,clsid)
self.waypts = []
return self.Wait
