def InitFromContour(self, contour):
# Prefilter on bounding rect because moments take some time
self.recX, self.recY, self.recW, self.recH = cv2.boundingRect(contour)
if self.recH < config['hand']['minimumHeight'] or self.recH > config['hand']['maximumHeight'] or self.recW < \
config['hand']['minimumWidth'] or self.recW > config['hand']['maximumWidth']:
return False
self.handContour = contour
self.properties['foundHand'] = True
# Get general info
self.moments = cv2.moments(contour)
self.hull = cv2.convexHull(self.handContour, returnPoints = False)
self.defects = cv2.convexityDefects(self.handContour, self.hull)
_,radius = cv2.minEnclosingCircle(self.handContour)
self.radius = int(radius / 1.2)
self.centerX = int(self.moments['m10'] / self.moments['m00'])
self.centerY = int(self.moments['m01'] / self.moments['m00'])
self.properties['centerX'] = self.centerX
self.properties['centerY'] = self.centerY
self.InitGestures()
return True
def get_convex_hull_properties_in_slice(segment): cnt = np.array(segment.get_mid_slice_contour()) cnt_len = cnt.shape[0] cnt = cnt.reshape((cnt_len,1,2)) if len(cnt)>3: hull = cv2.convexHull(cnt,returnPoints = False) segment.feature_dict["mid_slice_convex_hull_indices"] = [x[0] for x in hull] try: defects = cv2.convexityDefects(cnt,hull) if defects is not None: segment.feature_dict["mid_slice_convexity_deffects"] = [[i for i in defects[k][0]] for k in xrange(len(defects))] else: segment.feature_dict["mid_slice_convexity_deffects"] = [[]] except: segment.feature_dict["mid_slice_convexity_deffects"] = [[]] else: segment.feature_dict["mid_slice_convex_hull_indices"] = [] segment.feature_dict["mid_slice_convexity_deffects"] = [[]]
def plot_contours(self,frame):
img1,contours1, hierarchy1 = cv2.findContours(frame,cv2.RETR_CCOMP,cv2.CHAIN_APPROX_SIMPLE)
cnt = self.max_area_contour(contours1)
hull = cv2.convexHull(cnt,returnPoints = False)
defects = cv2.convexityDefects(cnt,hull)
drawing = np.zeros(frame.shape,np.uint8)
drawing = cv2.cvtColor(drawing,cv2.COLOR_GRAY2RGB)
# print defects.shape[0]
for i in range(defects.shape[0]):
k=1
s,e,f,d = defects[k,0]
start = tuple(cnt[s][0])
end = tuple(cnt[e][0])
far = tuple(cnt[f][0])
cv2.circle(drawing,far,5,[0,0,255],-1)
cv2.circle(drawing,end,5,[0,255,255],-1)
#rect = cv2.minAreaRect(cnt)
#angle=rect[2]
#width,height=rect[1]
#box = cv2.boxPoints(rect)
#box = np.int0(box)
#area = cv2.contourArea(cnt)
#print area
drawing = cv2.drawContours(drawing,[cnt],-1,150,1)
return drawing,far,end
def updateCanny(arg):
lt = cv2.getTrackbarPos('Edge Linking',frameName)
ht = cv2.getTrackbarPos('Strong Edges',frameName)
edges = cv2.Canny(frame,lt,ht)
frame2 = copy.deepcopy(frame)
contours, _ = cv2.findContours(edges,1,2)
poly = []
for cnt in contours:
hull = cv2.convexHull(cnt,returnPoints = False)
if(len(hull)>3 and len(cnt)>3):
defects = cv2.convexityDefects(cnt,hull)
if defects!=None:
for i in range(defects.shape[0]):
s,e,f,d = defects[i,0]
start = tuple(cnt[s][0])
end = tuple(cnt[e][0])
far = tuple(cnt[f][0])
if i == 0:
poly.append(start)
poly.append(end)
#cv2.line(frame2,start,end,[0,255,0],2)
#cv2.circle(frame2,far,5,[0,0,255],-1)
#print poly
cv2.fillConvexPoly(frame2,np.array([poly]),(255,255,255))#-1,1,[255,255,255],-1)
#cv2.drawContours(frame2,contours,-1,[255,0,0],3)
edges = cv2.cvtColor(edges,cv2.COLOR_GRAY2BGR)
frame2 = cv2.add(frame2,edges)
cv2.imshow(frameName,frame2)
def recognizeDigits( frame, level = 130 ):
gray = cv2.cvtColor( frame, cv2.COLOR_BGR2GRAY )
kernel = np.ones( (3,3), np.uint8)
gray = cv2.erode( gray, kernel )
ret, binary = cv2.threshold( gray, level, 255, cv2.THRESH_BINARY )
tmp = cv2.cvtColor( binary, cv2.COLOR_GRAY2BGR )
contours, hierarchy = cv2.findContours( binary, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE )
ret = False
c = None
for i,h in enumerate(hierarchy[0]):
n,p,child,parent = h
x,y,w,h = cv2.boundingRect( contours[i] )
if 20 < w < 140 and 20 < h < 180 and w < h < 2*w:
# print i, (x, y), (x+w, y+h), w, h
c = i
b = -1
if x > b and y > b and x+w < 640-b and y+h < 512-b:
if parent >= 0 and fitsIn( (x-b,y-b,w+2*b,h+2*b), contours[parent] ):
if validDigitPosition( x, y, w, h ):
hull = cv2.convexHull( contours[c], returnPoints = False )
defects = cv2.convexityDefects( contours[c], hull )
# print defects
cv2.drawContours(tmp, [contours[c]], -1, (0,255,0), 2)
cv2.rectangle( tmp, (x,y), (x+w,y+h), color=(0,128,255), thickness=2 )
ret = True
cv2.imshow( 'bin', tmp )
return ret
def convex_corners_cross(pil_img, show_plot=False):
mask_img = alpha_fill(pil_img)
_,contours,_ = cv2.findContours(mask_img,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE)
contour = contours[0]
rect = cv2.minAreaRect(contour)
box = cv2.boxPoints(rect)
box = numpy.int0(box)
hull = cv2.convexHull(contour,returnPoints = False)
print(hull)
defects = cv2.convexityDefects(contour,box)
mask_img = cv2.cvtColor(mask_img, cv2.COLOR_GRAY2RGB)
for i in range(defects.shape[0]):
s,e,f,d = defects[i,0]
start = tuple(contour[s][0])
end = tuple(contour[e][0])
far = tuple(contour[f][0])
cv2.line(mask_img,start,end,[0,255,0],2)
cv2.circle(mask_img,far,5,[0,0,255],-1)
cv2.imshow('img',mask_img)
cv2.waitKey(0)
cv2.destroyAllWindows()
def getFeatures(contoursClean):
# get features for feature clustering
X = np.zeros((IMAGES_N, FEATURES_N))
for k in xrange(IMAGES_N):
contour = contoursClean[k]
image = imagesColor[k]
hull = cv2.convexHull(contour, clockwise=True, returnPoints = False)
# valleys is the poins between fingers
# wanna find all defects and collect only valleys
defects = cv2.convexityDefects(contour, hull)
pointsContour = getPointsContour(defects, contour)
pointsSort = clockwisePoints(pointsContour)
# add distances between valleys as features
for i in xrange(4):
X[k, 8 + i] = length(pointsSort[2 * i % 4] - pointsSort[2 * (i + 1) % 4])
# show lines and add features
for i in xrange(len(pointsSort) - 1):
start = pointsSort[i]
end = pointsSort[i + 1]
distance = length(start - end)
# add distances between hole and nearest fingers as features
X[k,i] = distance
if args.save_images:
cv2.line(image, tuple(start), tuple(end), [0, 255, 0], 2)
cv2.imwrite(FOLDER_RESULT_LINES + "/" + objectsNames[k] + ENDING_LINES + FILE_EXTENSION, image)
if args.show_lines:
cv2.line(image, tuple(start), tuple(end), [0, 255, 0], 2)
cv2.imshow('image', image)
cv2.waitKey(0)
return X
def detect_concave_locations(contour, img, draw):
#get convext hull
hull = cv2.convexHull(contour, returnPoints = False)
#get defects in convexity
#each row of defects containts the following:
#(startPoint_of_defect, endPoint_of_defect, farthest point from hull, distance from hull)
defects = cv2.convexityDefects(contour, hull)
farPoints = []
for i in range(defects.shape[0]):
s,e,f,d = defects[i,0]
far = tuple(contour[f][0])
print 'this is distance'
print d
if d > 8000:
farPoints.append(far)
if draw is True:
#cv2.line(img, start, end, [0, 255, 0], 2)
cv2.circle(img, far, 5, [0, 0, 255], -1)
if draw is True:
cv2.imshow('img', img)
cv2.waitKey(0)
cv2.destroyAllWindows()
return farPoints
def handRecognition(img, sliderVals, gestures):
imgColor = img.copy()
img = cv2.cvtColor(img, cv2.COLOR_RGB2HSV)
minRange = cv2.cv.Scalar(sliderVals['Hmin'] , sliderVals['Smin'], 0.0)
maxRange = cv2.cv.Scalar(sliderVals['Hmax'], sliderVals['Smax'], 255.0)
img = cv2.inRange(img, minRange, maxRange)
img = 255 - img
kernel = np.ones((9, 9), np.uint8)
img = cv2.dilate(cv2.erode(img, kernel), kernel)
cv2.imshow("Details", img)
#tools.showImages(testing=img)
contours, _ = cv2.findContours(img, cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE)
#biggestContour[0] == contour, biggestContour[1] == props
if(len(contours) > 0):
biggestContour = contours[0]
biggestContourProps = regionProps.CalcContourProperties(contours[0], ["Area", "Centroid"])
for contour in contours:
#print contour
props = regionProps.CalcContourProperties(contour, ["Area", "Centroid"])
if(biggestContourProps["Area"] < props["Area"]):
biggestContour = contour
biggestContourProps = props
epsilon = cv2.arcLength(biggestContour, True)*0.0025
biggestContour = cv2.approxPolyDP(biggestContour, epsilon, True)
center = (int(biggestContourProps["Centroid"][0]), int(biggestContourProps["Centroid"][1]))
cv2.circle(imgColor, center, 5, (0, 0, 255), (int(biggestContourProps["Area"]*.00005) + 1) )
#cv2.drawContours(imgColor, [biggestContour], -1, (0, 255, 0))
convexhull = cv2.convexHull(biggestContour, returnPoints = False)
defects = cv2.convexityDefects(biggestContour, convexhull)
newGestures = detectGestures(imgColor, biggestContour, biggestContourProps, defects, gestures)
#cv2.drawContours(imgColor, [convexhull], -1, (0, 0, 255))
return imgColor, newGestures
def convexhullallcontours(contours):
number = len(contours)
status = numpy.zeros((number, 1))
for i, cnt1 in enumerate(contours):
x = i
if i != number - 1:
for j, cnt2 in enumerate(contours[i + 1:]):
x += 1
dist = arecontoursclose(cnt1, cnt2, distancethreshold=150)
dist = True
if dist is True:
val = min(status[i], status[x])
status[x] = status[i] = val
# ~ else:
# ~ print ("Do nothing")
maximum = int(status.max()) + 1
cont = None
unified = []
unified2 = []
for i in xrange(maximum):
pos = numpy.where(status == i)[0]
if pos.size != 0:
cont = numpy.vstack(contours[i] for i in pos)
hull = cv2.convexHull(cont)
hullindices = cv2.convexHull(cont, returnPoints=False)
unified.append(hull)
defects = cv2.convexityDefects(cont, hullindices)
unified2.append(defects)
return cont, unified, unified2
def find_concave_regions(mask, max_dist):
'''Finds convace regions along the contour of `mask`.
Parameters
----------
mask: numpy.ndarray[numpy.bool]
mask image
max_dist: int
maximally tolerated distance between concave point on object contour
and the convex hull
'''
contour_img = np.zeros(mask.shape, dtype=np.uint8)
contour_img[mask] = 255
contour_img, contours, _ = cv2.findContours(
contour_img, mode=cv2.RETR_EXTERNAL, method=cv2.CHAIN_APPROX_SIMPLE
)
concave_img = np.zeros(mask.shape, dtype=np.bool)
for cnt in contours:
hull = cv2.convexHull(cnt, returnPoints=False)
defects = cv2.convexityDefects(cnt, hull)
if defects is not None:
defect_pts = np.array([
cnt[defects[j, 0][2]][0]
for j in xrange(defects.shape[0])
if defects[j, 0][3]/float(256) > max_dist
])
if defect_pts.size != 0:
concave_img[defect_pts[:, 1], defect_pts[:, 0]] = True
return mh.label(concave_img)
def calibrate_threshold(self, img):
hsv1 = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
h1,s1,v1 = cv2.split(hsv1)
dummy, v1 = cv2.threshold(v1, self.threshold, 255, cv2.THRESH_BINARY)
v1 = self.smart_filter(h1, v1)
contours, hier = cv2.findContours(v1, cv2.RETR_LIST, cv2.CHAIN_APPROX_SIMPLE)
cnts = [cnt for cnt in contours if cv2.contourArea(cnt) > 3000]
c1 = self.biggest_cnt(cnts)
if c1 == None:
self.feedback()
return None
c1 = cv2.approxPolyDP(c1, 5, True)
self.rects = [cv2.boundingRect(c1)]
v1 = np.zeros(v1.shape, np.uint8)
cv2.drawContours(v1,[c1],-1,(255,0,0),-1)
hull = cv2.convexHull(c1, returnPoints = False)
defects = cv2.convexityDefects(c1, hull)
sum_area = 0
for i in range(defects.shape[0]):
s,e,f,d = defects[i][0]
start = tuple(c1[s][0])
end = tuple(c1[e][0])
far = tuple(c1[f][0])
t = np.array([[start], [end], [far]])
sum_area += cv2.contourArea(t)
self.defects_count = defects.shape[0]
self.avg_area = cv2.contourArea(c1)
self.prc_avg_area = self.avg_area/float(area(self.rects[0]))
self.prc_defect_area = (sum_area/defects.shape[0])/self.avg_area
self.feedback()
def getIndexofContourWithMostDefects(contours):
maxCount = 0
maxIndex = 0
for i in range(0, len(contours)):
countGoodDefects = 0
cnt = contours[i]
hull = cv2.convexHull(cnt, returnPoints=False)
defects = cv2.convexityDefects(cnt, hull)
# print defects
if defects is not None:
# print "hi"
for j in range(defects.shape[0]):
s, e, f, d = defects[j, 0]
start = tuple(cnt[s][0])
end = tuple(cnt[e][0])
far = tuple(cnt[f][0])
angle = getAngle(start, far, end)
if angle < 80:
countGoodDefects = countGoodDefects + 1
if countGoodDefects > maxCount:
maxCount = countGoodDefects
maxIndex = i
# print maxIndex
return maxIndex
def findHullAndDefects(self):
self.hullHandContour = cv2.convexHull(self.handContour,
returnPoints = False)
self.hullPoints = [self.handContour[i[0]] for i in self.hullHandContour]
self.hullPoints = np.array(self.hullPoints, dtype = np.int32)
self.defects = cv2.convexityDefects(self.handContour,
self.hullHandContour)
def defects(contour): hull = cv2.convexHull(contour, returnPoints=False) if hull is not None and len(hull > 3) and len(contour) > 3: defects = cv2.convexityDefects(contour, hull) return defects else: return None
def draw_convex_hull(a, original):
original = cv2.cvtColor(original, cv2.COLOR_GRAY2BGR)
ret, b = cv2.threshold(a, 255, 255, cv2.THRESH_BINARY)
contornos, jerarquia = cv2.findContours(a,
cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_SIMPLE)
for n, cnt in enumerate(contornos):
hull = cv2.convexHull(cnt)
foo = cv2.convexHull(cnt, returnPoints=False)
cv2.drawContours(original, contornos, n, (0, 35, 245))
if len(cnt) > 3 and len(foo) > 2:
defectos = cv2.convexityDefects(cnt, foo)
if defectos is not None:
defectos = defectos.reshape(-1, 4)
puntos = cnt.reshape(-1, 2)
for d in defectos:
if d[3] > 20:
cv2.circle(original, tuple(puntos[d[0]]), 5, (255, 255, 0), 2)
cv2.circle(original, tuple(puntos[d[1]]), 5, (255, 255, 0), 2)
cv2.circle(original, tuple(puntos[d[2]]), 5, (0, 0, 255), 2)
lista = numpy.reshape(hull, (1, -1, 2))
cv2.polylines(original, lista, True, (0, 255, 0), 3)
center, radius = cv2.minEnclosingCircle(cnt)
center = tuple(map(int, center))
radius = int(radius)
cv2.circle(original, center, radius, (255, 0, 0), 3)
cv2.imshow('Original', original)
def detect(frame):
cp = np.copy(frame)
contours, hierarchy = cv2.findContours(cp,cv2.RETR_TREE,cv2.CHAIN_APPROX_TC89_L1)
max_area = 0
if len(contours)>0:
ci = 0
for i in range(len(contours)):
cnt=contours[i]
area = cv2.contourArea(cnt)
if(area>max_area):
max_area=area
ci=i
cnt=contours[ci]
defects = []
hull = cv2.convexHull(cnt,returnPoints = False)
if (len(hull)>3 and len(cnt)>3):
defects = cv2.convexityDefects(cnt,hull)
M = cv2.moments(cnt)
cent_x = int(M['m10']/M['m00'])
cent_y = int(M['m01']/M['m00'])
# if len(defects)>0:
# for i in range(defects.shape[0]):
# s,e,f,d = defects[i,0]
# start = tuple(cnt[s][0])
# end = tuple(cnt[e][0])
# far = tuple(cnt[f][0])
# cv2.line(frame,start,end,[0,255,0],2)
# cv2.circle(frame,far,5,[0,0,255],-1)
# cv2.imshow('cvhull', frame)
# cv2.waitKey(99999)
try:
return (cent_x, cent_y), len(defects)
except:
return (0,0), 0
return (0,0), 0
def detectaMao(frame, mascara):
bckpmas = np.copy(mascara)
img, contornos, hieraquia = cv2.findContours(mascara, 0, 2)
if len(contornos) == 0:
return
maior_contorno = selecionaMaiorContorno(contornos)
cv2.polylines(frame, [maior_contorno], True, verde, 1)
hull = cv2.convexHull(maior_contorno, returnPoints=False)
defects = cv2.convexityDefects(maior_contorno, hull)
if defects == None:
return
defects_detectados = [] # Lista vazia de defects detectados
for i in xrange(defects.shape[0]):
a, b, c, d = defects[i, 0]
ini = tuple(maior_contorno[a, 0])
fim = tuple(maior_contorno[b, 0])
cvx = tuple(maior_contorno[c, 0])
cv2.line(frame,ini,fim,verde,1)
angulo = calculaAngulo(ini,cvx,fim)
if 10 < angulo < 120 and d > 15000:
pos = c*100/len(maior_contorno) # posicao do defect no contorno
dados = (ini,cvx,fim,angulo,pos)
defects_detectados.append(dados)
cv2.putText(frame,str(pos),cvx,1,1,branco)
cv2.circle(frame,cvx,3,vermelho,-1)
dedos = identificaDedos(defects_detectados)
exibeInfo(frame,bckpmas,dedos)
def gestureRecognizer(self):
cv2.rectangle(self.img, (300, 300), (100, 100), (0, 255, 0), 0)
crop_img = self.img[100:300, 100:300]
grey = cv2.cvtColor(crop_img, cv2.COLOR_BGR2GRAY)
value = (35, 35)
blurred = cv2.GaussianBlur(grey, value, 0)
_, thresh1 = cv2.threshold(blurred, 127, 255,
cv2.THRESH_BINARY_INV+cv2.THRESH_OTSU)
cv2.imshow('Thresholded', thresh1)
contours, hierarchy = cv2.findContours(thresh1.copy(), cv2.RETR_TREE, cv2.CHAIN_APPROX_NONE)
max_area = -1
for i in range(len(contours)):
cnt = contours[i]
area = cv2.contourArea(cnt)
if area > max_area:
max_area = area
ci = i
cnt = contours[ci]
x, y, w, h = cv2.boundingRect(cnt)
cv2.rectangle(crop_img, (x, y), (x+w, y+h), (0, 0, 255), 0)
hull = cv2.convexHull(cnt)
drawing = np.zeros(crop_img.shape, np.uint8)
cv2.drawContours(drawing, [cnt], 0, (0, 255, 0), 0)
cv2.drawContours(drawing, [hull], 0, (0, 0, 255), 0)
hull = cv2.convexHull(cnt, returnPoints=False)
defects = cv2.convexityDefects(cnt, hull)
count_defects = 0
cv2.drawContours(thresh1, contours, -1, (0, 255, 0), 3)
for i in range(defects.shape[0]):
s, e, f, d = defects[i, 0]
start = tuple(cnt[s][0])
end = tuple(cnt[e][0])
far = tuple(cnt[f][0])
a = math.sqrt((end[0] - start[0])**2 + (end[1] - start[1])**2)
b = math.sqrt((far[0] - start[0])**2 + (far[1] - start[1])**2)
c = math.sqrt((end[0] - far[0])**2 + (end[1] - far[1])**2)
angle = math.acos((b**2 + c**2 - a**2)/(2*b*c)) * 57
if angle <= 90:
count_defects += 1
cv2.circle(crop_img, far, 1, [0, 0, 255], -1)
cv2.line(crop_img, start, end, [0, 255, 0], 2)
# start actions when number of fingers is recognized
if count_defects == 2:
cv2.putText(self.img, "Volume Down", (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 2, 2)
self.volumeDown(1)
elif count_defects == 3:
cv2.putText(self.img, "Volume Up", (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 1, 2)
self.volumeUp(1)
elif count_defects == 4:
cv2.putText(self.img, "Play/Pause", (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 2, 2)
self.pauseAndStartVideo(2)
else:
cv2.putText(self.img, "Finger Control", (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 2, 2)
cv2.imshow('Gesture', self.img)
all_img = np.hstack((drawing, crop_img))
# Display the resulting frame
cv2.imshow('Contours', all_img)
def contour_reader(image):
temp = image # store here because findContours modifes source
contours, hierarchy = cv2.findContours(image,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE)
cnt = contours[0]
# check curve for convexity defects and correct it
# pass in contour points, hull, !returnPoints return indices
hull = cv2.convexHull(cnt,returnPoints = False)
defects = cv2.convexityDefects(cnt,hull) # array
image = temp # revert to original
cv2.drawContours(image, contours, 0, (255,255,255), 1)
image = colorize(image) # prep for drawing in color
# defects returns four arrays: start point, end point, farthest
# point (indices of cnt), and approx distance to farthest point
interdigitalis = 0 # representing tip of finger to convext points
if len(hull) > 3 and len(cnt) > 3 and (defects is not None):
for i in range(defects.shape[0]):
s,e,f,d = defects[i,0]
start = tuple(cnt[s][0])
end = tuple(cnt[e][0])
far = tuple(cnt[f][0])
# print start, end, far
cv2.line(image,start,end,[0,255,0],1)
cv2.circle(image,far,3,[255,0,255],-1)
# print d
if d > 6000: # set by trial and error
interdigitalis += 1
# print 'interdigitalis:', interdigitalis
# find centroid
M = cv2.moments(cnt)
cx = int(M['m10']/M['m00'])
cy = int(M['m01']/M['m00'])
centroid = (cx, cy)
cv2.circle(image, centroid, 6, (255,255,0), -1)
show(image, 1000)
# print 'centroid:', centroid
gesture = classify(interdigitalis)
# print 'gesture', gesture
# calculate size for height as image is a numpy array
h = len(image)
location,lst = locate(h,cx,cy)
# print location
# draw visual grid based on list coordinates returned by locate function
for i in xrange(0, len(lst), 2):
a = lst[i]
b = lst[i+1]
# print a,b
cv2.line(image,a,b,[128,128,128],1)
show(image, 1000)
pair = (gesture, location)
return image,pair
def tip(arrow):
hull = cv2.convexHull(arrow, returnPoints=False)
defects = cv2.convexityDefects(arrow, hull)
farpoints = [d[0][2] for d in defects]
if np.abs(farpoints[0] - farpoints[1]) == 4:
return arrow[sum(farpoints) / 2, 0]
else:
return arrow[0, 0]
def get_defects(self, cnt, hull):
"""
Get convexity defects from a contour.
"""
if len(cnt) > 3:
return cv2.convexityDefects(cnt, hull)
else:
return None
def _extract_convexity_defects(contour):
hull = cv2.convexHull(contour, returnPoints=False)
if len(hull) > 0:
defects = cv2.convexityDefects(contour, hull)
defects = defects.reshape(-1, 4)
return defects
else:
raise NoHandException
def find_contour_and_convexity_defect(im,imgray): ''' binarise the image on the basis of the threshold 127 ''' ret,thresh = cv2.threshold(imgray,127,255,0) #find countours on the binary image using the opencv draw_countours method contours,hierarchy = cv2.findContours(thresh,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE) #an array storing the length of countours found using findContours c = [len(c) for c in contours] #draw the contours on the image cv2.drawContours(im, [contours[c.index(max(c))]], 0, (0,255,0), 1) #only consider the contour having the largest number of points cnt = [contours[c.index(max(c))]][0] #draw convexhull on the contours enclosing the hand hull = cv2.convexHull(cnt,returnPoints = False) #find the convexity defectsin the convex hull defects = cv2.convexityDefects(cnt,hull) #count of the number of the filtered defects c = 0 if(defects == None) : ''' return a tupple using the image and the count if there are no defects ''' return (im, c) for i in range(defects.shape[0]): ''' for each convexity defects we have the start point the farthest point and the end point ''' s,e,f,d = defects[i,0] start = tuple(cnt[s][0]) end = tuple(cnt[e][0]) far = tuple(cnt[f][0]) #draw the convex hull edge corresponding to the convex defects cv2.line(im,start,end,[0,255,0],2) #find the angle of the convex hull defect angle = (find_angle(list(cnt[s][0]), list(cnt[f][0]), (cnt[e][0]))) if(angle < 90) : ''' only consider a defect if the angle corresponding to it is less than 90 degree ''' c += 1 #mark the points on the convex hull and the convex defects cv2.circle(im,start,5,[0,255,255],-1) cv2.circle(im,end,5,[0,255,255],-1) cv2.circle(im,far,5,[0,0,255],-1) return (im, c)
def compute_shape_metrics(labels):
reader = csv.reader(open(labels))
results = None
for row in reader:
if row[0] == "IMG":
continue
name = row[0]
if (row[1] == " metal_lines" or row[1]==" metal_tree"):
continue
buf = 10
x = int(row[2])
y = int(row[3])
x_size = int(row[4])/2 + buf
y_size = int(row[5])/2 + buf
# Load Images
img = cv2.imread(name+".jpg")
# Color Space Conversion
lab = cv2.cvtColor(img,cv2.COLOR_BGR2LAB)
win = img[y-y_size:y+y_size,x-x_size:x+x_size]
lab_win = lab[y-y_size:y+y_size,x-x_size:x+x_size]
mid = (float(np.max(lab_win[...,1]))+float(np.min(lab_win[...,1])))/2.
thresh = cv2.threshold(lab_win[...,1],mid,255,cv2.THRESH_BINARY)[1]
# Contours
contours, hier = cv2.findContours(thresh,cv2.RETR_EXTERNAL,cv2.CHAIN_APPROX_NONE)
largest_area = 0
largest_contour = 0
for i in contours:
area = cv2.contourArea(i)
if area > largest_area:
largest_area = area
largest_contour = i
# Hulls
hull = cv2.convexHull(largest_contour,returnPoints=False)
hull_pts = cv2.convexHull(largest_contour)
# Perimeter Difference
perimeter_ratio = cv2.arcLength(hull_pts,True)/cv2.arcLength(largest_contour,True)
# Area Difference
area_ratio = cv2.contourArea(largest_contour)/cv2.contourArea(hull_pts)
# Max Convexity Defect
defects = cv2.convexityDefects(largest_contour,hull)
depth = np.max(defects[...,-1])/256.0
rect = cv2.boundingRect(hull_pts)
defect_ratio = depth/np.max(rect[2:])
# Combined Score
print perimeter_ratio,area_ratio,defect_ratio,np.sum(defects[...,-1]),row[-1],row[1]
#print perimeter_ratio,area_ratio,np.max(defects[...,-1]),np.sum(defects[...,-1]),row[-1],row[1]
if results == None:
results = np.array([perimeter_ratio,area_ratio,defect_ratio,])
#results = np.array([perimeter_ratio,area_ratio,defect_ratio,np.sum(defects[...,-1])])
#results = np.array([perimeter_ratio,area_ratio,np.max(defects[...,-1]),np.sum(defects[...,-1])])
else:
results = np.vstack((results,np.array([perimeter_ratio,area_ratio,defect_ratio])))
#results = np.vstack((results,np.array([perimeter_ratio,area_ratio,defect_ratio,np.sum(defects[...,-1])])))
#results = np.vstack((results,np.array([perimeter_ratio,area_ratio,np.max(defects[...,-1]),np.sum(defects[...,-1])])))
return results
def extract(mask):
height, width = mask.shape
pixels = height * width
# spatial and central moments
moments = cv2.moments(mask, binary = 1)
huMoments = cv2.HuMoments(moments)
# distances from the gravity point
contour_seq = cv2.findContours(np.array(mask), cv2.CV_RETR_TREE, cv2.CV_CHAIN_APPROX_SIMPLE)
gravity_center = (int(moments.m10/moments.m00), int(moments.m01/moments.m00)) # (x, y)
gx, gy = gravity_center
distances = np.array([math.sqrt((gx - x)**2 + (gy - y)**2) for (x,y) in contour_seq])
dist_distri, bin_dist = np.histogram(distances, bins=10, normed=True)
dist_max = np.max(distances)
dist_min = np.min(distances)
dist_ratio_min_max = dist_min / dist_max
dist_mean = np.mean(distances)
dist_std = np.std(distances)
# number of petals
nbPetals = np.sum([min(x1,x2) < dist_mean < max(x1,x2) for x1,x2 in zip(distances[:-1], distances[1:])])/2
# normalize distance min and max
dist_max = dist_max / pixels
dist_min = dist_min / pixels
dist_mean = dist_mean / pixels
dist_std = dist_std / pixels
poly_seq = cv2.approxPolyDP(contour_seq, 2.8, True)
convex_hull = cv2.convexHull(poly_seq)
convexity_defects = cv2.convexityDefects(poly_seq, convex_hull)
# number of defects
nbDefects = len(convexity_defects)
convexity_depths = np.array([cd[3] for cd in convexity_defects])
convexity_depth_max = np.max(convexity_depths)
convexity_depth_min = np.min(convexity_depths)
convexity_depth_ratio_min_max = convexity_depth_min / convexity_depth_max
#normalize
convexity_depth_max = convexity_depth_max / pixels
area = cv2.contourArea(contour_seq)
perimeter = cv2.arcLength(contour_seq, True)
perimeterOarea = perimeter/area
features = []
features += list(huMoments) # 7
features += list(dist_distri) # 10
features += dist_ratio_min_max, nbPetals, nbDefects #, dist_max, dist_min, dist_mean, dist_std
features += convexity_depth_ratio_min_max, convexity_depth_max, perimeterOarea
return features
def run(self, contour):
contour = self._contour_approx(contour)
hull = cv2.convexHull(contour, returnPoints=False)
if len(hull) < self.MIN_HULL_POINTS:
return []
defects = cv2.convexityDefects(contour, hull)
box = cv2.boundingRect(contour)
return (defects, box)
def track(self, frame, hist):
tracking_frame = self.apply_hist_mask(frame, hist)
tracking_frame = cv2.morphologyEx(tracking_frame, cv2.MORPH_OPEN, np.ones((19, 19), np.uint8))
tracking_frame = cv2.morphologyEx(tracking_frame, cv2.MORPH_CLOSE, np.ones((51, 11), np.uint8))
cv2.GaussianBlur(tracking_frame, (17, 17), 0, tracking_frame)
gray = cv2.cvtColor(tracking_frame, cv2.COLOR_BGR2GRAY)
ret, thresh = cv2.threshold(gray, 0, 255, 0)
_, contours, hierarchy = cv2.findContours(thresh, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
if contours is not None and len(contours) > 0:
max_contour = self.calc_max_contour(contours)
hull = cv2.convexHull(max_contour)
cv2.drawContours(tracking_frame, [hull], 0, (0, 0, 255), 0)
cv2.drawContours(tracking_frame, max_contour, -1, (0, 255, 0), 3)
centroid = self.calc_centroid(max_contour)
(x, y), radius = cv2.minEnclosingCircle(max_contour)
cv2.circle(tracking_frame, (int(x), int(y)), int(radius), (0, 255, 0), 2)
self.hand_center = (int(x), int(y))
self.hand_radius = radius
defects = self.calc_defects(max_contour)
if centroid is not None and defects is not None and len(defects) > 0:
farthest_point = self.calc_farthest_point(defects, max_contour, centroid)
if farthest_point is not None:
cv2.circle(tracking_frame, farthest_point, 5, [0, 0, 255], -1)
defects = cv2.convexityDefects(max_contour, cv2.convexHull(max_contour, returnPoints=False))
count_defects = 0
for i in range(defects.shape[0]):
s, e, f, d = defects[i, 0]
start = tuple(max_contour[s][0])
end = tuple(max_contour[e][0])
far = tuple(max_contour[f][0])
a = math.sqrt((end[0] - start[0]) ** 2 + (end[1] - start[1]) ** 2)
b = math.sqrt((far[0] - start[0]) ** 2 + (far[1] - start[1]) ** 2)
c = math.sqrt((end[0] - far[0]) ** 2 + (end[1] - far[1]) ** 2)
angle = math.acos((b ** 2 + c ** 2 - a ** 2) / (2 * b * c)) * 57
if angle <= 90:
count_defects += 1
cv2.circle(tracking_frame, far, 1, [0, 0, 255], -1)
#dist = cv2.pointPolygonTest(max_contour,far,True)
cv2.line(tracking_frame, start, end, [0, 255, 0], 2)
self.fingers = count_defects
return tracking_frame
def _contourConvexityDefects(contours):
contourConvexityDefects = {}
for contour in contours:
hullIndices = cv2.convexHull(contour, returnPoints=False)
if len(hullIndices)>3 and len(contour)>3:
defects = cv2.convexityDefects(contour, hullIndices)
contour.flags.writeable = False
contourConvexityDefects[Helpers.hashable(contour)] = defects
# pass
return contourConvexityDefects
def get_defects(self, cnt, drawing):
defects = None
hull = cv2.convexHull(cnt)
cv2.drawContours(drawing, [cnt], 0, (0, 255, 0), 0)
cv2.drawContours(drawing, [hull], 0, (0, 0, 255), 0)
hull = cv2.convexHull(cnt, returnPoints=False) # For finding defects
if hull.size > 2:
defects = cv2.convexityDefects(cnt, hull)
return defects
def trackHand(self, cnt, frame, textPos,
width): # Tracks hand position and number of fingers
cv2 = self.cv2
math = self.math
count = 0
side = 0
moments = cv2.moments(cnt)
# Finds center coordinates for hand
cx = int(moments['m10'] / moments['m00'])
cy = int(moments['m01'] / moments['m00'])
hull = cv2.convexHull(
cnt, returnPoints=False
) # Finds convex hull of the hand contour (only indices)
pointsHull = cv2.convexHull(
cnt, returnPoints=True
) # Finds convex hull of the hand contour (actual points)
# Finds shortest distance from the center of the hand to the convex hull
radius = int(1.2 * cv2.pointPolygonTest(pointsHull, (cx, cy), True))
# Draws largest circle inside the convex hull centered at (cx, cy)
cv2.circle(frame, (cx, cy), radius, (255, 255, 255), 2)
x, y, w, h = cv2.boundingRect(
cnt) # Draws bounding rectangle around hand
cv2.rectangle(frame, (x, y), (x + w, y + h), (0, 255, 0), 2)
# Finds convexity defects along contour based on its convex hull
defects = cv2.convexityDefects(cnt, hull)
if cx < width / 2: # Center of the hand is on the left side of the screen
side = 0
elif cx > width / 2: # Center of the hand is on the right side of the screen
side = 1
try:
for i in range(
defects.shape[0]): # For all convexity defects found
s, e, f, d = defects[i, 0]
start = tuple(cnt[s][0])
end = tuple(cnt[e][0])
far = tuple(cnt[f][0])
# Find length of sides of triangle between two nearest points on convex hull and convexity defect
a = math.sqrt((end[0] - start[0])**2 + (end[1] - start[1])**2)
b = math.sqrt((far[0] - start[0])**2 + (far[1] - start[1])**2)
c = math.sqrt((end[0] - far[0])**2 + (end[1] - far[1])**2)
# Use Law of Cosines to find angle of triangle at convexity defect
angle = math.acos((b**2 + c**2 - a**2) / (2 * b * c)) * 57
# Determines end points to draw lines that most closely approximate the fingers
if side: # If the hand is on the right side, use the closest right point on the convex hull
fingerLen = b
defectPoint = start
else: # If the hand is on the left side, use the closest left point on the convex hull
fingerLen = c
defectPoint = end
# If the height of the bounding rectangle is smaller than the diameter of the center circle,
# then the hand must be in a fist, and therefore no fingers are up
if h > 2.4 * radius:
if fingerLen >= h / 4 and angle < 90:
count += 1
# cv2.circle(frame, far, 1, [0, 0, 255], 6)
# cv2.circle(frame, end, 1, [0, 0, 255], 12)
cv2.line(frame, defectPoint, far, [255, 0, 0],
2) # Draws lines that approximate fingers
else:
return (0, (side, (cx, cy))
) # No fingers are up, then the hand is in a fist
cv2.line(frame, start, end, [0, 255, 0], 2) #Draws contours
return (count + 1, (side, (cx, cy)))
except (AttributeError):
print("No defects found")
return (0, (side, (cx, cy)))
class hand(object):
#LOADING HAND CASCADE
hand_cascade = cv2.CascadeClassifier('hand_haar_cascade.xml')
# VIDEO CAPTURE
cap = cv2.VideoCapture(0)
while 1:
ret, img = cap.read()
blur = cv2.GaussianBlur(img, (5, 5),
0) # BLURRING IMAGE TO SMOOTHEN EDGES
#BGR->grey
gray = cv2.cvtColor(blur, cv2.COLOR_BGR2GRAY)
#threshold the image
retval2, thresh1 = cv2.threshold(
gray, 70, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)
#detecting hand in threshold image
hand = hand_cascade.detectMultiScale(thresh1, 1.3, 5)
#create mask
mask = np.zeros(thresh1.shape, dtype="uint8")
#detecting roi
for (x, y, w, h) in hand:
cv2.rectangle(img, (x, y), (x + w, y + h), (122, 122, 0), 2)
cv2.rectangle(mask, (x, y), (x + w, y + h), 255, -1)
img2 = cv2.bitwise_and(thresh1, mask)
final = cv2.GaussianBlur(img2, (7, 7), 0)
contours, hierarchy = cv2.findContours(final, cv2.RETR_TREE,
cv2.CHAIN_APPROX_SIMPLE)
cv2.drawContours(img, contours, 0, (255, 255, 0), 3)
cv2.drawContours(final, contours, 0, (255, 255, 0), 3)
if len(contours) > 0:
cnt = contours[0]
hull = cv2.convexHull(cnt, returnPoints=False)
# finding convexity defects
defects = cv2.convexityDefects(cnt, hull)
count_defects = 0
# applying Cosine Rule to find angle for all defects (between fingers)
# with angle > 90 degrees and ignore defect
if defects is not None:
for i in range(defects.shape[0]):
p, q, r, s = defects[i, 0]
finger1 = tuple(cnt[p][0])
finger2 = tuple(cnt[q][0])
dip = tuple(cnt[r][0])
# find length of all sides of triangle
a = math.sqrt((finger2[0] - finger1[0])**2 +
(finger2[1] - finger1[1])**2)
b = math.sqrt((dip[0] - finger1[0])**2 +
(dip[1] - finger1[1])**2)
c = math.sqrt((finger2[0] - dip[0])**2 +
(finger2[1] - dip[1])**2)
# apply cosine rule here
angle = math.acos(
(b**2 + c**2 - a**2) / (2 * b * c)) * 57.29
# ignore angles > 90 and highlight rest with red dots
if angle <= 90:
count_defects += 1
# define actions required
if count_defects == 0:
cv2.putText(img, "THIS IS 1", (50, 50),
cv2.FONT_HERSHEY_SIMPLEX, 2, 2)
elif count_defects == 1:
cv2.putText(img, "THIS IS 2", (50, 50),
cv2.FONT_HERSHEY_SIMPLEX, 2, 2)
elif count_defects == 2:
cv2.putText(img, "THIS IS 3", (50, 50),
cv2.FONT_HERSHEY_SIMPLEX, 2, 2)
elif count_defects == 3:
cv2.putText(img, "This is 4", (50, 50),
cv2.FONT_HERSHEY_SIMPLEX, 2, 2)
elif count_defects == 4:
cv2.putText(img, "THIS IS 5", (50, 50),
cv2.FONT_HERSHEY_SIMPLEX, 2, 2)
cv2.imshow('img', thresh1)
cv2.imshow('img1', img)
cv2.imshow('img2', img2)
k = cv2.waitKey(30) & 0xff
if k == 27:
break
cap.release()
cv2.destroyAllWindows()
def find_fingers(cnt, center, wrist, offset, rad):
bx, by, bw, bh = cv2.boundingRect(cnt)
hull = cv2.convexHull(cnt, returnPoints=False)
#deft = cv2.convexityDefects(cnt,hull)
defects = cv2.convexityDefects(cnt, hull)
if defects is not None:
#idx = np.sort(deft.T[3][0].argsort()[-num:]) # choose 4 largest distant
#defects = deft[idx]
cor = []
mid = []
for i in xrange(len(defects)):
s, e, f, d = defects[i, 0]
cor.append(tuple(cnt[s][0])) #append start pt
cor.append(tuple(cnt[e][0])) #append end pt
#---append twice ----
mid.append(tuple(cnt[f][0]))
mid.append(tuple(cnt[f][0]))
#-- select real finger points
cmtx = np.array(cor)
mmtx = np.array(mid)
distchk = ((cmtx.T[0] - center[0])**2 + (cmtx.T[1] - center[1])**
2)**0.5 > max(bw, bh) / 3. #finger len check
wristchk = ((mmtx.T[0] - wrist.x + offset[0])**2 +
(mmtx.T[1] - wrist.y + offset[1])**2)**0.5 > (
(center[0] - wrist.x + offset[0])**2 +
(center[1] - wrist.y + offset[1])**2)**0.5
aglchk = find_angle(cmtx, np.array(mid)) < 90 # finger angle check
#pdb.set_trace()
chkidx = np.where((aglchk & distchk & wristchk) == True)[0]
#chkidx = np.where((aglchk&distchk)==True)[0]
cor = list(set([cor[i] for i in chkidx])) #remove duplicate pts
cormtx = np.array(cor)
chk = len(cor)
X = np.array([])
Y = np.array([])
if chk > 1: # more than 5 finger points # merger close pt pairs
# calculate distance
XX1 = np.tile(cormtx.T[0], (chk, 1))
XX2 = np.tile(np.array([cormtx.T[0]]).T, (1, chk))
YY1 = np.tile(cormtx.T[1], (chk, 1))
YY2 = np.tile(np.array([cormtx.T[1]]).T, (1, chk))
distpt = ((XX1 - XX2)**2 + (YY1 - YY2)**2)**0.5 #pt dist mtx
#th = np.sort(list(set(distpt.flatten())))[chk-5] # set a threshold
th = rad / 5.
distpt[distpt > th] = 0
# find shortest dist in dist matrix (in upper triangle)
temp = np.nonzero(np.triu(distpt)) # points coordinate
dup = (np.bincount(temp[0], minlength=chk) > 1) | (np.bincount(
temp[1], minlength=chk) > 1)
duppts = np.where(dup)[0] #duplicat pts: one pt close to multi pts
rmidx = np.array([])
for i in duppts:
dupidx = np.where((temp[0] == i) | (temp[1] == i))[0]
minidx = distpt[temp[0][dupidx], temp[1][dupidx]].argmin()
delidx = np.delete(dupidx, minidx)
rmidx = np.append(rmidx, delidx)
X = np.delete(temp[0], rmidx)
Y = np.delete(temp[1], rmidx)
#-- merge points
fingeridx = np.delete(np.arange(chk), np.append(X, Y))
finger = [cor[i] for i in fingeridx]
for i, j in zip(X, Y):
finger.append(tuple(np.add(cor[i], cor[j]) // 2))
return finger
elif chk == 1:
return cor
else:
return False
else:
return False
def run(self, one):
coun_max = 0
while True:
if not self.paused:
ret, self.frame = self.cap.read()
if not ret:
break
frame_gray = cv2.cvtColor(self.frame, cv2.COLOR_BGR2GRAY)
blurgray = cv2.GaussianBlur(frame_gray, (5, 5), 0)
ret, thresh1 = cv2.threshold(
blurgray, 70, 255, cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)
#5cv2.imshow('thresh',thresh1)
image, contours, hierarchy = cv2.findContours(
thresh1, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
drawing = np.zeros(self.frame.shape, np.uint8)
max_area = 0
ci = 0
for i in range(len(contours)):
cnt = contours[i]
area = cv2.contourArea(cnt)
if (area > max_area):
max_area = area
ci = i
cnt = contours[ci]
hull = cv2.convexHull(cnt)
moments = cv2.moments(cnt)
if moments['m00'] != 0:
cx = int(moments['m10'] / moments['m00']) # cx = M10/M00
cy = int(moments['m01'] / moments['m00']) # cy = M01/M00
centr = (cx, cy)
cv2.circle(self.frame, centr, 5, [0, 0, 255], 2)
cv2.drawContours(drawing, [cnt], 0, (0, 255, 0), 2)
cv2.drawContours(drawing, [hull], 0, (0, 0, 255), 2)
cnt = cv2.approxPolyDP(cnt, 0.01 * cv2.arcLength(cnt, True),
True)
hull = cv2.convexHull(cnt, returnPoints=False)
if (1):
defects = cv2.convexityDefects(cnt, hull)
mind = 0
maxd = 0
coun = 0
for i in range(defects.shape[0]):
coun = coun + 1
s, e, f, d = defects[i, 0]
start = tuple(cnt[s][0])
end = tuple(cnt[e][0])
far = tuple(cnt[f][0])
dist = cv2.pointPolygonTest(cnt, centr, True)
cv2.line(self.frame, start, end, [0, 255, 0], 2)
cv2.circle(self.frame, far, 5, [0, 0, 255], -1)
cv2.circle(self.frame, start, 5, [255, 0, 0], -1)
i = 0
font = cv2.FONT_HERSHEY_SIMPLEX
cv2.putText(self.frame, str(coun), (0, 40), font, 1,
(0, 0, 0), 2)
cv2.putText(self.frame, str(coun_max), (0, 80), font, 1,
(0, 0, 0), 2)
if (coun == coun_max + 1):
while (one == 0):
(x, y) = win32api.GetCursorPos()
win32api.mouse_event(
win32con.MOUSEEVENTF_RIGHTDOWN,
int(screenWidth / 2 - 2 * x), int(2 * y), 0, 0)
win32api.mouse_event(win32con.MOUSEEVENTF_RIGHTUP,
int(screenWidth / 2 - 2 * x),
int(2 * y), 0, 0)
end = 1
one = 1
if (coun == coun_max - 1):
while (one == 0):
(x, y) = win32api.GetCursorPos()
win32api.mouse_event(win32con.MOUSEEVENTF_LEFTDOWN,
int(screenWidth / 2 - 2 * x),
int(2 * y), 0, 0)
win32api.mouse_event(win32con.MOUSEEVENTF_LEFTUP,
int(screenWidth / 2 - 2 * x),
int(2 * y), 0, 0)
end = 1
one = 1
#if(coun == coun_max-2):
#while(one==0):
#(x,y) = win32api.GetCursorPos()
#win32api.mouse_event(MOUSEEVENTF_WHEEL, x, y, -1, 0)
#end=1
#one=1
if (coun == coun_max):
end = 0
one = 0
for tracker in self.trackers:
tracker.update(frame_gray)
vis = self.frame.copy()
for tracker in self.trackers:
tracker.draw_state(vis, 0)
if len(self.trackers) > 0:
cv2.imshow('tracker state', self.trackers[-1].state_vis)
self.rect_sel.draw(vis)
cv2.imshow('frame', vis)
ch = cv2.waitKey(10)
if ch == ord('s'):
coun_max = coun
a = cx - 80
b = cy - 80
c = cx + 80
d = cy + 80
frame_gray = cv2.cvtColor(self.frame, cv2.COLOR_BGR2GRAY)
tracker = MOSSE(frame_gray, (a, b, c, d))
self.trackers.append(tracker)
if ch == 27:
self.cap.release()
break
if ch == ord(' '):
self.paused = not self.paused
if ch == ord('c'):
self.trackers = []
coun_max = 0
ret, thresh_blue = cv2.threshold(bnw_blue, 0, 255,
cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)
fgmask_blue = fgbg.apply(thresh_blue)
blur_blue = cv2.GaussianBlur(fgmask_blue, (5, 5), 0)
contimage, contours_blue, hierarchy = cv2.findContours(
blur_blue, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
for i in range(len(contours_blue)):
cnt1 = np.array(contours_blue[i])
nothing
hull_blue = cv2.convexHull(cnt1, returnPoints=False)
defects_blue = cv2.convexityDefects(cnt1, hull_blue)
(x, y), radius_blue = cv2.minEnclosingCircle(cnt1)
center_blue = (int(x), int(y))
radius_blue = int(radius_blue)
rect_blue = cv2.minAreaRect(cnt1)
box_blue = cv2.boxPoints(rect_blue)
box_blue = np.int0(box_blue)
final_blue = cv2.circle(frame, center_blue, radius_blue, (0, 255, 0), 2)
final2_blue = cv2.circle(frame, center_blue, 10, (255, 0, 0), 2)
final3_blue = cv2.drawContours(frame, [box_blue], 0, (0, 0, 255), 2)
if contours_blue == empty:
if gesture == empty1:
pass
else:
main_gesture.append(gesture)
def analyze_video(self):
"""Detect the gestures of the person and add them to output
Press 'p' to pause the analysis
Press 'q' to quit the analysis (output will be modified anyway)
"""
cap = cv2.VideoCapture(self.path)
#out = cv2.VideoWriter('finalVideoKeypoints.avi', cv2.VideoWriter_fourcc(*'MJPG'), 25, (1280, 720))
if cap.isOpened() == False:
print("Error opening video file")
while cap.isOpened():
ret, frame = cap.read()
# Get hand data from a rectangle sub window
vid_points = {
0: {
'x': (150, 300),
'y': (100, 350)
},
1: {
'x': (300, 600),
'y': (50, 300)
},
2: {
'x': (50, 250),
'y': (100, 300)
}
}
pt1 = [
(vid_points[0]['x'][0], vid_points[0]['y'][0]),
(vid_points[1]['x'][0], vid_points[1]['y'][0]),
(vid_points[2]['x'][0], vid_points[2]['y'][0])
]
pt2 = [
(vid_points[0]['x'][1], vid_points[0]['y'][1]),
(vid_points[1]['x'][1], vid_points[1]['y'][1]),
(vid_points[2]['x'][1], vid_points[2]['y'][1])
]
try:
# First video: (0, 0), (640, 360)
# Second video: (640, 0), (1280, 360)
# Third video: (0', 360), (640, 720)
first_video = frame[0:360, 0:640]
second_video = frame[0:360, 640:1280]
third_video = frame[360:720, 0:640]
cv2.rectangle(first_video, pt1[0], pt2[0], (0, 255, 255), 3)
cv2.rectangle(second_video, pt1[1], pt2[1], (0, 255, 255), 3)
cv2.rectangle(third_video, pt1[2], pt2[2], (0, 255, 255), 3)
crop_image_1 = first_video[pt1[0][1]:pt2[0][1], pt1[0][0]:pt2[0][0]]
crop_image_2 = second_video[pt1[1][1]:pt2[1][1], pt1[1][0]:pt2[1][0]]
crop_image_3 = third_video[pt1[2][1]:pt2[2][1], pt1[2][0]:pt2[2][0]]
crop_image = [crop_image_1, crop_image_2, crop_image_3]
except:
print(self.output)
cap.release()
cv2.destroyAllWindows()
sys.exit(0)
blur_1 = cv2.GaussianBlur(crop_image_1, (3, 3), 0)
blur_2 = cv2.GaussianBlur(crop_image_2, (3, 3), 0)
blur_3 = cv2.GaussianBlur(crop_image_3, (3, 3), 0)
hsv_1 = cv2.cvtColor(blur_1, cv2.COLOR_BGR2HSV)
hsv_2 = cv2.cvtColor(blur_2, cv2.COLOR_BGR2HSV)
hsv_3 = cv2.cvtColor(blur_3, cv2.COLOR_BGR2HSV)
# Create a binary image with where white will be skin colors
# and rest is black
lowerb = np.array([0, 0, 0])
upperb = np.array([12, 255, 255])
mask_1 = cv2.inRange(hsv_1, lowerb, upperb)
mask_2 = cv2.inRange(hsv_2, lowerb, upperb)
mask_3 = cv2.inRange(hsv_3, lowerb, upperb)
kernel = np.ones((5, 5))
# Apply morphological transformations to filter out the background noise
dilation_1 = cv2.dilate(mask_1, kernel, iterations=1)
dilation_2 = cv2.dilate(mask_2, kernel, iterations=1)
dilation_3 = cv2.dilate(mask_3, kernel, iterations=1)
erosion_1 = cv2.erode(dilation_1, kernel, iterations=1)
erosion_2 = cv2.erode(dilation_2, kernel, iterations=1)
erosion_3 = cv2.erode(dilation_3, kernel, iterations=1)
filtered_1 = cv2.GaussianBlur(erosion_1, (3, 3), 0)
filtered_2 = cv2.GaussianBlur(erosion_2, (3, 3), 0)
filtered_3 = cv2.GaussianBlur(erosion_3, (3, 3), 0)
ret_1, thresh_1 = cv2.threshold(filtered_1, self.threshold[0], self.threshold[1], 0)
ret_2, thresh_2 = cv2.threshold(filtered_2, self.threshold[0], self.threshold[1], 0)
ret_3, thresh_3 = cv2.threshold(filtered_3, self.threshold[0], self.threshold[1], 0)
cv2.imshow("Thresholded 1", thresh_1)
cv2.imshow("Thresholded 2", thresh_2)
cv2.imshow("Thresholded 3", thresh_3)
# Find contours
contours_1, hierarchy_1 = cv2.findContours(thresh_1.copy(), cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
contours_2, hierarchy_2 = cv2.findContours(thresh_2.copy(), cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
contours_3, hierarchy_3 = cv2.findContours(thresh_3.copy(), cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
try:
# Find contour with maximum area
contour_1 = max(contours_1, key=lambda x: cv2.contourArea(x))
contour_2 = max(contours_2, key=lambda x: cv2.contourArea(x))
contour_3 = max(contours_3, key=lambda x: cv2.contourArea(x))
contour = [contour_1, contour_2, contour_3]
# Create bounding rectangle around the contour
x_1, y_1, w_1, h_1 = cv2.boundingRect(contour_1)
x_2, y_2, w_2, h_2 = cv2.boundingRect(contour_2)
x_3, y_3, w_3, h_3 = cv2.boundingRect(contour_3)
cv2.rectangle(crop_image_1, (x_1, y_1), (x_1 + w_1, y_1 + h_1), (255, 0, 255), 0)
cv2.rectangle(crop_image_2, (x_2, y_2), (x_2 + w_2, y_2 + h_2), (255, 0, 255), 0)
cv2.rectangle(crop_image_3, (x_3, y_3), (x_3 + w_3, y_3 + h_3), (255, 0, 255), 0)
hull_1 = cv2.convexHull(contour_1)
hull_2 = cv2.convexHull(contour_2)
hull_3 = cv2.convexHull(contour_3)
# Draw contour
drawing_1 = np.zeros(crop_image_1.shape, np.uint8)
drawing_2 = np.zeros(crop_image_2.shape, np.uint8)
drawing_3 = np.zeros(crop_image_3.shape, np.uint8)
cv2.drawContours(drawing_1, [contour_1], -1, (0, 255, 0), 0)
cv2.drawContours(drawing_2, [contour_2], -1, (0, 255, 0), 0)
cv2.drawContours(drawing_3, [contour_3], -1, (0, 255, 0), 0)
cv2.drawContours(drawing_1, [hull_1], -1, (0, 0, 255), 0)
cv2.drawContours(drawing_2, [hull_2], -1, (0, 0, 255), 0)
cv2.drawContours(drawing_2, [hull_3], -1, (0, 0, 255), 0)
# Find convexity defects
hull_1 = cv2.convexHull(contour_1, returnPoints=False)
hull_2 = cv2.convexHull(contour_2, returnPoints=False)
hull_3 = cv2.convexHull(contour_3, returnPoints=False)
defects_1 = cv2.convexityDefects(contour_1, hull_1)
defects_2 = cv2.convexityDefects(contour_2, hull_2)
defects_3 = cv2.convexityDefects(contour_3, hull_3)
defects = [defects_1, defects_2, defects_3]
# Use cosine rule to find the angle of the far point from the start
# and end point
count_defects = 0
for j in range(len(defects)):
for i in range(defects[j].shape[0]):
contour_temp = contour[j]
crop_image_temp = crop_image[j]
s, e, f, d = defects[j][i, 0]
start = tuple(contour_temp[s][0])
end = tuple(contour_temp[e][0])
far = tuple(contour_temp[f][0])
a = math.sqrt((end[0] - start[0]) ** 2 + (end[1] - start[1]) ** 2)
b = math.sqrt((far[0] - start[0]) ** 2 + (far[1] - start[1]) ** 2)
cv2.line(crop_image_temp, (start[0], start[1]), (far[0], far[1]), [0, 50, 120], 4)
c = math.sqrt((end[0] - far[0]) ** 2 + (end[1] - far[1]) ** 2)
cv2.line(crop_image_temp, (far[0], far[1]), (end[0], end[1]), [0, 50, 120], 4)
cv2.circle(crop_image_temp, far, 4, [255, 255, 255], -1)
angle = (math.acos((b ** 2 + c ** 2 - a ** 2) / (2 * b * c)) * 180) / math.pi
if angle <= 90:
count_defects += 1
cv2.circle(crop_image_temp, far, 4, [0, 0, 255], -1)
cv2.line(crop_image_temp, start, end, [255, 255, 0], 2)
current_time = time.time() - self.initial_time
first_three_sec = current_time > 3
for j in range(len(defects)):
cond_delay = current_time > self.gesture_delay[j]
# Print the number of fingers applying a delay between gesture
if cond_delay and first_three_sec: # or first_three_sec
if count_defects == 0:
self.text = "ONE"
self.output[str(j)]["one-finger"].append(round(time.time() - self.initial_time, 2))
self.gesture_delay[j] = (time.time()-self.initial_time) + 5
self.gestures_counter[j] += 1
elif count_defects == 1:
self.text = "TWO"
self.output[str(j)]["two-fingers"].append(round(time.time() - self.initial_time, 2))
self.gesture_delay[j] = (time.time()-self.initial_time) + 5
self.gestures_counter[j] += 1
elif count_defects == 2:
self.text = "THREE"
self.output[str(j)]["three-fingers"].append(round(time.time() - self.initial_time, 2))
self.gesture_delay[j] = (time.time()-self.initial_time) + 5
self.gestures_counter[j] += 1
#cv2.putText(frame, str(self.gestures_counter[j]) + ": " + self.text, self.text_position[j], cv2.FONT_HERSHEY_SIMPLEX, 2, (0, 0, 255), 2)
except:
pass
# Show required images
cv2.imshow("Gesture", frame)
out.write(frame)
if cv2.waitKey(1) == ord('q'):
break
elif cv2.waitKey(1) == ord('p'):
cv2.waitKey(-1)
cap.release()
cv2.destroyAllWindows()
self.return_values()
def gestures():
lowerBound = np.array([110, 150, 0])
upperBound = np.array([150, 225, 255])
cap = cv2.VideoCapture(0)
fgbg = cv2.createBackgroundSubtractorMOG2()
kernel = np.ones((3, 3), np.uint8)
while True:
#capturing a real time video
ret, frame = cap.read()
frame = cv2.flip(frame, 1)
#convert BGR to HSV
hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV)
# create the Mask
mask = cv2.inRange(hsv, lowerBound, upperBound)
#extrapolate the hand to fill dark spots within
mask1 = cv2.dilate(mask, kernel, iterations=2)
#combining the original bgr frame with masked frame
#result = cv2.bitwise_and(frame,frame,mask = mask1)
#cv2.imshow('frame', frame)
#cv2.imshow('mask', mask)
#cv2.imshow('result',result)
#subtracting background from the frames
#bgsub = fgbg.apply(mask)
#cv2.imshow('bgsub', bgsub)
#edge detection
#edges = cv2.Canny(mask1,100,200)
#cv2.imshow('Edges',edges)
#blur the image
#blur = cv2.GaussianBlur(edges,(5,5),100)
#cv2.imshow('blur', blur)
#find contours
contours = cv2.findContours(mask1, cv2.RETR_TREE,
cv2.CHAIN_APPROX_SIMPLE)
contours = imutils.grab_contours(contours)
max_area = -1
for i in range(len(contours)):
cnt = contours[i]
area = cv2.contourArea(cnt)
if (area > max_area):
max_area = area
ci = i
cnt = contours[ci]
cv2.drawContours(frame, [cnt], -1, (0, 255, 255), 2)
topmost = tuple(cnt[cnt[:, :, 1].argmin()][0])
bottommost = tuple(cnt[cnt[:, :, 1].argmax()][0])
leftmost = tuple(cnt[cnt[:, :, 0].argmin()][0])
rightmost = tuple(cnt[cnt[:, :, 0].argmax()][0])
cv2.circle(frame, topmost, 8, (0, 255, 0), -1)
cv2.circle(frame, bottommost, 8, (0, 255, 0), -1)
cv2.circle(frame, leftmost, 8, (0, 255, 0), -1)
cv2.circle(frame, rightmost, 8, (0, 255, 0), -1)
cv2.line(frame, bottommost, topmost, (0, 0, 255), 3)
cv2.line(frame, bottommost, leftmost, (0, 0, 255), 3)
cv2.line(frame, bottommost, rightmost, (0, 0, 255), 3)
x, y, w, h = cv2.boundingRect(cnt)
cv2.rectangle(frame, (x, y), (x + w, y + h), (0, 0, 255), 0)
hull = cv2.convexHull(cnt)
drawing = np.zeros(frame.shape, np.uint8)
cv2.drawContours(drawing, [cnt], 0, (0, 255, 0), 0)
cv2.drawContours(drawing, [hull], 0, (0, 0, 255), 0)
hull = cv2.convexHull(cnt, returnPoints=False)
defects = cv2.convexityDefects(cnt, hull)
count_defects = 0
cv2.drawContours(mask1, contours, -1, (0, 255, 0), 3)
for i in range(defects.shape[0]):
s, e, f, d = defects[i, 0]
start = tuple(cnt[s][0])
end = tuple(cnt[e][0])
far = tuple(cnt[f][0])
a = math.sqrt((end[0] - start[0])**2 + (end[1] - start[1])**2)
b = math.sqrt((far[0] - start[0])**2 + (far[1] - start[1])**2)
c = math.sqrt((end[0] - far[0])**2 + (end[1] - far[1])**2)
angle = math.acos((b**2 + c**2 - a**2) / (2 * b * c)) * 57
if angle <= 90:
count_defects += 1
cv2.circle(frame, far, 1, [0, 0, 255], -1)
#dist = cv2.pointPolygonTest(cnt,far,True)
cv2.line(frame, start, end, [0, 255, 0], 2)
if count_defects == 1:
cv2.putText(frame, "one", (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 2, 2)
elif count_defects == 2:
cv2.putText(frame, "Two", (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 2, 2)
elif count_defects == 3:
cv2.putText(frame, "Three", (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 2,
2)
elif count_defects == 4:
cv2.putText(frame, "Four", (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 2,
2)
elif count_defects == 5:
cv2.putText(frame, "Five", (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 2,
2)
elif count_defects == 0:
cv2.putText(frame, "Fist", (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 2,
2)
else:
cv2.putText(frame, "no hand detected", (50, 50),
cv2.FONT_HERSHEY_SIMPLEX, 2, 2)
cv2.imshow('frame', frame)
k = cv2.waitKey(5) & 0xFF
if k == 27:
break
#releasing the resource and destroying all windows
cap.release()
cv2.destroyAllWindows()
for i in range(len(contours)):
cnt = contours[i]
area = cv2.contourArea(cnt)
if (area > max_area):
max_area = area
ci = i
#Largest area contour
cnts = contours[ci]
#Find convex hull
hull = cv2.convexHull(cnts)
#Find convex defects
hull2 = cv2.convexHull(cnts, returnPoints=False)
defects = cv2.convexityDefects(cnts, hull2)
#Get defect points and draw them in the original image
FarDefect = []
for i in range(defects.shape[0]):
s, e, f, d = defects[i, 0]
start = tuple(cnts[s][0])
end = tuple(cnts[e][0])
far = tuple(cnts[f][0])
FarDefect.append(far)
cv2.line(frame, start, end, [0, 255, 0], 1)
cv2.circle(frame, far, 10, [100, 255, 255], 3)
#Find moments of the largest contour
moments = cv2.moments(cnts)
bigboi = max(contours, key=cv2.contourArea)
cv2.drawContours(frame, [bigboi], -1, (0, 255, 0), 3)
hull = cv2.convexHull(bigboi)
cv2.drawContours(frame, [hull], -1, (255, 0, 255), 3)
# print(hull)
# for h in hull:
# try:
# x = h[0][0]
# y = h[0][1]
# print("x: ", x, "y: ", y)
# cv2.circle(frame, (x, y), 3, (40, 10, 40), -1)
# except:
# pass
defects = cv2.convexityDefects(bigboi, cv2.convexHull(bigboi, returnPoints=False))
num_defect = 0
for i in range(defects.shape[0]):
s, e, f, d = defects[i, 0]
start = tuple(bigboi[s][0])
end = tuple(bigboi[e][0])
far = tuple(bigboi[f][0])
a = math.sqrt((end[0] - start[0]) ** 2 + (end[1] - start[1]) ** 2)
b = math.sqrt((far[0] - start[0]) ** 2 + (far[1] - start[1]) ** 2)
c = math.sqrt((end[0] - far[0]) ** 2 + (end[1] - far[1]) ** 2)
angle = math.acos((b ** 2 + c ** 2 - a ** 2) / (2 * b * c)) * 57
if angle <= 60:
area = cv2.contourArea(cnt)
if area > max_area:
max_area = area
ci = i
cnt = contours[ci]
x, y, w, h = cv2.boundingRect(cnt)
cv2.rectangle(crop_img, (x, y), (x + w, y + h), (0, 0, 255), 0)
hull = cv2.convexHull(cnt)
drawing = np.zeros(crop_img.shape, np.uint8)
cv2.drawContours(drawing, [cnt], 0, (0, 255, 0), 0)
cv2.drawContours(drawing, [hull], 0, (0, 0, 255), 0)
# to construct a convex hull of given coordinates
hull = cv2.convexHull(cnt, returnPoints=False)
defects = cv2.convexityDefects(cnt, hull)
count_defects = 0
cv2.drawContours(thresh1, contours, -1, (0, 255, 0), 3)
used_defect = None
for i in range(defects.shape[0]):
s, e, f, d = defects[i, 0]
start = tuple(cnt[s][0])
end = tuple(cnt[e][0])
far = tuple(cnt[f][0])
# importing math library using maths function
a = math.sqrt((end[0] - start[0])**2 + (end[1] - start[1])**2)
b = math.sqrt((far[0] - start[0])**2 + (far[1] - start[1])**2)
c = math.sqrt((end[0] - far[0])**2 + (end[1] - far[1])**2)
angle = math.acos((b**2 + c**2 - a**2) / (2 * b * c)) * 57
def main():
UDP_IP = "127.0.0.1"
####HAND
UDP_HAND_PORT = 5070
SHOTS_PER_SECOND = 0.5
SHOOT_WAIT_TIME = False
POWER_ACTIVE_TIME = 5.0
POWER_WAIT_TIME = 5.0
POWER_ENABLED = False
POWER_WAIT_ENABLED = False
power_start_time = 0
power_start_wait_time = 0
print("UDP target IP:", UDP_IP)
print("UDP hand target port:", UDP_HAND_PORT)
sock = socket.socket(
socket.AF_INET, # Internet
socket.SOCK_DGRAM) # UDP
video_font = 'http://192.168.1.136:4747/video'
cap = cv2.VideoCapture(video_font)
screenColor = ScreenColor(cap, "yellow", sock)
# recieve from unity
sock_recieve = threading.Thread(target=screenColor.recv_socket)
sock_recieve.start()
#cap = cv2.VideoCapture(0)
while (cap.isOpened()):
ret, img = cap.read()
cv2.rectangle(img, (200, 150), (50, 300), (0, 255, 0), 0)
crop_img = img[150:300, 50:200]
grey = cv2.cvtColor(crop_img, cv2.COLOR_BGR2GRAY)
value = (35, 35)
blurred = cv2.GaussianBlur(grey, value, 0)
_, thresh1 = cv2.threshold(blurred, 127, 255,
cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)
cv2.imshow('Thresholded', thresh1)
contours, hierarchy = cv2.findContours(thresh1.copy(),cv2.RETR_TREE, \
cv2.CHAIN_APPROX_NONE)
max_area = -1
for i in range(len(contours)):
cnt = contours[i]
area = cv2.contourArea(cnt)
if (area > max_area):
max_area = area
ci = i
cnt = contours[ci]
x, y, w, h = cv2.boundingRect(cnt)
cv2.rectangle(crop_img, (x, y), (x + w, y + h), (0, 0, 255), 0)
hull = cv2.convexHull(cnt)
drawing = np.zeros(crop_img.shape, np.uint8)
cv2.drawContours(drawing, [cnt], 0, (0, 255, 0), 0)
cv2.drawContours(drawing, [hull], 0, (0, 0, 255), 0)
hull = cv2.convexHull(cnt, returnPoints=False)
defects = cv2.convexityDefects(cnt, hull)
count_defects = 0
cv2.drawContours(thresh1, contours, -1, (0, 255, 0), 3)
try:
ans = len(defects.shape) >= 0
except:
ans = 0
if not ans:
continue
for i in range(defects.shape[0]):
s, e, f, d = defects[i, 0]
start = tuple(cnt[s][0])
end = tuple(cnt[e][0])
far = tuple(cnt[f][0])
a = math.sqrt((end[0] - start[0])**2 + (end[1] - start[1])**2)
b = math.sqrt((far[0] - start[0])**2 + (far[1] - start[1])**2)
c = math.sqrt((end[0] - far[0])**2 + (end[1] - far[1])**2)
angle = math.acos((b**2 + c**2 - a**2) / (2 * b * c)) * 57
if angle <= 90:
count_defects += 1
cv2.circle(crop_img, far, 1, [0, 0, 255], -1)
#dist = cv2.pointPolygonTest(cnt,far,True)
cv2.line(crop_img, start, end, [0, 255, 0], 2)
#cv2.circle(crop_img,far,5,[0,0,255],-1)
if count_defects >= 4:
cv2.putText(img, "DISPARAAAA", (50, 50), cv2.FONT_HERSHEY_SIMPLEX,
2, 2)
if not SHOOT_WAIT_TIME: #shoot wait not activated
start_time = time.perf_counter()
SHOOT_WAIT_TIME = True
info = "100"
else: #shoot wait activated
end_time = time.perf_counter()
if (end_time -
start_time) >= SHOTS_PER_SECOND: #can shoot again
SHOOT_WAIT_TIME = False
else:
info = "0"
else:
cv2.putText(img, "NO HACER NADA", (50, 50),
cv2.FONT_HERSHEY_SIMPLEX, 2, 2)
info = "0"
#print("Info", info)
send_socket(info, sock, UDP_IP, UDP_HAND_PORT)
#cv2.imshow('drawing', drawing)
#cv2.imshow('end', crop_img)
#cv2.imshow('Gesture', img)
all_img = np.hstack((drawing, crop_img))
cv2.imshow('Contours', all_img)
k = cv2.waitKey(10)
if k == 27:
break
if not POWER_ENABLED:
cv2.putText(img, "NO PODER", (250, 250), cv2.FONT_HERSHEY_SIMPLEX,
2, 2)
elif POWER_WAIT_ENABLED:
cv2.putText(img, "NO SE PUEDE ACTIVAR PODER", (250, 250),
cv2.FONT_HERSHEY_SIMPEX, 2, 2)
else:
cv2.putText(img, "PODER", (250, 250), cv2.FONT_HERSHEY_SIMPLEX, 2,
2)
#### SCREEN COLOR #####
screenColor.loop(img)
if screenColor.activate_power():
if not POWER_WAIT_ENABLED and not POWER_ENABLED: # can use power
power_start_time = time.perf_counter()
POWER_ENABLED = True
cv2.putText(img, "PODER", (250, 250), cv2.FONT_HERSHEY_SIMPLEX,
2, 2)
SHOTS_PER_SECOND = 0.25
info = "400" # Power IS activated
send_socket(info, sock, UDP_IP, UDP_HAND_PORT)
# Check power duration
power_end_time = time.perf_counter()
if (POWER_ENABLED and int(power_start_time) != 0
and (power_end_time - power_start_time) >=
POWER_ACTIVE_TIME): # power over
POWER_ENABLED = False
SHOTS_PER_SECOND = 0.5
info = "200" # Cant activate power
send_socket(info, sock, UDP_IP, UDP_HAND_PORT)
POWER_WAIT_ENABLED = True
power_start_wait_time = time.perf_counter()
# Enable Power
if POWER_WAIT_ENABLED:
# Check if power can be enabled
power_end_wait_time = time.perf_counter()
if (power_end_wait_time -
power_start_wait_time) >= POWER_WAIT_TIME:
POWER_WAIT_ENABLED = False
info = "300" # Can activate power
send_socket(info, sock, UDP_IP, UDP_HAND_PORT)
# Tạo hình chữ nhật giới hạn xung quanh đường viền
x, y, w, h = cv2.boundingRect(contour)
cv2.rectangle(crop_image, (x, y), (x + w, y + h), (0, 0, 255), 0)
# Tìm vỏ lồi
hull = cv2.convexHull(contour)
# Vẽ đường viền
drawing = np.zeros(crop_image.shape, np.uint8)
cv2.drawContours(drawing, [contour], -1, (0, 255, 0), 0)
cv2.drawContours(drawing, [hull], -1, (0, 0, 255), 0)
# Khiếm khuyết lồi Fi
hull = cv2.convexHull(contour, returnPoints=False)
defects = cv2.convexityDefects(contour, hull)
# Sử dụng quy tắc cosine để tìm góc của điểm xa từ điểm bắt đầu và điểm kết thúc, tức là các điểm lồi (ngón tay
count_defects = 0
for i in range(defects.shape[0]):
s, e, f, d = defects[i, 0]
start = tuple(contour[s][0])
end = tuple(contour[e][0])
far = tuple(contour[f][0])
a = math.sqrt((end[0] - start[0])**2 + (end[1] - start[1])**2)
b = math.sqrt((far[0] - start[0])**2 + (far[1] - start[1])**2)
c = math.sqrt((end[0] - far[0])**2 + (end[1] - far[1])**2)
angle = (math.acos(
(b**2 + c**2 - a**2) / (2 * b * c)) * 180) / 3.14
def convex_hull(frame_1, frame):
frame = cv2.GaussianBlur(frame, (5, 5), 100)
frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
ret, thresh = cv2.threshold(frame, 30, 255, cv2.THRESH_BINARY)
contours, hierarchy = cv2.findContours(thresh, cv2.RETR_TREE,
cv2.CHAIN_APPROX_SIMPLE)
cnt = max(contours, key=lambda x: cv2.contourArea(x))
epsilon = 0.0005 * cv2.arcLength(cnt, True)
approx = cv2.approxPolyDP(cnt, epsilon, True)
hull = cv2.convexHull(cnt)
areahull = cv2.contourArea(hull)
areacnt = cv2.contourArea(cnt)
arearatio = ((areahull - areacnt) / areacnt) * 100
hull = cv2.convexHull(approx, returnPoints=False)
defects = cv2.convexityDefects(approx, hull)
l = 0
frame_x = frame_1.copy()
for i in range(defects.shape[0]):
s, e, f, d = defects[i, 0]
start = tuple(approx[s][0])
end = tuple(approx[e][0])
far = tuple(approx[f][0])
pt = (100, 180)
# find length of all sides of triangle
a = math.sqrt((end[0] - start[0])**2 + (end[1] - start[1])**2)
b = math.sqrt((far[0] - start[0])**2 + (far[1] - start[1])**2)
c = math.sqrt((end[0] - far[0])**2 + (end[1] - far[1])**2)
s = (a + b + c) / 2
ar = math.sqrt(s * (s - a) * (s - b) * (s - c))
#distance between point and convex hull
d = (2 * ar) / a
# apply cosine rule here
angle = math.acos((b**2 + c**2 - a**2) / (2 * b * c)) * 57
# ignore angles > 90 and ignore points very close to convex hull(they generally come due to noise)
if angle <= 90 and d > 30:
l += 1
cv2.circle(frame_x, far, 3, [255, 0, 0], -1)
#draw lines around hand
cv2.line(frame_x, start, end, [0, 255, 0], 2)
l += 1
font = cv2.FONT_HERSHEY_SIMPLEX
if l == 1:
if areacnt < 2000:
cv2.putText(frame_x, 'Put hand in the box', (0, 50), font, 2,
(0, 0, 255), 3, cv2.LINE_AA)
else:
if arearatio < 12:
cv2.putText(frame_x, '0', (0, 50), font, 2, (0, 0, 255), 3,
cv2.LINE_AA)
elif arearatio < 17.5:
cv2.putText(frame_x, 'Best of luck', (0, 50), font, 2,
(0, 0, 255), 3, cv2.LINE_AA)
else:
cv2.putText(frame_x, '1', (0, 50), font, 2, (0, 0, 255), 3,
cv2.LINE_AA)
elif l == 2:
cv2.putText(frame_x, '2', (0, 50), font, 2, (0, 0, 255), 3,
cv2.LINE_AA)
elif l == 3:
if arearatio < 27:
cv2.putText(frame_x, '3', (0, 50), font, 2, (0, 0, 255), 3,
cv2.LINE_AA)
else:
cv2.putText(frame_x, 'ok', (0, 50), font, 2, (0, 0, 255), 3,
cv2.LINE_AA)
elif l == 4:
cv2.putText(frame_x, '4', (0, 50), font, 2, (0, 0, 255), 3,
cv2.LINE_AA)
elif l == 5:
cv2.putText(frame_x, '5', (0, 50), font, 2, (0, 0, 255), 3,
cv2.LINE_AA)
elif l == 6:
cv2.putText(frame_x, 'reposition', (0, 50), font, 2, (0, 0, 255), 3,
cv2.LINE_AA)
else:
cv2.putText(frame_x, 'reposition', (10, 50), font, 2, (0, 0, 255), 3,
cv2.LINE_AA)
return frame_x
def dedoscamp():
cap = cv2.VideoCapture(0)
bg = None
while True:
ret, frame = cap.read()
if ret == False:
break
# Redimensionar la imagen para que tenga un ancho de 640
frame = imutils.resize(frame, width=640)
frame = cv2.flip(frame, 1)
frameAux = frame.copy()
if bg is not None:
# Determinar la región de interés
ROI = frame[50:300, 380:600]
cv2.rectangle(frame, (380 - 2, 50 - 2), (600 + 2, 300 + 2),
color_dedos, 1)
grayROI = cv2.cvtColor(ROI, cv2.COLOR_BGR2GRAY)
# Región de interés del fondo de la imagen
bgROI = bg[50:300, 380:600]
# Determinar la imagen binaria (background vs foreground)
dif = cv2.absdiff(grayROI, bgROI)
_, th = cv2.threshold(dif, 30, 255, cv2.THRESH_BINARY)
th = cv2.medianBlur(th, 7)
# Encontrando los contornos de la imagen binaria
cnts, _ = cv2.findContours(th, cv2.RETR_EXTERNAL,
cv2.CHAIN_APPROX_SIMPLE)
cnts = sorted(cnts, key=cv2.contourArea, reverse=True)[:1]
for cnt in cnts:
# Encontrar el centro del contorno
M = cv2.moments(cnt)
if M["m00"] == 0:
M["m00"] = 1
x = int(M["m10"] / M["m00"])
y = int(M["m01"] / M["m00"])
cv2.circle(ROI, tuple([x, y]), 5, (0, 255, 0), -1)
# Punto más alto del contorno
ymin = cnt.min(axis=1)
cv2.circle(ROI, tuple(ymin[0]), 5, color_ymin, -1)
# Contorno encontrado a través de cv2.convexHull
hull1 = cv2.convexHull(cnt)
cv2.drawContours(ROI, [hull1], 0, color_contorno, 2)
# Defectos convexos
hull2 = cv2.convexHull(cnt, returnPoints=False)
defects = cv2.convexityDefects(cnt, hull2)
# Seguimos con la condición si es que existen defectos convexos
if defects is not None:
inicio = [
] # Contenedor en donde se almacenarán los puntos iniciales de los defectos convexos
fin = [
] # Contenedor en donde se almacenarán los puntos finales de los defectos convexos
dedos = 0 # Contador para el número de dedos levantados
for i in range(defects.shape[0]):
s, e, f, d = defects[i, 0]
start = cnt[s][0]
end = cnt[e][0]
far = cnt[f][0]
# Encontrar el triángulo asociado a cada defecto convexo para determinar ángulo
a = np.linalg.norm(far - end)
b = np.linalg.norm(far - start)
c = np.linalg.norm(start - end)
angulo = np.arccos(
(np.power(a, 2) + np.power(b, 2) - np.power(c, 2))
/ (2 * a * b))
angulo = np.degrees(angulo)
angulo = int(angulo)
# Se descartarán los defectos convexos encontrados de acuerdo a la distnacia
# entre los puntos inicial, final y más alelago, por el ángulo y d
if np.linalg.norm(start - end
) > 20 and angulo < 90 and d > 12000:
# Almacenamos todos los puntos iniciales y finales que han sido
# obtenidos
inicio.append(start)
fin.append(end)
# Visualización de distintos datos obtenidos
cv2.circle(ROI, tuple(start), 5, color_comienzo, 2)
cv2.circle(ROI, tuple(end), 5, color_terminacion,
2)
cv2.circle(ROI, tuple(far), 7, color_far, -1)
# Si no se han almacenado puntos de inicio (o fin), puede tratarse de
# 0 dedos levantados o 1 dedo levantado
if len(inicio) == 0:
minY = np.linalg.norm(ymin[0] - [x, y])
if minY >= 110:
dedos = dedos + 1
cv2.putText(ROI, '{}'.format(dedos),
tuple(ymin[0]), 1, 1.7, (color_dedos),
1, cv2.LINE_AA)
# Si se han almacenado puntos de inicio, se contará el número de dedos levantados
for i in range(len(inicio)):
dedos = dedos + 1
cv2.putText(ROI, '{}'.format(dedos), tuple(inicio[i]),
1, 1.7, (color_dedos), 1, cv2.LINE_AA)
if i == len(inicio) - 1:
dedos = dedos + 1
cv2.putText(ROI, '{}'.format(dedos), tuple(fin[i]),
1, 1.7, (color_dedos), 1, cv2.LINE_AA)
# Se visualiza el número de dedos levantados en el rectángulo izquierdo
cv2.putText(frame, '{}'.format(dedos), (390, 45), 1, 4,
(color_dedos), 2, cv2.LINE_AA)
if dedos != 0:
if dedos == 1:
rigth = True
left = False
down = False
up = False
action = False
if dedos == 2:
rigth = False
left = True
down = False
up = False
action = False
if dedos == 3:
rigth = False
left = False
down = True
up = False
action = False
if dedos == 4:
rigth = False
left = False
down = False
up = True
action = False
if dedos == 5:
rigth = False
left = False
down = False
up = False
action = True
print(rigth)
print(left)
print(down)
print(up)
print(action)
cv2.imshow('th', th)
cv2.imshow('Frame', frame)
k = cv2.waitKey(20)
if k == ord('i'):
bg = cv2.cvtColor(frameAux, cv2.COLOR_BGR2GRAY)
if k == 27:
break
cap.release()
cv2.destroyAllWindows()
def extractDefects(contour, hull):
return cv2.convexityDefects(contour, hull)
def main():
#subprocess.Popen(["bash", "ultimate.sh"])
pygame.init()
song = pygame.mixer.Sound('A.wav')
clock = pygame.time.Clock()
sound_tick = 0
var_A = 0
# Testing Bash Script in python
# subprocess.call("test_bash.sh")
def createFile(dest):
name = letter_correspond
if not(path.isfile(dest + name)):
f = open(dest + name, 'w')
f.write('\n'*5)
f.close()
co_ordinates_for_CONVEX_HULL = [[]]
abc = []
font = cv2.FONT_HERSHEY_SIMPLEX
put_text_color = (18, 0, 255)
put_text_pos = (60, 50)
lower_thresh1 = 129
upper_thresh1 = 255
j = 0
PI = math.pi
"""img_test = cv2.imread("1.jpg",0)
ret_1,thresh_test = cv2.threshold(img_test,125,255,0)
xyz_test,contours_test,hierarchy_test = cv2.findContours(thresh_test,2,1)
cnt_test_yo = [4]
ret_1 = cv2.matchShapes(cnt_test_yo,cnt,1,0.0)
print ret_1"""
#letter_I_match_shape = cv2.imread('I.jpg',0)
camera_index = int(sys.argv[1])
cap = cv2.VideoCapture(camera_index)
while(cap.isOpened()):
img_dofh = cv2.imread("D.png", 0)
ret, img = cap.read()
cv2.rectangle(img, (60, 60), (300, 300),
(255, 255, 2), 4) # outer most rectangle
crop_img = img[70:300, 70:300]
crop_img_2 = img[70:300, 70:300]
grey = cv2.cvtColor(crop_img, cv2.COLOR_BGR2GRAY)
#edges = cv2.Canny(crop_img,100,200)
# Corners Detection
hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
lower_red = np.array([0, 150, 50])
upper_red = np.array([195, 255, 255])
mask = cv2.inRange(hsv, lower_red, upper_red)
res = cv2.bitwise_and(img, img, mask=mask)
value = (35, 35)
blurred = cv2.GaussianBlur(grey, value, 0)
_, thresh1 = cv2.threshold(
blurred, lower_thresh1, upper_thresh1, cv2.THRESH_BINARY_INV+cv2.THRESH_OTSU)
_, thresh_dofh = cv2.threshold(
img_dofh, lower_thresh1, upper_thresh1, cv2.THRESH_BINARY_INV+cv2.THRESH_OTSU)
#defrt,thresh_for_I = cv2.threshold(letter_I_match_shape,125,255,0)
contours, hierarchy = cv2.findContours(
thresh1.copy(), cv2.RETR_TREE, cv2.CHAIN_APPROX_NONE)
contours_dofh, hierarchy_dofh = cv2.findContours(
thresh_dofh, 2, 1)
#cnt_dofh = contours_dofh[1]
cnt = max(contours, key=lambda x: cv2.contourArea(x))
#cnt_m = contours[0]
# print cnt_m
area_of_contour = cv2.contourArea(cnt)
x, y, w, h = cv2.boundingRect(cnt)
cv2.rectangle(crop_img, (x, y), (x+w, y+h),
(0, 0, 255), 1) # Red Rectangle
#ratio_of_red_rect = h/w
# print ratio_of_red_rect
hull = cv2.convexHull(cnt)
drawing = np.zeros(crop_img.shape, np.uint8)
cv2.drawContours(drawing, [cnt], 0, (0, 255, 0), 0)
cv2.drawContours(drawing, [hull], 0, (0, 255, 255), 0)
hull = cv2.convexHull(cnt, returnPoints=False)
#co_ordinates_for_CONVEX_HULL = cv2.convexHull(cnt,returnPoints = True)
defects = cv2.convexityDefects(cnt, hull)
count_defects = 0
cv2.drawContours(thresh1, contours, -1, (0, 255, 0), 3)
for i in range(defects.shape[0]):
s, e, f, d = defects[i, 0]
start = tuple(cnt[s][0])
end = tuple(cnt[e][0])
far = tuple(cnt[f][0])
a = math.sqrt((end[0] - start[0])**2 + (end[1] - start[1])**2)
b = math.sqrt((far[0] - start[0])**2 + (far[1] - start[1])**2)
c = math.sqrt((end[0] - far[0])**2 + (end[1] - far[1])**2)
angle = math.acos((b**2 + c**2 - a**2)/(2*b*c)) * 60
# Red Circles###########################3
cv2.circle(crop_img, far, 4, [0, 0, 255], -1)
# cv2.line(crop_img,start,end,[255,255,0],2)
if angle <= 90:
count_defects += 1
#yeyo = cv2.circle(crop_img,far,1,[255,0,0],-1)
cv2.line(crop_img, start, end, [0, 255, 0], 3)
moment = cv2.moments(cnt)
perimeter = cv2.arcLength(cnt, True)
area = cv2.contourArea(cnt)
(x, y), radius = cv2.minEnclosingCircle(cnt)
center = (int(x), int(y))
radius = int(radius)
cv2.circle(crop_img, center, radius, (255, 0, 0), 2)
area_of_circle = PI*radius*radius
hull_test = cv2.convexHull(cnt)
hull_area = cv2.contourArea(hull_test)
solidity = float(area)/hull_area
aspect_ratio = float(w)/h
rect_area = w*h
extent = float(area)/rect_area
(x, y), (MA, ma), angle_t = cv2.fitEllipse(cnt)
#ret_1 = cv2.matchShapes(cnt,contours_dofh,1,0.0)
# print ret_1
if area_of_circle - area < 5000:
letter_correspond = "A.txt"
destination = 'Letters_stash_for_sounds/'
createFile(destination)
# raw_input("done!!!")
output = "A"
outFile = open('Letters_stash_for_sounds/A.txt', 'w')
outFile.write(output)
cv2.putText(img, "The Letter is : A (CALIBRATED)",
put_text_pos, font, 1, put_text_color, 2, cv2.LINE_AA)
#cv2.putText(img, "The letter is :", (60,50), font, 1 , put_text_color, 2, cv2.LINE_AA)
#cv2.putText(img, "A", (320,55), font, 2 , (50,100,190), 3, cv2.LINE_AA)
elif count_defects == 1:
if angle_t < 10:
letter_correspond = "V.txt"
destination = 'Letters_stash_for_sounds/'
createFile(destination)
# raw_input("done!!!")
output = "V"
outFile = open('Letters_stash_for_sounds/V.txt', 'w')
outFile.write(output)
#letter_correspond = "V.txt"
cv2.putText(img, "The Letter is : V", put_text_pos,
font, 1, put_text_color, 2, cv2.LINE_AA)
elif 40 < angle_t < 66:
letter_correspond = "C.txt"
destination = 'Letters_stash_for_sounds/'
createFile(destination)
# raw_input("done!!!")
output = "C"
outFile = open('Letters_stash_for_sounds/C.txt', 'w')
outFile.write(output)
cv2.putText(img, "The Letter is : C", put_text_pos,
font, 1, put_text_color, 2, cv2.LINE_AA)
elif 20 < angle_t < 35:
letter_correspond = "L.txt"
destination = 'Letters_stash_for_sounds/'
createFile(destination)
# raw_input("done!!!")
output = "L"
outFile = open('Letters_stash_for_sounds/L.txt', 'w')
outFile.write(output)
cv2.putText(img, "The Letter is : L", put_text_pos,
font, 1, put_text_color, 2, cv2.LINE_AA)
else:
letter_correspond = "Y.txt"
destination = 'Letters_stash_for_sounds/'
createFile(destination)
# raw_input("done!!!")
output = "Y"
outFile = open('Letters_stash_for_sounds/Y.txt', 'w')
outFile.write(output)
cv2.putText(img, "The Letter is : Y", put_text_pos,
font, 1, put_text_color, 2, cv2.LINE_AA)
# print "Its 2"
elif count_defects == 2: # Its either W or F
if angle_t > 100:
letter_correspond = "F.txt"
destination = 'Letters_stash_for_sounds/'
createFile(destination)
# raw_input("done!!!")
output = "F"
outFile = open('Letters_stash_for_sounds/F.txt', 'w')
outFile.write(output)
cv2.putText(img, "The Letter is : F", put_text_pos,
font, 1, put_text_color, 2, cv2.LINE_AA)
else:
letter_correspond = "W.txt"
destination = 'Letters_stash_for_sounds/'
createFile(destination)
# raw_input("done!!!")
output = "W"
outFile = open('Letters_stash_for_sounds/W.txt', 'w')
outFile.write(output)
cv2.putText(img, "The Letter is : W", put_text_pos,
font, 1, put_text_color, 2, cv2.LINE_AA)
elif count_defects == 4:
letter_correspond = "CALIBRATE.txt"
destination = 'Letters_stash_for_sounds/'
createFile(destination)
# raw_input("done!!!")
output = "CALIBRATE"
outFile = open('Letters_stash_for_sounds/CALIBRATE.txt', 'w')
outFile.write(output)
cv2.putText(img, "Hello There ! Callibrate by letter A",
put_text_pos, font, 1, put_text_color, 2, cv2.LINE_AA)
else:
if area > 12000:
letter_correspond = " B.txt"
destination = 'Letters_stash_for_sounds/'
createFile(destination)
# raw_input("done!!!")
output = "B"
outFile = open('Letters_stash_for_sounds/B.txt', 'w')
outFile.write(output)
cv2.putText(img, "The Letter is : B", put_text_pos,
font, 1, put_text_color, 2, cv2.LINE_AA)
else:
if solidity < 0.85:
if aspect_ratio < 1:
if angle_t < 20:
letter_correspond = " D.txt"
destination = 'Letters_stash_for_sounds/'
createFile(destination)
# raw_input("done!!!")
output = "D"
outFile = open(
'Letters_stash_for_sounds/D.txt', 'w')
outFile.write(output)
cv2.putText(img, "The Letter is : D", put_text_pos,
font, 1, put_text_color, 2, cv2.LINE_AA)
elif 169 < angle_t < 180:
letter_correspond = " I.txt"
destination = 'Letters_stash_for_sounds/'
createFile(destination)
# raw_input("done!!!")
output = "I"
outFile = open(
'Letters_stash_for_sounds/I.txt', 'w')
outFile.write(output)
cv2.putText(img, "The Letter is : I", put_text_pos,
font, 1, put_text_color, 2, cv2.LINE_AA)
elif angle_t < 168:
letter_correspond = " J.txt"
destination = 'Letters_stash_for_sounds/'
createFile(destination)
# raw_input("done!!!")
output = "J"
outFile = open(
'Letters_stash_for_sounds/J.txt', 'w')
outFile.write(output)
cv2.putText(img, "The Letter is : J", put_text_pos,
font, 1, put_text_color, 2, cv2.LINE_AA)
elif aspect_ratio > 1.01:
letter_correspond = " Y.txt"
destination = 'Letters_stash_for_sounds/'
createFile(destination)
# raw_input("done!!!")
output = "Y"
outFile = open('Letters_stash_for_sounds/Y.txt', 'w')
outFile.write(output)
cv2.putText(img, "The Letter is : Y", put_text_pos,
font, 1, put_text_color, 2, cv2.LINE_AA)
else:
if angle_t > 30 and angle_t < 100:
letter_correspond = " H.txt"
destination = 'Letters_stash_for_sounds/'
createFile(destination)
# raw_input("done!!!")
output = "H"
outFile = open('Letters_stash_for_sounds/H.txt', 'w')
outFile.write(output)
cv2.putText(img, "The Letter is : H", put_text_pos,
font, 1, put_text_color, 2, cv2.LINE_AA)
elif angle_t > 120:
letter_correspond = " I.txt"
destination = 'Letters_stash_for_sounds/'
createFile(destination)
# raw_input("done!!!")
output = "I"
outFile = open('Letters_stash_for_sounds/I.txt', 'w')
outFile.write(output)
cv2.putText(img, "The Letter is : I", put_text_pos,
font, 1, put_text_color, 2, cv2.LINE_AA)
else:
letter_correspond = " U.txt"
destination = 'Letters_stash_for_sounds/'
createFile(destination)
# raw_input("done!!!")
output = "U"
outFile = open('Letters_stash_for_sounds/U.txt', 'w')
outFile.write(output)
cv2.putText(img, "The Letter is : U", put_text_pos,
font, 1, put_text_color, 2, cv2.LINE_AA)
# crop_img[dst>0.01*dst.max()]=[255,0,0]
cv2.imshow('Gesture', img)
cv2.imshow('Contours', drawing)
cv2.imshow('Defects', crop_img)
cv2.imshow('Binary Image', thresh1)
# Testing Parameters :
# print co_ordinates_for_CONVEX_HULL
# print abc
# cv2.imshow('something',yeyo)
# print area_of_contour,co_ordinates_for_CONVEX_HULL
# print moment
print("Perimeter is :", perimeter)
# print "The area is:", area
# print defects
# print crop_img.shape
# print defects.shape
# print dist_from_farthest
# print count_defectsd
# print hull
# print far
# print equi_diameter
print('Radius:', PI*radius*radius, "Area:", area)
print(area_of_circle - area)
print("The solidity is:", solidity)
print("The aspect ratio is :", aspect_ratio)
print("The number of convexity defects are :", count_defects)
print("The extent is :", extent)
print("the angle is:", angle_t)
print('\033[91m'+"The area of effective circle is " + '\033[0m', area_of_circle - area)
"""
img1 = cv2.imread('match_shapes_test.png',0)
#img2 = cv2.imread(thresh1,0)
ret, thresh_8 = cv2.threshold(img1, 127, 255,0)
ret, thresh_9 = cv2.threshold(thresh1, 127, 255,0)
xcv,contours_8,hierarchy = cv2.findContours(thresh_8,2,1)
cnt1 = contours_8[0]
vbgf,contours_9,hierarchy = cv2.findContours(thresh_9,2,1)
cnt_9 = max(contours_9, key = lambda s: cv2.contourArea(s))
ret = cv2.matchShapes(cnt1,cnt_9,1,0.0)
print "The probability is :",ret """
k = cv2.waitKey(10) & 0xFF
# if k == ord('s'):
# cv2.imwrite('testing_match.png',crop_img_2)
if k == ord('s'):
os.system(
"gnome-terminal -e 'bash -c \"bash ultimate.sh; exec bash\"'")
pid = os.getpid()
elif k == 27 or k == ord('q'):
break
def getConvexHull(self):
hullPoints = cv2.convexHull(self.mainContour)
hull = cv2.convexHull(self.mainContour, hullPoints, False, False)
defects = cv2.convexityDefects(self.mainContour, hull.flatten())
return hullPoints, hull, defects
def PiGestureDetect():
camera = PiCamera()
camera.resolution = (640, 480)
camera.framerate = 15
camera.vflip = True
rawCapture = PiRGBArray(camera, size=(640, 480))
fgbg = cv2.createBackgroundSubtractorMOG2(varThreshold=80,
detectShadows=False)
while camera.analog_gain <= 1:
time.sleep(0.1)
camera.shutter_speed = camera.exposure_speed
camera.exposure_mode = 'off'
g = camera.awb_gains
camera.awb_mode = 'off'
camera.awb_gains = g
camera.capture(rawCapture, format="bgr")
img_background = rawCapture.array
rawCapture.truncate(0)
for frame in camera.capture_continuous(rawCapture,
format="bgr",
use_video_port=True):
image = frame.array
#Color segmentation:
#gray_image = cv2.cvtColor(image,cv2.COLOR_BGR2GRAY)
#blur = cv2.GaussianBlur(gray_image,(5,5),0)
#ret,thresh = cv2.threshold(blur,70,255,cv2.THRESH_BINARY_INV+cv2.THRESH_OTSU)
#hsv_image = cv2.cvtColor(image,cv2.COLOR_BGR2HSV)
#thresh = cv2.inRange(hsv_image, (0,0.28*255,0), (25,0.68*255,255))
fgmask = fgbg.apply(image, learningRate=0.001)
fgmask = cv2.medianBlur(fgmask, 7)
thresh, contours, hierarchy = cv2.findContours(fgmask, cv2.RETR_TREE,
cv2.CHAIN_APPROX_SIMPLE)
drawing = np.zeros(image.shape, np.uint8)
max_area = 0
ci = -1
for i in range(len(contours)):
cnt = contours[i]
area = cv2.contourArea(cnt)
if (area > max_area):
max_area = area
ci = i
if contours and ci >= 0:
cnt = contours[ci]
hull = cv2.convexHull(cnt)
moments = cv2.moments(cnt)
if moments['m00'] != 0:
cx = int(moments['m10'] / moments['m00'])
cy = int(moments['m01'] / moments['m00'])
centr = (cx, cy)
cv2.circle(image, centr, 5, [0, 0, 255], 2)
#cv2.drawContours(drawing,[cnt],0,(0,255,0),2)
#cv2.drawContours(drawing,[hull],0,(0,0,255),2)
cnt = cv2.approxPolyDP(cnt, 0.01 * cv2.arcLength(cnt, True), True)
hull = cv2.convexHull(cnt, returnPoints=False)
defects = cv2.convexityDefects(cnt, hull)
if defects is not None:
for j in range(defects.shape[0]):
s, e, f, d = defects[j, 0]
start = tuple(cnt[s][0])
end = tuple(cnt[e][0])
far = tuple(cnt[f][0])
cv2.line(image, start, end, [0, 255, 0], 2)
cv2.circle(image, far, 5, [0, 0, 255], -1)
if j >= 4 and cv2.contourArea(cnt) > 30000:
socket.send_string("detected")
msg = socket.recv_string()
print(msg)
print(j)
#cv2.imshow("Thresh",thresh)
#cv2.imshow("Output",drawing)
cv2.imshow("Input", image)
key = cv2.waitKey(1) & 0xFF
rawCapture.truncate(0)
if key == ord('q'):
break
def get_all_convexity_defects(self, contour): hull_no_points = cv2.convexHull(contour, returnPoints=False) return cv2.convexityDefects(contour, hull_no_points)
def finalCall():
paths = ['C:\\Users\\kwadehra\\Desktop\\asl_dataset']
trainX = [] # training lists X
trainY = [] # trainig lists y
nb_classes = 3 # number of classes to be classified
img_rows, img_cols = 100, 100 # size of training images
img_channels = 3 # BGR channels OpenCV reads images in BGR
batch_size = 32 # batch size used during training the model
nb_epoch = 2 # iterations for training
data_augmentation = True
# # dictionary for classes from char to numbers
classes = {
# '0': 0,
# '1': 0,
# '2': 1,
# '3': 2,
# '4': 3,
# '5': 5,
# '6': 6,
# '7': 7,
# '8': 8,
# '9': 9,
# 'a': 0,
'b': 0,
# 'c': 2,
'd': 1,
# 'e': 4,
'f': 2,
# 'g': 6,
# 'h': 7,
# 'i': 2,
# 'j': 9,
# 'k': 10,
# 'l': 11,
# 'm': 12,
# 'n': 13,
# 'o': 14,
# 'p': 3,
# 'q': 16,
# 'r': 4,
# 's': 18,
# 't': 19,
# 'u': 20,
# 'v': 21,
# 'y': 24,
# 'z': 25,
# 'w': 22,
# 'x': 23,
}
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.6)
model = load_model('modelbdf.hdf5')
def identifyGesture(handTrainImage):
# saving the sent image for checking
# converting the image to same resolution as training data by padding to reach 1:1 aspect ratio and then
# resizing to 60 x 60.
# Same is done with training data in preprocess_image.py. Opencv image is first
# converted to Pillow image to do this.
handTrainImage = cv2.cvtColor(handTrainImage, cv2.COLOR_BGR2RGB) # OpenCV inputs images in BGR
img = Image.fromarray(handTrainImage)
img_w, img_h = img.size
M = max(img_w, img_h)
background = Image.new('RGB', (M, M), (0, 0, 0))
bg_w, bg_h = background.size
offset = ((bg_w - img_w) // 2, (bg_h - img_h) // 2)
background.paste(img, offset)
size = 100, 100
background = background.resize(size, Image.ANTIALIAS)
# # saving the processed image for checking.
# background.save("C:\\Users\\kwadehra\\Desktop\\SignLanguage\\SavedBackgrounds")
# # get image as numpy array and predict using model
open_cv_image = np.array(background)
background = open_cv_image.astype('float32')
background = background / 255
background = background.reshape((1,) + background.shape)
predictions = model.predict_classes(background)
print(predictions)
# print predicted class and get the class name (character name) for the given class number and return it
# key = next( key for key, value in classes.items() if value == predictions[0])
# return key
for key, value in classes.items():
if value == predictions[0]:
print(key)
return key
# # # Create a window to display the camera feed
cv2.namedWindow('Camera Output')
cv2.namedWindow('Hand')
cv2.namedWindow('HandTrain')
def nothing(k):
pass
# # TrackBars for fixing skin color of the person
cv2.createTrackbar('B for min', 'Camera Output', 0, 255, nothing)
cv2.createTrackbar('G for min', 'Camera Output', 0, 255, nothing)
cv2.createTrackbar('R for min', 'Camera Output', 0, 255, nothing)
cv2.createTrackbar('B for max', 'Camera Output', 0, 255, nothing)
cv2.createTrackbar('G for max', 'Camera Output', 0, 255, nothing)
cv2.createTrackbar('R for max', 'Camera Output', 0, 255, nothing)
# # Default skin color values in natural lighting
cv2.setTrackbarPos('B for min', 'Camera Output', 52)
cv2.setTrackbarPos('G for min', 'Camera Output', 128)
cv2.setTrackbarPos('R for min', 'Camera Output', 0)
cv2.setTrackbarPos('B for max', 'Camera Output', 255)
cv2.setTrackbarPos('G for max', 'Camera Output', 140)
cv2.setTrackbarPos('R for max', 'Camera Output', 146)
# # Default skin color values in indoor lighting
cv2.setTrackbarPos('B for min', 'Camera Output', 0)
cv2.setTrackbarPos('G for min', 'Camera Output', 130)
cv2.setTrackbarPos('R for min', 'Camera Output', 103)
cv2.setTrackbarPos('B for max', 'Camera Output', 255)
cv2.setTrackbarPos('G for max', 'Camera Output', 182)
cv2.setTrackbarPos('R for max', 'Camera Output', 130)
# Get pointer to video frames from primary device
videoFrame = cv2.VideoCapture(1)
# Process the video frames
keyPressed = -1 # -1 indicates no key pressed. Can press any key to exit
# cascade xml file for detecting palm. Haar classifier
palm_cascade = cv2.CascadeClassifier('palm.xml')
# previous values of cropped variable
x_crop_prev, y_crop_prev, w_crop_prev, h_crop_prev = 0, 0, 0, 0
# previous cropped frame if we need to compare histograms of previous image with this to see the change.
# Not used but may need later.
# _, prevHandImage = videoFrame.read()
# # previous frame contour of hand. Used to compare with new contour to find if gesture has changed.
prevcnt = np.array([], dtype=np.int32)
# gesture static increments when gesture doesn't change till it reaches 10 (frames) and then resets to 0.
# gesture detected is set to 10 when gesture static reaches 10."Gesture Detected is displayed for next
# 10 frames till gestureDetected decrements to 0.
gestureStatic = 0
gestureDetected = 0
while keyPressed < 0: # any key pressed has a value >= 0
# Getting min and max colors for skin
min_YCrCb = np.array([cv2.getTrackbarPos('B for min', 'Camera Output'),
cv2.getTrackbarPos('G for min', 'Camera Output'),
cv2.getTrackbarPos('R for min', 'Camera Output')], np.uint8)
max_YCrCb = np.array([cv2.getTrackbarPos('B for max', 'Camera Output'),
cv2.getTrackbarPos('G for max', 'Camera Output'),
cv2.getTrackbarPos('R for max', 'Camera Output')], np.uint8)
# Grab video frame, Decode it and return next video frame
readSuccess, sourceImage = videoFrame.read()
if not readSuccess:
print("Frame not read correctly")
continue
# Convert image to YCrCb
imageYCrCb = cv2.cvtColor(sourceImage, cv2.COLOR_BGR2YCR_CB)
imageYCrCb = cv2.GaussianBlur(imageYCrCb, (5, 5), 0)
# Find region with skin tone in YCrCb image
skinRegion = cv2.inRange(imageYCrCb, min_YCrCb, max_YCrCb)
# Do contour detection on skin region
_, contours, hierarchy = cv2.findContours(skinRegion, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
# sorting contours by area. Largest area first.
contours = sorted(contours, key=cv2.contourArea, reverse=True)
if len(contours) <= 0:
print("Contour not found")
continue
# get largest contour and compare with largest contour from previous frame.
# set previous contour to this one after comparison.
cnt = contours[0]
ret = cv2.matchShapes(cnt, prevcnt, 2, 0.0)
prevcnt = contours[0]
# once we get contour, extract it without background into a new window called handTrainImage
stencil = np.zeros(sourceImage.shape).astype(sourceImage.dtype)
color = [255, 255, 255]
cv2.fillPoly(stencil, [cnt], color)
handTrainImage = cv2.bitwise_and(sourceImage, stencil)
# if comparison returns a high value (shapes are different), start gestureStatic over. Else increment it.
if ret > 0.70:
gestureStatic = 0
else:
gestureStatic += 1
# crop coordinates for hand.
x_crop, y_crop, w_crop, h_crop = cv2.boundingRect(cnt)
# place a rectangle around the hand.
cv2.rectangle(sourceImage, (x_crop, y_crop), (x_crop + w_crop, y_crop + h_crop), (255, 0, 0), 2)
# if the crop area has changed drastically form previous frame, update it.
if (abs(x_crop - x_crop_prev) > 50 or abs(y_crop - y_crop_prev) > 50 or
abs(w_crop - w_crop_prev) > 50 or abs(h_crop - h_crop_prev) > 50):
x_crop_prev = x_crop
y_crop_prev = y_crop
h_crop_prev = h_crop
w_crop_prev = w_crop
# create crop image
handImage = sourceImage.copy()[max(0, y_crop_prev - 50):y_crop_prev + h_crop_prev + 50,
max(0, x_crop_prev - 50):x_crop_prev + w_crop_prev + 50]
# cv2.imshow('hel', handImage)
# Training image with black background
handTrainImage = handTrainImage[max(0, y_crop_prev - 15):y_crop_prev + h_crop_prev + 15,
max(0, x_crop_prev - 15):x_crop_prev + w_crop_prev + 15]
# cv2.imshow('hi', handTrainImage)
# if gesture is static for 10 frames, set gestureDetected to 10 and display "gesture detected"
# on screen for 10 frames.
if gestureStatic == 5:
gestureDetected = 5
print("Gesture Detected")
letterDetected = identifyGesture(handTrainImage)
if gestureDetected > 0:
if letterDetected != None:
img = np.zeros((512, 512, 3), np.uint8)
cv2.rectangle(img,(384,0),(510,128),(0,255,0),3)
font = cv2.FONT_HERSHEY_SIMPLEX
cv2.putText(img, str(letterDetected), (10, 500), font, 4, (255, 255, 255), 2, cv2.LINE_AA)
cv2.imshow('image', img)
gestureDetected -= 1
gray = cv2.cvtColor(handImage, cv2.COLOR_BGR2HSV)
palm = palm_cascade.detectMultiScale(gray)
for (x, y, w, h) in palm:
cv2.rectangle(sourceImage, (x, y), (x + w, y + h), (147, 20, 255), 2)
# roi_gray = gray[y:y + h, x:x + w]
roi_color = sourceImage[y:y + h, x:x + w]
# to show convex hull in the image
hull = cv2.convexHull(cnt, returnPoints=False)
defects = cv2.convexityDefects(cnt, hull)
# counting defects in convex hull. To find center of palm. Center is average of defect points.
count_defects = 0
if defects is None:
print("Convexity defects can't be detected")
continue
for i in range(defects.shape[0]):
s, e, f, d = defects[i, 0]
start = tuple(cnt[s][0])
end = tuple(cnt[e][0])
far = tuple(cnt[f][0])
centre = far
if count_defects == 0:
cent = far
a = math.sqrt((end[0] - start[0]) ** 2 + (end[1] - start[1]) ** 2)
b = math.sqrt((far[0] - start[0]) ** 2 + (far[1] - start[1]) ** 2)
c = math.sqrt((end[0] - far[0]) ** 2 + (end[1] - far[1]) ** 2)
angle = math.acos((b ** 2 + c ** 2 - a ** 2) / (2 * b * c)) * 57
if angle <= 90:
count_defects += 1
if count_defects < 5:
cv2.circle(sourceImage, far, 5, [0, 0, 255], -1)
center_of_palm = ((far[0] + centre[0]) / 2, (far[1] + centre[1]) / 2)
cv2.line(sourceImage, start, end, [0, 0, 255], 2)
# cv2.circle(sourceImage, avr, 10, [255, 255, 255], -1)
# drawing the largest contour
cv2.drawContours(sourceImage, contours, 0, (0, 255, 0), 1)
# Display the source image and cropped image
cv2.imshow('Camera Output', sourceImage)
cv2.imshow('Hand', handImage)
cv2.imshow('HandTrain', handTrainImage)
# Check for user input to close program
keyPressed = cv2.waitKey(30) # wait 30 miliseconds in each iteration of while loop
# Close window and camera after exiting the while loop
videoFrame.release()
cv2.destroyWindow('Camera Output')
def detect(self):
vel = Twist()
vel.linear.x = 0
vel.angular.z = 0 #left
self.pub_vel.publish(vel)
min_YCrCb = np.array([0, 133, 77], np.uint8)
max_YCrCb = np.array([235, 173, 127], np.uint8)
'''Callback function of subscribed topic.
Here images get converted and features detected'''
while self.move:
(grabbed, self.img) = camera.read()
self.img = imutils.resize(self.img, width=400)
imageYCrCb = cv2.cvtColor(self.img, cv2.COLOR_BGR2YCR_CB)
skinMask = cv2.inRange(imageYCrCb, min_YCrCb, max_YCrCb)
# apply a series of erosions and dilations to the mask
# using an elliptical kernel
kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (11, 11))
skinMask = cv2.erode(skinMask, kernel, iterations=2)
skinMask = cv2.dilate(skinMask, kernel, iterations=2)
# blur the mask to help remove noise, then apply the
# mask to the frame
skinMask = cv2.GaussianBlur(skinMask, (3, 3), 0)
skin = cv2.bitwise_and(self.img, self.img, mask=skinMask)
grey = cv2.cvtColor(skin, cv2.COLOR_BGR2GRAY)
value = (35, 35)
blurred = cv2.GaussianBlur(grey, value, 0)
_, thresh1 = cv2.threshold(blurred, 127, 255,
cv2.THRESH_BINARY_INV + cv2.THRESH_OTSU)
#cv2.imshow('Thresholded', thresh1)
_, contours, hierarchy = cv2.findContours(thresh1.copy(),
cv2.RETR_TREE,
cv2.CHAIN_APPROX_NONE)
max_area = -1
for i in range(len(contours)):
cnt = contours[i]
area = cv2.contourArea(cnt)
if (area > max_area):
max_area = area
ci = i
cnt = contours[ci]
x, y, w, h = cv2.boundingRect(cnt)
cv2.rectangle(skin, (x, y), (x + w, y + h), (0, 0, 255), 0)
hull = cv2.convexHull(cnt)
drawing = np.zeros(skin.shape, np.uint8)
cv2.drawContours(drawing, [cnt], 0, (0, 255, 0), 0)
cv2.drawContours(drawing, [hull], 0, (0, 0, 255), 0)
hull = cv2.convexHull(cnt, returnPoints=False)
defects = cv2.convexityDefects(cnt, hull)
count_defects = 0
cv2.drawContours(thresh1, contours, -1, (0, 255, 0), 3)
if (defects.shape[0] != None):
for i in range(defects.shape[0]):
s, e, f, d = defects[i, 0]
start = tuple(cnt[s][0])
end = tuple(cnt[e][0])
far = tuple(cnt[f][0])
a = math.sqrt((end[0] - start[0])**2 +
(end[1] - start[1])**2)
b = math.sqrt((far[0] - start[0])**2 +
(far[1] - start[1])**2)
c = math.sqrt((end[0] - far[0])**2 + (end[1] - far[1])**2)
angle = math.acos((b**2 + c**2 - a**2) / (2 * b * c)) * 57
if angle <= 90:
count_defects += 1
cv2.circle(skin, far, 1, [0, 0, 255], -1)
cv2.line(skin, start, end, [0, 255, 0], 2)
if count_defects == 1: #2
self.count(2)
elif count_defects == 2: #3
self.count(3)
elif count_defects == 3: #4
self.count(4)
elif count_defects == 4: #5
self.count(5)
else: #0-1
#self.move0()
a = 0
all_img = np.hstack((drawing, skin))
cv2.imshow('Contours', all_img)
k = cv2.waitKey(10)
def getFingerPosition(max_contour, img_result, debug):
points1 = []
# Image Moments : 이미지 모멘트는 객체의 무게중심, 객체의 면적 등과 같은 특성을 계산할 때 유용
# cv.moments() 함수는 이미지 모멘트를 계산하고 이를 사전형 자료에 담아 리턴함.
M = cv.moments(max_contour)
cx = int(M['m10'] / M['m00'])
cy = int(M['m01'] / M['m00'])
# 다각형 추출
# approxPolyDP(): 다각형을 대상으로 꼭지점을 점점 줄여나가는 함수. epsilon(오차)만큼을 최대한으로 해서 꼭지점을 줄여나감
# 그래서 epsilon 값이 작을수록 원본과 비슷한 결과가 도출되고 epsilon(오차)값이 크면 클수록 꼭지점의 개수가 점점 더 줄어든다.
# , 인자로 주어진 곡선 또는 다각형을 epsilon 값에 따라 꼭지점을 줄여 새로운 곡선이나 다각형을 생성하여 리턴
# 인자 > cnt : numpy Array 형식의 곡선 또는 다각형. epsilon:근사 정확도를 위한 값. 오리지널 커브와 근사 커브 거리의 최대값으로 사용,
# True : True면 폐곡선, False면 양끝이 열려있는 곡선임을 의미
approx = cv.approxPolyDP(max_contour,
0.02 * cv.arcLength(max_contour, True), True)
# convex Hull 볼록체 : 윤곽선(points, contours)의 경계면을 둘러싸는 다각형을 구하는 알고리즘.
# 오목한 부분을 피해서 contour을 그린다.
hull = cv.convexHull(approx)
for i, point in enumerate(hull):
if cy > point[0][1]:
points1.append(tuple(point[0]))
if debug:
cv.drawContours(img_result, [hull], 0, (0, 255, 0), 2)
for point in points1:
cv.circle(img_result, tuple(point), 15, (0, 0, 0), -1)
hull = cv.convexHull(approx, returnPoints=False)
defects = cv.convexityDefects(approx, hull)
if defects is None:
return -1, None
points2 = []
for i in range(defects.shape[0]):
s, e, f, d = defects[i, 0]
start = tuple(approx[s][0])
end = tuple(approx[e][0])
far = tuple(approx[f][0])
angle = caculateAngle(
np.array(end) - np.array(far),
np.array(start) - np.array(far))
if angle > 0 and angle < 45 and d > 10000:
if start[1] < cy:
points2.append(start)
if end[1] < cy:
points2.append(end)
if debug:
cv.drawContours(img_result, [approx], 0, (255, 0, 255), 2)
for point in points2:
cv.circle(img_result, tuple(point), 20, (0, 255, 0), 5)
points1 = points1 + points2
points1 = list(set(points1))
new_points = []
for point1 in points1:
idx = -1
for j, points2 in enumerate(approx):
if point1 == tuple(points2[0]):
idx = j
break
if idx == -1:
continue
if idx - 1 >= 0:
pre = np.array(approx[idx - 1][0])
else:
pre = np.array(approx[len(approx) - 1][0])
if idx + 1 < len(approx):
next = np.array(approx[idx + 1][0])
else:
next = np.array(approx[0][0])
angle = caculateAngle(pre - point1, next - point1)
distnace1 = distanceBetweenTwoPoints(pre, point1)
distnace2 = distanceBetweenTwoPoints(next, point1)
if angle < 45 and distnace1 > 40 and distnace2 > 40:
new_points.append(point1)
return 1, new_points
def num_fingers(image):
#grab the raw NumPy array representing the image, then initialize the timestamp and occupied/unoccupied text
im = cv2.imread(image)
im1 = cv2.imread(image)
#frame=cv2.flip(frame,1)
#hsv = cv2.cvtColor(img, cv2.COLOR_BGR2HSV)
#lower_blue = np.array([100,50,50])
#upper_blue = np.array([140,255,255])
thresh = segment_colour(im)
thresh2 = segment_colour(im)
#cv2.imshow('frame',img)
#cv2.imshow('mask',mask)
kernel = np.ones((10, 10), np.uint8)
#erosion = cv2.erode(thresh,kernel,iterations =2)
im2, contours, hierarchy = cv2.findContours(thresh, cv2.RETR_TREE,
cv2.CHAIN_APPROX_SIMPLE)
#print (len(contours))
if (len(contours) == 0):
return 0
#x=int((len(contours)/2))
#cnt = contours[7]
#area = cv2.contourArea(cnt)
#cv2.drawContours(im1,[cnt],0,(0,0,255),100 )
#print area
ci = 0
max_area = 0
for i in range(len(contours)):
#image=im
cnt1 = contours[i]
#cv2.drawContours(image,[cnt],0,(0,255,0),1)
#plt.subplot(2,2,i+1),plt.imshow(im,cmap = 'gray')
#plt.title(area), plt.xticks([]), plt.yticks([])
#max_area=0
area = cv2.contourArea(cnt1)
#cv2.drawContours(image,[cnt],0,(0,255,0),1)
#plt.subplot(2,2,i+1),plt.imshow(im,cmap = 'gray')
#plt.title(area), plt.xticks([]), plt.yticks([])
#print area,i
if (area > max_area):
max_area = area
ci = i
#print (ci)
cnt = contours[ci]
cv2.drawContours(im, [cnt], 0, (0, 0, 255), 5)
x, y, w, h = cv2.boundingRect(cnt)
#print w,h
lbr = 2 * (w + h)
#cv2.drawContours(im1,[cnt1],0,(0,0,255),100 )
hull = cv2.convexHull(cnt)
cv2.drawContours(im1, [hull], 0, (0, 255, 0), 10)
hull = cv2.convexHull(cnt, returnPoints=False)
defects = cv2.convexityDefects(cnt, hull)
drawing = np.zeros(im.shape, np.uint8)
#print defects
#cv2.drawContours(im,[cnt],0,(0,255,0),1)
#cv2.drawContours(im,[hull],0,(0,0,255),1)
#defects = cv2.convexityDefects(cnt,hull)
#print defects
mind = 0
maxd = 0
count = 0
for i in range(1, defects.shape[0]):
s, e, f, d = defects[i, 0]
start = tuple(cnt[s][0])
end = tuple(cnt[e][0])
far = tuple(cnt[f][0])
#cv2.line(im1,start,end,[0 ,0,255],100)
#cv2.circle(im1,far,100,[0,0,255],-1)
if ((d >= 0.4 * lbr) and (angle(far, start, end) < 90)):
#print(1)
cv2.line(im1, start, end, [0, 0, 255], 10)
cv2.circle(im1, far, 5, [0, 0, 255], -1)
count = count + 1
"""
# cv2.namedWindow('thresh', cv2.WINDOW_NORMAL)
#cv2.imshow('thresh',im1)
#cv2.waitKey(2000)
# cv2.destroyAllWindows()
"""
#print i,d
numberoffinger = count + 1
cv2.namedWindow('thresh', cv2.WINDOW_NORMAL)
cv2.imshow('thresh', im1)
cv2.waitKey(1000)
cv2.destroyAllWindows()
return numberoffinger
def hand():
try:
cap = cv2.VideoCapture(0)
cap.set(3, 320)
cap.set(4, 240)
except:
print('camera_error')
return
beforecnt = 0
while True:
ret, frame = cap.read()
if not ret:
print('camera_error')
break
dst = frame.copy()
test = cv2.cvtColor(dst, cv2.COLOR_BGR2YCrCb)
mask_hand = cv2.inRange(test, np.array([0, 132, 77], dtype="uint8"),
np.array([255, 173, 133], dtype="uint8"))
blur = cv2.blur(mask_hand, (2, 2))
ret, thr = cv2.threshold(blur, 132, 255, cv2.THRESH_BINARY)
_, contours, hierachy = cv2.findContours(thr, cv2.RETR_TREE,
cv2.CHAIN_APPROX_SIMPLE)
cnt = 0
if contours:
contours = max(contours, key=lambda x: cv2.contourArea(x))
hull = cv2.convexHull(contours, returnPoints=False)
defects = cv2.convexityDefects(contours, hull)
#cv2.drawContours(dst, contours, -1, (0, 255, 255), 2)
if defects is not None:
cnt = 0
for i in range(defects.shape[0]):
startd, endd, fard, d = defects[i][0]
start = tuple(contours[startd][0])
end = tuple(contours[endd][0])
far = tuple(contours[fard][0])
a = np.sqrt((end[0] - start[0])**2 +
(end[1] - start[1])**2)
b = np.sqrt((far[0] - start[0])**2 +
(far[1] - start[1])**2)
c = np.sqrt((end[0] - far[0])**2 + (end[1] - far[1])**2)
angle = np.arccos((b**2 + c**2 - a**2) / (2 * b * c))
if angle >= np.pi / 10 and angle <= np.pi / 2.3:
cnt += 1
cv2.circle(dst, far, 4, [0, 0, 255], -1)
if cnt > 0:
cnt = cnt + 1
beforecnt = beforecnt * 0.97 + cnt * 0.03
else:
cnt = 0
setCount(round(beforecnt))
cv2.putText(dst, str(round(beforecnt)), (0, 90),
cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 2,
cv2.LINE_AA)
cv2.putText(dst, str(round(recordcnt)), (0, 130),
cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 0), 2, cv2.LINE_AA)
if mode == -1:
colorCode = "c" + str(recordcnt)
setColor(colorCode)
cv2.imshow('dst', dst)
#cv2.imshow('ret', thr)
#cv2.imshow('key', test)
k = cv2.waitKey(1) & 0xFF
if k == 27:
break
cap.release()
cv2.destroyAllWindows()
def processImage(image):
# Apply Gaussian blur
image = cv.GaussianBlur(image, (9, 9), 4)
# Convert to HSV color-space
hsvImage = cv.cvtColor(image, cv.COLOR_BGR2HSV)
# Calculate the lower and upper HS values
minH = 0
maxH = 50
minS = 255 * 0.21
maxS = 255 * 0.68
lower = (minH, minS, 0)
upper = (maxH, maxS, 255)
# Mask HS Values to create a binary image
mask = cv.inRange(hsvImage, lower, upper)
# Morphological transformations to filter
erode = cv.erode(mask, np.ones((3, 3)), iterations=2)
dilation = cv.dilate(erode, np.ones((3, 3)), iterations=3)
#filtered = cv.GaussianBlur(dilation, (5, 5), 4)
#ret, threshold = cv.threshold(filtered, 120, 255, 0)
# Find the contours
ret, contours, hierarchy = cv.findContours(dilation, cv.RETR_TREE, cv.CHAIN_APPROX_NONE)
# Find contour with maximum area
contour = max(contours, key=lambda x: cv.contourArea(x))
# Create bounding rectangle around the contour
x, y, w, h = cv.boundingRect(contour)
cv.rectangle(image, (x, y), (x+w, y+h), (0, 0, 255), 0)
# Find convex hull
hull = cv.convexHull(contour)
# Draw contour
drawing = np.zeros(image.shape, np.uint8)
cv.drawContours(drawing, [contour], -1, (0, 255, ), 0)
cv.drawContours(drawing, [hull], -1, (0, 0, 255), 0)
# Find convexity defects
hull = cv.convexHull(contour, returnPoints=False)
defects = cv.convexityDefects(contour, hull)
count_defects = 0
for i in range(defects.shape[0]):
s, e, f, d = defects[i, 0]
start = tuple(contour[s][0])
end = tuple(contour[e][0])
far = tuple(contour[f][0])
cv.line(image, start, end, [0, 255, 0], 2)
a = math.sqrt((end[0] - start[0])**2 + (end[1] - start[1])**2)
b = math.sqrt((far[0] - start[0])**2 + (far[1] - start[1])**2)
c = math.sqrt((end[0] - far[0])**2 + (end[1] - far[1])**2)
angle = (math.acos((b**2 + c**2 - a**2)/(2*b*c))*180)/3.14
# far point is below the start and end points
if far[1] < start[1] or far[1] < end[1]:
continue
# if angle > 90 draw a circle at the far point
if angle <= 90:
count_defects += 1
cv.circle(image, far, 5, [0, 0, 255], -1)
cv.circle(image, end, 5, [255, 0, 255], -1)
cv.circle(image, start, 5, [255, 0, 255], -1)
# Show each mask used
image = cv.resize(image, (500, 500))
mask = cv.resize(mask, (500, 500))
cv.imshow("Hand", image)
cv.imshow("Mask", mask)
#cv.imshow("HSV", hsvImage)
#cv.imshow("Thresholded", threshold)
while(cv.waitKey(0) != 27): continue
num_fingers = count_defects + 1
hands = []
hands.append(num_fingers)
return hands
def findDefects(cnt):
if cnt is None or cnt == []:
return None
hull = cv2.convexHull(cnt, returnPoints=False)
defects = cv2.convexityDefects(cnt, hull)
return defects
def getFingerPosition(max_contour, img_result, debug):
points1 = []
# STEP 6-1
M = cv.moments(max_contour)
cx = int(M['m10'] / M['m00'])
cy = int(M['m01'] / M['m00'])
max_contour = cv.approxPolyDP(max_contour,
0.02 * cv.arcLength(max_contour, True), True)
hull = cv.convexHull(max_contour)
for point in hull:
if cy > point[0][1]:
points1.append(tuple(point[0]))
if debug:
cv.drawContours(img_result, [hull], 0, (0, 255, 0), 2)
for point in points1:
cv.circle(img_result, tuple(point), 15, [0, 0, 0], -1)
# STEP 6-2
hull = cv.convexHull(max_contour, returnPoints=False)
defects = cv.convexityDefects(max_contour, hull)
if defects is None:
return -1, None
points2 = []
for i in range(defects.shape[0]):
s, e, f, d = defects[i, 0]
start = tuple(max_contour[s][0])
end = tuple(max_contour[e][0])
far = tuple(max_contour[f][0])
angle = calculateAngle(
np.array(start) - np.array(far),
np.array(end) - np.array(far))
if angle < 90:
if start[1] < cy:
points2.append(start)
if end[1] < cy:
points2.append(end)
if debug:
cv.drawContours(img_result, [max_contour], 0, (255, 0, 255), 2)
for point in points2:
cv.circle(img_result, tuple(point), 20, [0, 255, 0], 5)
# STEP 6-3
points = points1 + points2
points = list(set(points))
# STEP 6-4
new_points = []
for p0 in points:
i = -1
for index, c0 in enumerate(max_contour):
c0 = tuple(c0[0])
if p0 == c0 or distanceBetweenTwoPoints(p0, c0) < 20:
i = index
break
if i >= 0:
pre = i - 1
if pre < 0:
pre = max_contour[len(max_contour) - 1][0]
else:
pre = max_contour[i - 1][0]
next = i + 1
if next > len(max_contour) - 1:
next = max_contour[0][0]
else:
next = max_contour[i + 1][0]
if isinstance(pre, np.ndarray):
pre = tuple(pre.tolist())
if isinstance(next, np.ndarray):
next = tuple(next.tolist())
angle = calculateAngle(
np.array(pre) - np.array(p0),
np.array(next) - np.array(p0))
if angle < 90:
new_points.append(p0)
return 1, new_points
def detect(self, frame, x, y, z, show=False, flipped=False):
"""
classify operator's hand based on rgb/d data
:param frame: rgb 1920x1080 image frame as array
:param x: x color pixel coordinate
:param y: y color pixel coordinate
:param z: z depth distance of hand from rgbd sensor
:param show: boolean to show the classified hand state
:param flipped: boolean to flip the frame to match the pixel coordinates
:return: None
"""
# Import libraries here for optimization
import cv2
import numpy as np
# Loop until flag
if not self._done:
try:
if flipped:
frame = cv2.flip(
frame, 1
) # If the image is flipped flip it again for better accuracy
roi = self.roi(frame, x, y, z) # Compute area of hand only
# cv2.imshow('ROI', roi)
# cv2.imwrite('images/roi1.png', roi)
# Change track bars position
if self._show:
self.lower_skin = np.array(
[
cv2.getTrackbarPos("L_R", "Settings"),
cv2.getTrackbarPos("L_G", "Settings"),
cv2.getTrackbarPos("L_B", "Settings")
],
dtype=np.uint8) # lower limit for skin color
self.upper_skin = np.array(
[
cv2.getTrackbarPos("U_R", "Settings"),
cv2.getTrackbarPos("U_G", "Settings"),
cv2.getTrackbarPos("U_B", "Settings")
],
dtype=np.uint8) # upper limit for skin color
# Convert from BGR to HSV
hsv = cv2.cvtColor(roi, cv2.COLOR_BGR2HSV)
# cv2.imshow('HSV', hsv)
# cv2.imwrite('images/roi2.png', hsv)
# Apply mask for skin detection
mask = cv2.inRange(hsv, self.lower_skin, self.upper_skin)
# cv2.imshow('Skin Detection', mask)
# cv2.imwrite('images/roi3.png', mask)
# Dilate image to fill black spots and spots inside hand
mask = cv2.dilate(mask, self.kernel, iterations=4)
# cv2.imshow('Dilate', mask)
# cv2.imwrite('images/roi4.png', mask)
# Apply Gaussian Blur to reduce noise
mask = cv2.GaussianBlur(mask, (5, 5), 100)
# cv2.imshow('GaussianBlur', mask)
# cv2.imwrite('images/roi5.png', mask)
# Compute contours in remaining image
contours, hierarchy = cv2.findContours(mask, cv2.RETR_TREE,
cv2.CHAIN_APPROX_SIMPLE)
# Find max contour in given area
max_contour = max(contours, key=lambda x: cv2.contourArea(x))
# Find useful data of these contour like center of mass
# M = cv2.moments(max_contour)
# cx = int(M['m10']/M['m00'])
# cy = int(M['m01']/M['m00'])
# cv2.circle(roi, (int(cx), int(cy)), 3, [255, 0, 255], -1)
# Find minimum enclosing area
# (circle_x, circle_y), radius = cv2.minEnclosingCircle(max_contour)
# center = (int(circle_x), int(circle_y))
# radius = int(radius)
# cv2.circle(roi, center, radius, [128, 0, 128], 2)
# Find maximum distance of contour
# distance = cv2.distanceTransform(mask, cv2.DIST_L2, 3)
# cv2.circle(roi, center, int(np.amax(distance)), [255, 0, 0], 2)
# cv2.imshow('DISTANCES', distance)
# cv2.drawContours(roi, [max_contour], 0, (0, 0, 255), 2)
# cv2.imshow('Max Contour', roi)
# cv2.imwrite('images/roi6.png', roi)
# Compute contour and convex hull area to compare
epsilon = 0.0005 * cv2.arcLength(max_contour, True)
approx = cv2.approxPolyDP(max_contour, epsilon, True)
hull = cv2.convexHull(max_contour)
area_hull = cv2.contourArea(hull)
area_contour = cv2.contourArea(max_contour)
area_ratio = ((area_hull - area_contour) / area_contour) * 100
# print(area_ratio)
hull = cv2.convexHull(approx, returnPoints=False)
defects = cv2.convexityDefects(approx, hull)
fingers = 0
# Search for defects on convex hull of max contour area found
for i in range(defects.shape[0]):
# Get starting point, ending point, farthest point and distance to farthest point for each line
s, e, f, d = defects[i, 0]
"""
The triangles are generated when two fingers are apart and the area_hull - area_contour leaves a triangle shape behind.
The upside triangle between two fingers apart
"""
# Find sides of triangle
start = approx[s][0]
end = approx[e][0]
far = approx[f][0]
# Calculate the length of each triangle line
a = np.sqrt((end[0] - start[0])**2 +
(end[1] - start[1])**2)
b = np.sqrt((far[0] - start[0])**2 +
(far[1] - start[1])**2)
c = np.sqrt((end[0] - far[0])**2 + (end[1] - far[1])**2)
# Calculate the semi - perimeter of the triangle
semi_perimeter = (a + b + c) / 2
# Calculate the area of the triangle using Heron's formula that uses only the lengths of the triangle
ar = np.sqrt(semi_perimeter * (semi_perimeter - a) *
(semi_perimeter - b) * (semi_perimeter - c))
# Then from the area of the triangle we can find the height assuming that the triangle is upside down, so the a side is always he base.
# The height is the distance of the farthest point from the convexhull
d = (2 * ar) / a
# Cosine rule for triangle angles
angle = np.arccos(
(b**2 + c**2 - a**2) / (2 * b * c)) * (180 / np.pi)
# if the angle is greater than 90 then there is no defect and the line is on the perimeter of the hand
# if the distance of the farthest point is too close then the point is inside the hand and its just noise.
if angle <= 90 and d > 30:
fingers += 1 # find defects on convex hull ( defects usually are fingers )
if show:
cv2.circle(roi, tuple(far), 3, [255, 0, 0], -1)
if show:
cv2.line(roi, tuple(start), tuple(end), [0, 255, 0], 2)
fingers += 1
# Update state based on number of defects/edges found in convex hull
if fingers == 1:
if area_ratio < 26 / z:
if show:
cv2.putText(roi, 'Closed', (0, 50),
cv2.FONT_HERSHEY_SIMPLEX, 0.5,
(0, 0, 255), 3, cv2.LINE_AA)
self.state = 'CLOSED'
self.fingers = 0
else:
if show:
cv2.putText(roi, 'Open', (0, 50),
cv2.FONT_HERSHEY_SIMPLEX, 0.5,
(0, 0, 255), 3, cv2.LINE_AA)
self.state = 'OPEN'
self.fingers = 1
elif fingers == 2:
if area_ratio < 26 / z:
if show:
cv2.putText(roi, 'Closed', (0, 50),
cv2.FONT_HERSHEY_SIMPLEX, 0.5,
(0, 0, 255), 3, cv2.LINE_AA)
self.state = 'CLOSED'
self.fingers = 0
else:
if show:
cv2.putText(roi, 'Open', (0, 50),
cv2.FONT_HERSHEY_SIMPLEX, 0.5,
(0, 0, 255), 3, cv2.LINE_AA)
self.state = 'OPEN'
self.fingers = 2
elif fingers == 3:
if show:
cv2.putText(roi, '3', (0, 50),
cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255),
3, cv2.LINE_AA)
self.state = 'OPEN'
self.fingers = 3
elif fingers == 4:
if show:
cv2.putText(roi, '4', (0, 50),
cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255),
3, cv2.LINE_AA)
self.state = 'OPEN'
self.fingers = 4
elif fingers == 5:
if show:
cv2.putText(roi, '5', (0, 50),
cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255),
3, cv2.LINE_AA)
self.state = 'OPEN'
self.fingers = 5
else:
if show:
cv2.putText(roi, 'UNKNOWN', (0, 50),
cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255),
3, cv2.LINE_AA)
self.state = 'UNKNOWN'
if show:
# Show ROI of Color Frame
cv2.imshow('Hand Classifier', roi)
"""
# Capture using q or space
if cv2.waitKey(1) & 0xff == ord('q') or cv2.waitKey(1) % 256 == 32:
print(f'[GESTURE CLASSIFIER]: SAVED IMAGE')
cv2.imwrite('images/roi1.png', roi)
cv2.imwrite('images/roi7.png', roi)
print(f'[FIN]')
import time
time.sleep(1)
"""
# Handle any exception
except Exception as e:
print(f'[GESTURE CLASSIFIER]: {e}')
# Destroy all windows
else:
cv2.destroyAllWindows()
def predict_frame(self):
try:
ret, frame = self.video.read()
frame = cv2.flip(frame, 1)
kernel = np.ones((3, 3), np.uint8)
# define region of interest
roi = frame[100:400, 100:400]
cv2.rectangle(frame, (100, 100), (400, 400), (0, 255, 0), 0)
hsv = cv2.cvtColor(roi, cv2.COLOR_BGR2HSV)
# define range of skin color in HSV
lower_skin = np.array([0, 20, 70], dtype=np.uint8)
upper_skin = np.array([20, 255, 255], dtype=np.uint8)
# extract skin colur imagw
mask = cv2.inRange(hsv, lower_skin, upper_skin)
# extrapolate the hand to fill dark spots within
mask = cv2.dilate(mask, kernel, iterations=4)
# blur the image
mask = cv2.GaussianBlur(mask, (5, 5), 100)
# find contours
contours, hierarchy = cv2.findContours(mask, cv2.RETR_TREE,
cv2.CHAIN_APPROX_SIMPLE)
# find contour of max area(hand)
cnt = max(contours, key=lambda x: cv2.contourArea(x))
# approx the contour a little
epsilon = 0.0005 * cv2.arcLength(cnt, True)
approx = cv2.approxPolyDP(cnt, epsilon, True)
# make convex hull around hand
hull = cv2.convexHull(cnt)
# define area of hull and area of hand
areahull = cv2.contourArea(hull)
areacnt = cv2.contourArea(cnt)
# find the percentage of area not covered by hand in convex hull
arearatio = ((areahull - areacnt) / areacnt) * 100
# find the defects in convex hull with respect to hand
hull = cv2.convexHull(approx, returnPoints=False)
defects = cv2.convexityDefects(approx, hull)
# l = no. of defects
l = 0
# code for finding no. of defects due to fingers
for i in range(defects.shape[0]):
s, e, f, d = defects[i, 0]
start = tuple(approx[s][0])
end = tuple(approx[e][0])
far = tuple(approx[f][0])
pt = (100, 180)
# find length of all sides of triangle
a = math.sqrt((end[0] - start[0])**2 + (end[1] - start[1])**2)
b = math.sqrt((far[0] - start[0])**2 + (far[1] - start[1])**2)
c = math.sqrt((end[0] - far[0])**2 + (end[1] - far[1])**2)
s = (a + b + c) / 2
ar = math.sqrt(s * (s - a) * (s - b) * (s - c))
# distance between point and convex hull
d = (2 * ar) / a
# apply cosine rule here
angle = math.acos((b**2 + c**2 - a**2) / (2 * b * c)) * 57
# ignore angles > 90 and ignore points very close to convex hull(they generally come due to noise)
if angle <= 90 and d > 30:
l += 1
cv2.circle(roi, far, 3, [255, 0, 0], -1)
# draw lines around hand
cv2.line(roi, start, end, [0, 255, 0], 2)
l += 1
# print corresponding gestures which are in their ranges
if l == 1:
if areacnt < 2000:
text = 'Put hand in the box'
else:
if arearatio < 12:
text = '0'
elif arearatio < 17.5:
text = 'Best of luck'
else:
text = '1'
elif l == 2:
text = '2'
elif l == 3:
if arearatio < 27:
text = '3'
else:
text = 'ok'
elif l == 4:
text = '4'
elif l == 5:
text = '5'
elif l == 6:
text = 'reposition'
else:
text = 'reposition'
# show the windows
return text
except Exception as ex:
pass
cv2.CHAIN_APPROX_SIMPLE)
#Acha contorno da maior área
cnt = max(contours, key=lambda x: cv2.contourArea(x))
#Aproxima um pouco o contorno
epsilon = 0.0005 * cv2.arcLength(cnt, True)
approx = cv2.approxPolyDP(cnt, epsilon, True)
#Casco convexo em torno da mão
hull = cv2.convexHull(cnt)
#Área da mão e área do casco
areahull = cv2.contourArea(hull)
areacnt = cv2.contourArea(cnt)
#Porcentagem da área da mão que não cobre a area do casco
porcentagemArea = ((areahull - areacnt) / areacnt) * 100
#Acha os defeitos de convexidades
hull = cv2.convexHull(approx, returnPoints=False)
defects = cv2.convexityDefects(approx, hull)
# l = numero de defeitos
l = 0
#lista de distancias
distancias = []
#numero de convexidades nos dedos(muita matemática);
for i in range(defects.shape[0]):
s, e, f, d = defects[i, 0]
comeco = tuple(approx[s][0])
fim = tuple(approx[e][0])
far = tuple(approx[f][0])
pt = (100, 180)
a = math.sqrt((fim[0] - comeco[0])**2 + (fim[1] - comeco[1])**2)
b = math.sqrt((far[0] - comeco[0])**2 + (far[1] - comeco[1])**2)
c = math.sqrt((fim[0] - far[0])**2 + (fim[1] - far[1])**2)
s = (a + b + c) / 2
