def __init__(self, port):
f=open('/media/lzz/65c50da0-a3a2-4117-8a72-7b37fd81b574/sign/project/src/EducationNew.pickle')
self.pickle=pickle.load(f)
f.close()
self.fr=0
self.one=0
self.exist=0
self.lock = threading.Lock()
self.condition=threading.Condition()
self.cond=threading.Condition()
self.classifier = EduClassifier()
self.p_thres=0.4
self.h_thres=0.5
self.h_thres_test=0.3
self.converter = FrameConverter()
self.caffedl=caffeDL('../../proto/lenet_test_edu.prototxt','../../model/lenet_iter_5000.caffemodel')
self.caffedlInter=caffeDL('../../proto_inter/lenet_test_edu.prototxt','../../model/lenet__iter_400.caffemodel')
self.signDic={}
self.queue = Queue.Queue(maxsize = 10000)
self.server = EchoServer(port, self.received_data)
self.mode=''
thread.start_new_thread(self.threadfunc,())
asyncore.loop()
def __init__(self, port):
shutil.rmtree('/home/lzz/sign/realtime/handshape')
shutil.rmtree('/home/lzz/sign/realtime/rawshape')
os.mkdir('/home/lzz/sign/realtime/handshape')
os.mkdir('/home/lzz/sign/realtime/rawshape')
self.fr=0
self.one=0
self.exist=0
self.lock = threading.Lock()
self.condition=threading.Condition()
self.classifier=Classifier()
self.classifier.filelist.append('/home/lzz/sign/svm/')
self.classifier.dic['/home/lzz/sign/svm/']=SignWord('/home/lzz/sign/svm/',0)
self.classifier.dic['/home/lzz/sign/svm/'].skeletons=[]
self.classifier.dic['/home/lzz/sign/svm/'].positions=[]
#f=self.dic[path].loadData()
#self.classifier.buildDic('/home/lzz/svm/',1)
self.converter = FrameConverter()
self.caffedl=caffeDL('/home/lzz/sign/proto/lenet_test.prototxt','/home/lzz/sign/model/lenet_iter_5000.caffemodel')
self.caffedlInter=caffeDL('/home/lzz/sign/proto_inter/lenet_test.prototxt','/home/lzz/sign/model/lenet__iter_400.caffemodel')
self.signDic={}
self.queue = Queue.Queue(maxsize = 10000)
Label1=open('/home/lzz/sign/svm/dictionary.csv','rb')
reader = csv.reader(Label1)
for row in reader:
self.signDic[int(row[0])]=row[1]
self.server = EchoServer(port, self.received_data)
thread.start_new_thread(self.threadfunc,())
asyncore.loop()
def readVideo(fileName, frameCallback, result):
frameConverter = FrameConverter()
labelFrameData = CSVFrameData()
labelFrameData.load(join(labelDirectory, fileName))
labelFrameData.setDebug(DEBUG_MODE)
fileName = fileName[:-4]
skeletonFrameData = CSVFrameData()
skeletonFrameData.load(
join(skeletonDirectory,
fileName + skeletonVideoSuffix + skeletonFilenameExtension))
skeletonFrameData.setDebug(DEBUG_MODE)
srcVideoPath = join(videoDirectory,
fileName + depthVideoSuffix + videoFilenameExtension)
depthFrameData = VideoFrameData()
depthFrameData.load(srcVideoPath)
i = 0
resultImages = []
depthRetval, depthFrame = depthFrameData.readFrame()
labelRetval, labelFrame = labelFrameData.readFrame()
skeletonRetval, skeletonFrame = skeletonFrameData.readFrame()
if not depthRetval or not labelRetval or not skeletonRetval:
return
h, w = depthFrame.shape[0:2]
encodedFrame = frameConverter.encode(depthFrame, labelFrame, skeletonFrame)
if SAVE_RESULT_VIDEO:
videoPath = join(resultDirectory,
'croppedHand-' + fileName + videoFilenameExtension)
videoWriter = cv2.VideoWriter()
fourcc = cv2.cv.CV_FOURCC('m', 'p', '4', 'v')
videoWriter.open(videoPath, fourcc, 30, (w * 2, h))
else:
videoWriter = None
while (labelRetval and depthRetval and skeletonRetval):
encodedFrame = frameConverter.encode(depthFrame, labelFrame,
skeletonFrame)
res, resultImage = frameCallback(depthFrame, labelFrame, videoWriter)
resultImages.append(resultImage)
if VISUALIZE_RESULT:
if cv2.waitKey(1) & 0xFF == ord('q'):
depthRetval = False
continue
depthRetval, depthFrame = depthFrameData.readFrame()
labelRetval, labelFrame = labelFrameData.readFrame()
skeletonRetval, skeletonFrame = skeletonFrameData.readFrame()
depthFrameData.close()
labelFrameData.close()
skeletonFrameData.close()
if SAVE_RESULT_VIDEO & (videoWriter is not None):
message = "Saving Video..."
savingImage = np.zeros((h, w * 2, 3), np.uint8)
savingImageTextSize, _ = cv2.getTextSize(message,
cv2.FONT_HERSHEY_SIMPLEX, 1,
2)
savingImageTextLocation = (w - savingImageTextSize[0] / 2,
h / 2 - savingImageTextSize[1] / 2)
for i in xrange(0, len(resultImages)):
savingImage[:] = 0
cv2.rectangle(
savingImage, (0, savingImageTextLocation[1]),
(i * w * 2 / len(resultImages),
savingImageTextLocation[1] + savingImageTextSize[1]),
(0, 255, 0), cv2.cv.CV_FILLED)
cv2.putText(savingImage, message, savingImageTextLocation,
cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 2,
cv2.CV_AA)
cv2.imshow('Depth Video', savingImage)
videoWriter.write(resultImages[i])
class HandShapeServer(object):
def __init__(self, port):
shutil.rmtree('/home/lzz/sign/realtime/handshape')
shutil.rmtree('/home/lzz/sign/realtime/rawshape')
os.mkdir('/home/lzz/sign/realtime/handshape')
os.mkdir('/home/lzz/sign/realtime/rawshape')
self.fr=0
self.one=0
self.exist=0
self.lock = threading.Lock()
self.condition=threading.Condition()
self.classifier=Classifier()
self.classifier.filelist.append('/home/lzz/sign/svm/')
self.classifier.dic['/home/lzz/sign/svm/']=SignWord('/home/lzz/sign/svm/',0)
self.classifier.dic['/home/lzz/sign/svm/'].skeletons=[]
self.classifier.dic['/home/lzz/sign/svm/'].positions=[]
#f=self.dic[path].loadData()
#self.classifier.buildDic('/home/lzz/svm/',1)
self.converter = FrameConverter()
self.caffedl=caffeDL('/home/lzz/sign/proto/lenet_test.prototxt','/home/lzz/sign/model/lenet_iter_5000.caffemodel')
self.caffedlInter=caffeDL('/home/lzz/sign/proto_inter/lenet_test.prototxt','/home/lzz/sign/model/lenet__iter_400.caffemodel')
self.signDic={}
self.queue = Queue.Queue(maxsize = 10000)
Label1=open('/home/lzz/sign/svm/dictionary.csv','rb')
reader = csv.reader(Label1)
for row in reader:
self.signDic[int(row[0])]=row[1]
self.server = EchoServer(port, self.received_data)
thread.start_new_thread(self.threadfunc,())
asyncore.loop()
def received_data(self, received_data):
#print (len(received_data), binascii.crc32(received_data))
#decompressed_data = bz2.decompress(received_data)
decoded_data = self.converter.decode(received_data)
return self.process_data(decoded_data)
'''def process_data_handshape(self, decoded_data):
if(self.queue.full()==0):
self.condition.acquire()
self.queue.put(decoded_data)
self.condition.notify()
self.condition.release()
else:
self.server.send("full"+"#TERMINATOR#")
def threadfunc_handshape(self):
while(1):
if(self.queue.empty()==1):
self.condition.acquire()
self.condition.wait()
self.condition.release()
decoded_data=self.queue.get()
img=decoded_data['right']
sp=img.shape
if sp[0]>sp[1]:
img2=cv2.copyMakeBorder(img, 0,0, int(abs(sp[0]-sp[1])/2),int(abs(sp[0]-sp[1])/2), cv2.BORDER_CONSTANT, value=(0, 0, 0, 0))
else:
img2=cv2.copyMakeBorder(img, int(abs(sp[0]-sp[1])/2),int(abs(sp[0]-sp[1])/2),0,0, cv2.BORDER_CONSTANT, value=(0, 0, 0, 0))
img3=cv2.resize(img2,(128,128))
img3=img3/255.0
self.classifier.batch.append(img3)
self.caffedl.net.predict(self.classifier.batch,False)
feat = self.caffedl.net.blobs['prob'].data[0].flatten().tolist()
maxi=0
result=-1
for i in range(len(feat)):
if maxi<feat[i]:
maxi=feat[i]
result=i
print result
self.server.handler.send(str(result)+'#TERMINATOR#')'''
def process_data(self, decoded_data):
#print decoded_data
if(self.queue.full()==0):
if self.exist==0:
if decoded_data["label"]=='End':
if self.lock.acquire():
self.exist=1#exist means there is a End in the queue, that is, a complete sign exists
self.lock.release()
else:
if self.lock.acquire():
self.queue.queue.clear()
self.classifier.buffer=[]
self.exist=0
self.lock.release()
'''timedata={}
timedata['label']='time'
timedata['position']=datetime.datetime.now()
#print "starttime",timedata['position']
self.queue.put(timedata)'''
self.queue.put(decoded_data)
self.condition.acquire()
self.condition.notify()
self.condition.release()
else:
self.server.handler.send("Redo"+'#TERMINATOR#')
if self.lock.acquire():
self.queue.queue.clear()
self.classifier.buffer=[]
self.exist=0
self.lock.release()
return "0"
def threadfunc(self):
while(1):
self.condition.acquire()
if(self.queue.empty()==1):
self.condition.wait()
self.condition.release()
decoded_data=self.queue.get()
if self.lock.acquire():
self.classifier.buffer.append(decoded_data)
self.lock.release()
if(decoded_data["label"]=="End"):
#print len(self.classifier.buffer)
if self.lock.acquire():
self.exist=0
self.lock.release()
if len(self.classifier.buffer)<10:
self.server.handler.send("Redo"+"#TERMINATOR#")
else:
self.process_all()
if self.lock.acquire():
self.classifier.buffer=[]
self.lock.release()
'''starttimedic=self.queue.get()
starttime=starttimedic['position']
endtime = datetime.datetime.now()
print "endtime",endtime
print (endtime - starttime).seconds
if (endtime - starttime).seconds<60 and len(self.classifier.buffer)>10:'''
def ret(self):
self.fr=0
self.one=0
self.classifier.reset()
def process_all(self):
path="/home/lzz/sign/svm/"
buf=self.classifier.buffer
for decoded_data in buf:
if decoded_data["label"]=="time":
self.ret()
return
if(decoded_data["label"]=="End"):
break
read_index=[1,3,4,5,6,7,8,9]
raw=decoded_data["skeleton"].split(",")
posraw=decoded_data["position"].split(",")
if raw=='untracked' or posraw=='':
continue
skeleton=[]
position=[]
if raw!='':
raw=[0]+raw
for index in read_index:
skeleton.append(float(raw[index*7+1]))
skeleton.append(float(raw[index*7+2]))
skeleton.append(float(raw[index*7+3]))
if self.one==0:
#print self.classifier.dic
self.classifier.dic[path].headpos=[int(raw[4]),int(raw[5])]
self.classifier.dic[path].shoulder=abs(int(raw[11])-int(raw[25]))
self.classifier.dic[path].hip=int(raw[89])
self.classifier.dic[path].tall=int(raw[89])-int(raw[5])
self.one=1
if len(position)>5:
for i in range(4,6):
position.append(int(posraw[i]))
for i in range(4,6):
position.append(int(posraw[i]))
else:
for i in range(4):
position.append(int(posraw[i]))
self.classifier.dic[path].positions.append(position)
self.classifier.dic[path].skeletons.append(skeleton)
else:
if len(position)>5:
for i in range(4,6):
position.append(int(posraw[i]))
for i in range(4,6):
position.append(int(posraw[i]))
else:
for i in range(4):
position.append(int(posraw[i]))
self.classifier.dic[path].positions.append(position)
rightimg=None
leftimg=None
ftype=decoded_data['label']
if decoded_data['right']!=None:
img=decoded_data['right']
rightimg=img
cv2.imwrite("/home/lzz/sing/svm/handshaperight/"+str(self.fr)+str(ftype)+".bmp",rightimg)
if decoded_data['left']!=None:
img=decoded_data['left']
leftimg=img
cv2.imwrite("/home/lzz/sign/svm/handshape/"+str(self.fr)+"l.bmp",leftimg)
self.classifier.dic[path].dict[self.fr]=frame(self.fr,skeleton,ftype,rightimg,leftimg,position,0)
self.classifier.dic[path].framelist.append(self.fr)
self.fr+=1
if self.classifier.dic[path].skeletons==[] or self.classifier.dic[path].positions==[]:
self.fr=0
self.one=0
self.classifier.reset()
return
#self.classifier.dic[path].heightenough()
self.classifier.dic[path].getBothSeparate()
self.classifier.dic[path].getInter()
self.classifier.dic[path].getVelo()
self.classifier.dic[path].getVelocity()
self.classifier.dic[path].findTopHandshape()
self.classifier.getHogFeature()
try:
ret=self.classifier.dic[path].consTrajectory()
except:
print "Exception in user code:"
print '-'*60
traceback.print_exc(file=sys.stdout)
print '-'*60
ret=0
if ret==0:
self.ret()
return
#self.slword.displacement=hodmodule.hod(self.slword.skeletons)
#self.slword.checkDecisionTreeInter()
#self.slword.getHogFeature()
'''import glob
imgset=glob.glob("/home/lzz/1/*.bmp")
for i in imgset:
img=cv2.imread(i)
img=img/255.0
self.slword.batch.append(img)'''
'''print "batch"
print len(self.slword.batch)
if len(self.slword.batch)==0:
self.fr=0
del self.slword
self.slword=slWord.SlWord()
return
self.caffedl.net.predict(self.slword.batch,False)
feature=[]
for s in range(self.slword.keyNo):
feat = self.caffedl.net.blobs['ip1'].data[s].flatten().tolist()
feature.append(feat)
#print feature
self.slword.handshape=self.slword.pooling(feature,1)'''
self.classifier.getCaffeFeature(self.caffedl.net,self.caffedlInter.net)
self.classifier.separateCaffeFeature()
'''if self.slword.bothseparate==1:
leftfeature=[]
for s in range(self.slword.leftkeyNo):
feat = self.caffedl.net.blobs['ip1'].data[self.slword.keyNo+s].flatten().tolist()
leftfeature.append(feat)
self.slword.lefthandshape=self.slword.pooling(leftfeature,1)'''
self.classifier.dic[path].combineFeature()
if self.classifier.dic[path].intersect==0 and self.classifier.dic[path].bothseparate==0:
print "class 1"
svmModel= svm_load_model("/home/lzz/sign/svm/single.model")
elif self.classifier.dic[path].intersect==0 and self.classifier.dic[path].bothseparate==1:
print "class 2"
svmModel= svm_load_model("/home/lzz/sign/svm/both.model")
elif self.classifier.dic[path].intersect==1 and self.classifier.dic[path].bothseparate==0:
print "class 3"
svmModel= svm_load_model("/home/lzz/sign/svm/inter.model")
svm_res1=test_svm_model(svmModel,[0],[self.classifier.dic[path].combinedFeature])
#print "result",svm_res1
pred_labels=svm_res1[0]
print self.signDic[int(pred_labels[0])]
self.server.handler.send(self.signDic[int(pred_labels[0])]+'#TERMINATOR#')
self.ret()
def flatten(self, d, parent_key='', sep='.'):
items = []
for k, v in d.items():
new_key = parent_key + sep + k if parent_key else k
if isinstance(v, collections.MutableMapping):
items.extend(self.flatten(v, new_key, sep=sep).items())
else:
items.append((new_key, v))
return dict(items)
def play(self,pred_labels):
modeldic={}
modelindex=open("/home/lzz/ModelIndex.txt","r")
for line in open("/home/lzz/ModelIndex.txt"):
line = modelindex.readline()
sep=line.split(" ")
modeldic[sep[0]]=sep[1]
wordname=modeldic[str(int(pred_labels))]
dirs=os.listdir("/media/lzz/Data1/Aaron/1-250/")
for dir in dirs:
if(dir.find(wordname)):
video=dir
break
filename = "/media/lzz/Data1/Aaron/1-250/"+video+"/"+video+"_c.avi"
#print filename
win_name=filename
capture = cv2.cv.CaptureFromFile(filename)
cv2.cv.NamedWindow(win_name, cv2.cv.CV_WINDOW_AUTOSIZE)
while 1:
image = cv2.cv.QueryFrame(capture)
cv2.cv.ShowImage(win_name, image)
c = cv2.cv.WaitKey(33)
if(image==None):
break
cv2.cv.DestroyWindow(win_name)
class Edu_Server(object):
def __init__(self, port):
f=open('/media/lzz/65c50da0-a3a2-4117-8a72-7b37fd81b574/sign/project/src/EducationNew.pickle')
self.pickle=pickle.load(f)
f.close()
self.fr=0
self.one=0
self.exist=0
self.lock = threading.Lock()
self.condition=threading.Condition()
self.cond=threading.Condition()
self.classifier = EduClassifier()
self.p_thres=0.4
self.h_thres=0.5
self.h_thres_test=0.3
self.converter = FrameConverter()
self.caffedl=caffeDL('../../proto/lenet_test_edu.prototxt','../../model/lenet_iter_5000.caffemodel')
self.caffedlInter=caffeDL('../../proto_inter/lenet_test_edu.prototxt','../../model/lenet__iter_400.caffemodel')
self.signDic={}
self.queue = Queue.Queue(maxsize = 10000)
self.server = EchoServer(port, self.received_data)
self.mode=''
thread.start_new_thread(self.threadfunc,())
asyncore.loop()
def received_data(self, received_data):
decoded_data = self.converter.decode(received_data)
return self.process_data(decoded_data)
def process_data(self, decoded_data):
#print decoded_data
if decoded_data["label"]=='guide':
word_name=decoded_data['wordname']
self.mode='guide'
self.standard=self.pickle[word_name]
self.stepno=0
if self.lock.acquire():
self.queue.queue.clear()
self.lock.release()
self.fr=0
self.one=0
self.cond.acquire()
self.cond.notify()
self.cond.release()
self.classifier.reset()
return '0'
elif decoded_data['label']=='evaluation':
print 'evaluation'
word_name=decoded_data['wordname']
self.mode='evaluation'
self.standard=self.pickle[word_name]
self.queue.queue.clear()
self.buffer=[]
self.fr=0
self.one=0
self.cond.acquire()
self.cond.notify()
self.cond.release()
print word_name
self.classifier.reset()
return '0'
elif self.mode=='guide':
self.queue.put(decoded_data)
self.condition.acquire()
self.condition.notify()
self.condition.release()
#self.process_oneframe(decoded_data)
return '0'
elif self.mode=='evaluation':
if(self.queue.full()==0):
if self.exist==0:
if decoded_data["label"]=='End':
if self.lock.acquire():
self.exist=1#exist means there is a End in the queue, that is, a complete sign exists
self.lock.release()
else:
if self.lock.acquire():
self.queue.queue.clear()
self.classifier.buffer=[]
self.exist=0
self.lock.release()
self.queue.put(decoded_data)
self.condition.acquire()
self.condition.notify()
self.condition.release()
else:
self.server.handler.send("Redo"+'#TERMINATOR#')
if self.lock.acquire():
self.queue.queue.clear()
self.classifier.buffer=[]
self.exist=0
self.lock.release()
return '0'
def threadfunc(self):
while(1):
if(self.mode==''):
self.cond.acquire()
self.cond.wait()
self.cond.release()
if self.mode=='guide':
self.condition.acquire()
if(self.queue.empty()==1):
self.condition.wait()
self.condition.release()
while self.queue._qsize()>1:
self.queue.get()
self.process_oneframe(self.queue.get())
if self.mode=='evaluation':
self.condition.acquire()
if(self.queue.empty()==1):
self.condition.wait()
self.condition.release()
decoded_data=self.queue.get()
if self.lock.acquire():
self.classifier.buffer.append(decoded_data)
self.lock.release()
if(decoded_data["label"]=="End"):
if self.lock.acquire():
self.exist=0
self.lock.release()
if len(self.classifier.buffer)<10:
self.server.handler.send("Redo"+"#TERMINATOR#")
else:
self.process_all()
if self.lock.acquire():
self.classifier.buffer=[]
self.lock.release()
def process_oneframe(self,decoded_data):
path="/home/lzz/sign/svm/"
read_index=[1,3,4,5,6,7,8,9]
raw=decoded_data["skeleton"].split(",")
posraw=decoded_data["position"].split(",")
if raw=='untracked' or posraw=='':
return
skeleton=[]
if raw!='':
raw=[0]+raw
for index in read_index:
skeleton.append(float(raw[index*7+1]))
skeleton.append(float(raw[index*7+2]))
skeleton.append(float(raw[index*7+3]))
if self.one==0:
self.classifier.dic[path].headpos=[int(raw[4]),int(raw[5])]
self.classifier.dic[path].shoulder=abs(int(raw[11])-int(raw[25]))
self.classifier.dic[path].hip=int(raw[89])
self.classifier.dic[path].tall=int(raw[89])-int(raw[5])
self.one=1
if self.stepno>=len(self.standard):
#print self.stepno,len(self.standard)
return '0'
step=self.standard[self.stepno]
pos0=step['pos']
pos1=[(float(posraw[0])-self.classifier.dic[path].headpos[0])/self.classifier.dic[path].shoulder,
(float(posraw[1])-self.classifier.dic[path].headpos[1])/self.classifier.dic[path].tall,
(float(posraw[2])-self.classifier.dic[path].headpos[0])/self.classifier.dic[path].shoulder,
(float(posraw[3])-self.classifier.dic[path].headpos[1])/self.classifier.dic[path].tall,
]
tickle=0
if decoded_data['label']==step['type']:
if step['type']=='Right':
#print pos0,pos1
print 'pos',np.sqrt((pos1[0]-pos0[0])**2+(pos1[1]-pos0[1])**2)
if np.sqrt((pos1[0]-pos0[0])**2+(pos1[1]-pos0[1])**2)<0.4:
start=time.time()
h0=step['handshape']
img=decoded_data['right']
if img!=None:
sp=img.shape
if sp[0]>sp[1]:
img2=cv2.copyMakeBorder(img, 0,0, int(abs(sp[0]-sp[1])/2),int(abs(sp[0]-sp[1])/2), cv2.BORDER_CONSTANT, value=(0, 0, 0, 0))
else:
img2=cv2.copyMakeBorder(img, int(abs(sp[0]-sp[1])/2),int(abs(sp[0]-sp[1])/2),0,0, cv2.BORDER_CONSTANT, value=(0, 0, 0, 0))
img3=cv2.resize(img2,(128,128))
img3=img3/255.0
self.caffedl.net.predict([img3],False)
h1 = self.caffedl.net.blobs['ip1'].data[0].flatten().tolist()
print 'hand',1 - spatial.distance.cosine(h0,h1)
if 1 - spatial.distance.cosine(h0,h1)>self.h_thres:
self.stepno+=1
tickle=1
print 'time:',time.time()-start
elif step['type']=='Both':
if np.sqrt((pos1[0]-pos0[0])**2+(pos1[1]-pos0[1])**2)<self.p_thres and np.sqrt((pos1[2]-pos0[2])**2+(pos1[3]-pos0[3])**2)<self.p_thres:
h0=step['handshape']
lefth0=step['lefthandshape']
img=decoded_data['right']
leftimg=decoded_data['left']
if img!=None and leftimg!=None:
sp=img.shape
if sp[0]>sp[1]:
img2=cv2.copyMakeBorder(img, 0,0, int(abs(sp[0]-sp[1])/2),int(abs(sp[0]-sp[1])/2), cv2.BORDER_CONSTANT, value=(0, 0, 0, 0))
else:
img2=cv2.copyMakeBorder(img, int(abs(sp[0]-sp[1])/2),int(abs(sp[0]-sp[1])/2),0,0, cv2.BORDER_CONSTANT, value=(0, 0, 0, 0))
img3=cv2.resize(img2,(128,128))
imgr=img3/255.0
sp=leftimg.shape
if sp[0]>sp[1]:
img2=cv2.copyMakeBorder(leftimg, 0,0, int(abs(sp[0]-sp[1])/2),int(abs(sp[0]-sp[1])/2), cv2.BORDER_CONSTANT, value=(0, 0, 0, 0))
else:
img2=cv2.copyMakeBorder(leftimg, int(abs(sp[0]-sp[1])/2),int(abs(sp[0]-sp[1])/2),0,0, cv2.BORDER_CONSTANT, value=(0, 0, 0, 0))
img3=cv2.resize(img2,(128,128))
imgl=img3/255.0
self.caffedl.net.predict([imgr,imgl],False)
h1r = self.caffedl.net.blobs['ip1'].data[0].flatten().tolist()
h1l = self.caffedl.net.blobs['ip1'].data[1].flatten().tolist()
if 1 - spatial.distance.cosine(h0,h1r)>self.h_thres and 1 - spatial.distance.cosine(lefth0,h1l)>self.h_thres:
self.stepno+=1
tickle=1
elif step['type']=='Intersect':
if np.sqrt((pos1[0]-pos0[0])**2+(pos1[1]-pos0[1])**2)<self.p_thres:
h0=step['handshape']
img=decoded_data['right']
if img!=None:
sp=img.shape
if sp[0]>sp[1]:
img2=cv2.copyMakeBorder(img, 0,0, int(abs(sp[0]-sp[1])/2),int(abs(sp[0]-sp[1])/2), cv2.BORDER_CONSTANT, value=(0, 0, 0, 0))
else:
img2=cv2.copyMakeBorder(img, int(abs(sp[0]-sp[1])/2),int(abs(sp[0]-sp[1])/2),0,0, cv2.BORDER_CONSTANT, value=(0, 0, 0, 0))
img3=cv2.resize(img2,(128,128))
img3=img3/255.0
self.caffedlInter.net.predict([img3],False)
h1 = self.caffedlInter.net.blobs['ip1'].data[0].flatten().tolist()
if 1 - spatial.distance.cosine(h0,h1)>self.h_thres:
self.stepno+=1
tickle=1
if tickle==1:
if self.stepno>=len(self.standard)-1:
self.server.handler.send("Finish"+'#TERMINATOR#')
else:
self.server.handler.send("Next"+'#TERMINATOR#')
def ret(self):
self.fr=0
self.one=0
self.classifier.reset()
def process_all(self):
print 'process all'
buf=self.classifier.buffer
buffer=buf
current=0
ticklelist={}
cnt=0
for s in range(len(self.standard)):
ticklelist[s]={}
validno=[]
belong=[]
for i in range(len(buf)):
belong.append(-1)
for s in range(len(self.standard)):
ticklelist[s]['position']=0
ticklelist[s]['handshape']=0
if self.standard[s]['type']=='Both':
ticklelist[s]['lefthandshape']=0
for i in range(current,len(buf)-1):
path="/home/lzz/sign/svm/"
read_index=[1,3,4,5,6,7,8,9]
raw=buf[i]["skeleton"].split(",")
posraw=buf[i]["position"].split(",")
if raw=='untracked' or posraw=='':
return
skeleton=[]
if raw!='':
raw=[0]+raw
for index in read_index:
skeleton.append(float(raw[index*7+1]))
skeleton.append(float(raw[index*7+2]))
skeleton.append(float(raw[index*7+3]))
if self.one==0:
self.classifier.dic[path].headpos=[int(raw[4]),int(raw[5])]
self.classifier.dic[path].shoulder=abs(int(raw[11])-int(raw[25]))
self.classifier.dic[path].hip=int(raw[89])
self.classifier.dic[path].tall=int(raw[89])-int(raw[5])
self.one=1
if s>=len(self.standard):
print s,len(self.standard)
return '0'
pos1=[(float(posraw[0])-self.classifier.dic[path].headpos[0])/self.classifier.dic[path].shoulder,
(float(posraw[1])-self.classifier.dic[path].headpos[1])/self.classifier.dic[path].tall,
(float(posraw[2])-self.classifier.dic[path].headpos[0])/self.classifier.dic[path].shoulder,
(float(posraw[3])-self.classifier.dic[path].headpos[1])/self.classifier.dic[path].tall,
]
decoded_data=buf[i]
step=self.standard[s]
pos0=step['pos']
tickle=0
if decoded_data['label']==step['type']:
if step['type']=='Right':
print 'pos',np.sqrt((pos1[0]-pos0[0])**2+(pos1[1]-pos0[1])**2)
if np.sqrt((pos1[0]-pos0[0])**2+(pos1[1]-pos0[1])**2)<self.p_thres:
cnt+=1
ticklelist[s]['position']=1
validno.append(i)
belong[i]=s
tickle=1
elif step['type']=='Both':
if np.sqrt((pos1[0]-pos0[0])**2+(pos1[1]-pos0[1])**2)<self.p_thres and np.sqrt((pos1[2]-pos0[2])**2+(pos1[3]-pos0[3])**2)<self.p_thres:
ticklelist[s]['position']=1
validno.append(i)
belong[i]=s
tickle=1
elif step['type']=='Intersect':
if np.sqrt((pos1[0]-pos0[0])**2+(pos1[1]-pos0[1])**2)<self.p_thres:
ticklelist[s]['position']=1
validno.append(i)
belong[i]=s
tickle=1
if tickle==1:
current=i
if i==len(buf)-1:
break
for s in range(len(self.standard)):
if ticklelist[s]['position']==0:
self.server.handler.send(str(ticklelist)+'#TERMINATOR#')
return
totallen=cnt
jump=max(1,int(totallen/30))
cnt=0
havetickled=[]
print belong
for u in range(len(validno)):
i=validno[u]
cnt+=1
step=self.standard[belong[u]]
if belong[u] in havetickled:
continue
if cnt%jump!=0:
continue
tickle=0
decoded_data=buffer[i]
if step['type']=='Right':
img=buffer[i]['right']
h0=step['handshape']
if img!=None:
sp=img.shape
if sp[0]>sp[1]:
img2=cv2.copyMakeBorder(img, 0,0, int(abs(sp[0]-sp[1])/2),int(abs(sp[0]-sp[1])/2), cv2.BORDER_CONSTANT, value=(0, 0, 0, 0))
else:
img2=cv2.copyMakeBorder(img, int(abs(sp[0]-sp[1])/2),int(abs(sp[0]-sp[1])/2),0,0, cv2.BORDER_CONSTANT, value=(0, 0, 0, 0))
img3=cv2.resize(img2,(128,128))
img3=img3/255.0
self.caffedl.net.predict([img3],False)
h1 = self.caffedl.net.blobs['ip1'].data[0].flatten().tolist()
print 'hand', 1 - spatial.distance.cosine(h0,h1),belong[i]
if 1 - spatial.distance.cosine(h0,h1)>self.h_thres_test:
tickle=1
print 'tickle',belong[i]
ticklelist[belong[i]]['handshape']=1
elif step['type']=='Both':
righttickle=0
lefttickle=0
img=buffer[i]['right']
h0=step['handshape']
if img!=None:
sp=img.shape
if sp[0]>sp[1]:
img2=cv2.copyMakeBorder(img, 0,0, int(abs(sp[0]-sp[1])/2),int(abs(sp[0]-sp[1])/2), cv2.BORDER_CONSTANT, value=(0, 0, 0, 0))
else:
img2=cv2.copyMakeBorder(img, int(abs(sp[0]-sp[1])/2),int(abs(sp[0]-sp[1])/2),0,0, cv2.BORDER_CONSTANT, value=(0, 0, 0, 0))
img3=cv2.resize(img2,(128,128))
img3=img3/255.0
self.caffedl.net.predict([img3],False)
h1 = self.caffedl.net.blobs['ip1'].data[0].flatten().tolist()
if 1 - spatial.distance.cosine(h0,h1)>self.h_thres_test:
righttickle=1
ticklelist[belong[i]]['handshape']=1
img=buffer[i]['left']
h0=step['lefthandshape']
if img!=None:
sp=img.shape
if sp[0]>sp[1]:
img2=cv2.copyMakeBorder(img, 0,0, int(abs(sp[0]-sp[1])/2),int(abs(sp[0]-sp[1])/2), cv2.BORDER_CONSTANT, value=(0, 0, 0, 0))
else:
img2=cv2.copyMakeBorder(img, int(abs(sp[0]-sp[1])/2),int(abs(sp[0]-sp[1])/2),0,0, cv2.BORDER_CONSTANT, value=(0, 0, 0, 0))
img3=cv2.resize(img2,(128,128))
img3=img3/255.0
self.caffedl.net.predict([img3],False)
h1 = self.caffedl.net.blobs['ip1'].data[0].flatten().tolist()
if 1 - spatial.distance.cosine(h0,h1)>self.h_thres_test:
lefttickle=1
ticklelist[belong[i]]['lefthandshape']=1
if righttickle==1 and lefttickle==1:
tickle=1
elif step['type']=='Intersect':
img=buffer[i]['right']
h0=step['handshape']
if img!=None:
sp=img.shape
if sp[0]>sp[1]:
img2=cv2.copyMakeBorder(img, 0,0, int(abs(sp[0]-sp[1])/2),int(abs(sp[0]-sp[1])/2), cv2.BORDER_CONSTANT, value=(0, 0, 0, 0))
else:
img2=cv2.copyMakeBorder(img, int(abs(sp[0]-sp[1])/2),int(abs(sp[0]-sp[1])/2),0,0, cv2.BORDER_CONSTANT, value=(0, 0, 0, 0))
img3=cv2.resize(img2,(128,128))
img3=img3/255.0
self.caffedl.net.predict([img3],False)
h1 = self.caffedl.net.blobs['ip1'].data[0].flatten().tolist()
if 1 - spatial.distance.cosine(h0,h1)>self.h_thres_test:
ticklelist[belong[i]]['handshape']=1
tickle=1
if tickle==1:
havetickled.append(belong[i])
if i==len(buf)-1:
break
self.server.handler.send(str(ticklelist)+'#TERMINATOR#')
def flatten(self, d, parent_key='', sep='.'):
items = []
for k, v in d.items():
new_key = parent_key + sep + k if parent_key else k
if isinstance(v, collections.MutableMapping):
items.extend(self.flatten(v, new_key, sep=sep).items())
else:
items.append((new_key, v))
return dict(items)
class HandShapeServer(object):
def __init__(self, port):
self.converter = FrameConverter()
self.slword = slWord.SlWord()
self.caffedl = caffeDL()
self.server = EchoServer(port, self.received_data)
def received_data(self, received_data):
print(len(received_data), binascii.crc32(received_data))
#decompressed_data = bz2.decompress(received_data)
decoded_data = self.converter.decode(received_data)
#import pprint
#pprint.pprint(decoded_data)
return self.process_data(decoded_data)
def process_data(self, decoded_data):
sleep_time = random.randint(1, 10)
if (decoded_data["label"] == "End"):
self.process_all()
return "0"
self.slword.loadSkeleton(decoded_data["skeleton"])
if (decoded_data['depth_image'] != ''):
print "skl"
img = decoded_data['depth_image']
sp = img.shape
print sp
img2 = cv2.copyMakeBorder(img,
0,
0,
int(abs(sp[0] - sp[1]) / 2),
int(abs(sp[0] - sp[1]) / 2),
cv2.BORDER_CONSTANT,
value=(0, 0, 0, 0))
img3 = cv2.resize(img2, (128, 128))
img3 = img3 / 255.0
self.slword.imgset.append(img3)
return "0"
else:
self.slword.imgset.append([])
return "0"
def play(self, pred_labels):
modeldic = {}
modelindex = open("/home/lzz/ModelIndex.txt", "r")
for line in open("/home/lzz/ModelIndex.txt"):
line = modelindex.readline()
sep = line.split(" ")
modeldic[sep[0]] = sep[1]
wordname = modeldic[str(int(pred_labels))]
dirs = os.listdir("/media/lzz/Data1/Aaron/1-250/")
for dir in dirs:
if (dir.find(wordname)):
video = dir
break
filename = "/media/lzz/Data1/Aaron/1-250/" + video + "/" + video + "_c.avi"
print filename
win_name = filename
capture = cv2.cv.CaptureFromFile(filename)
cv2.cv.NamedWindow(win_name, cv2.cv.CV_WINDOW_AUTOSIZE)
while 1:
image = cv2.cv.QueryFrame(capture)
cv2.cv.ShowImage(win_name, image)
c = cv2.cv.WaitKey(33)
if (image == None):
break
cv2.cv.DestroyWindow(win_name)
def process_all(self):
self.slword.displacement, rightV, rightH = hodmodule.hod(
self.slword.skeletons)
print self.slword.displacement, rightV
print len(self.slword.displacement), len(rightV)
self.slword.getVelocity(rightV, rightH)
self.slword.findTopHandshape()
self.caffedl.net.predict(self.slword.batch)
feature = []
for s in range(len(self.slword.batch)):
feat = self.caffedl.net.blobs['ip1'].data[s].flatten().tolist()
feature.append(feat)
self.slword.handshape = self.slword.pooling(feature, 1)
self.slword.combineFeature()
svmModel = svm_load_model("/home/lzz/svmModel")
print self.slword.combinedFeature
print len(self.slword.combinedFeature)
svm_res1 = test_svm_model(0, self.slword.combinedFeature, svmModel)
pred_labels = svm_res1[0]
print "pred_labels", pred_labels
self.play(pred_labels)
def flatten(self, d, parent_key='', sep='.'):
items = []
for k, v in d.items():
new_key = parent_key + sep + k if parent_key else k
if isinstance(v, collections.MutableMapping):
items.extend(self.flatten(v, new_key, sep=sep).items())
else:
items.append((new_key, v))
return dict(items)
def __init__(self, port):
self.converter = FrameConverter()
self.slword = slWord.SlWord()
self.caffedl = caffeDL()
self.server = EchoServer(port, self.received_data)
depthFrameData.load(srcVideoPath)
srcVideoPath = join(videoDirectory,firstFile+colorVideoSuffix+videoFilenameExtension)
colorFrameData = VideoFrameData()
colorFrameData.load(srcVideoPath)
i = 0
resultImages = []
depthRetval,depthFrame = depthFrameData.readFrame()
colorRetval,colorFrame = colorFrameData.readFrame()
labelRetval,labelFrame = labelFrameData.readFrame()
skeletonRetval,skeletonFrame = skeletonFrameData.readFrame()
if not depthRetval or not colorRetval or not labelRetval or not skeletonRetval:
exit
encodedFrame = FrameConverter().encode(depthFrame, colorFrame, labelFrame, skeletonFrame)
plt.subplot(1,2,1), plt.imshow(depthFrame)
plt.subplot(1,2,2), plt.imshow(colorFrame)
plt.show()
argc = len(sys.argv)
if argc > 1:
host = sys.argv[1]
else:
host = 'localhost'
client = HandShapeClient(host, port)
client.send_data(encodedFrame)
def readVideo(fileName, frameCallback, result):
frameConverter = FrameConverter()
labelFrameData = CSVFrameData()
labelFrameData.load(join(labelDirectory, fileName))
labelFrameData.setDebug(DEBUG_MODE)
fileName = fileName[:-4]
skeletonFrameData = CSVFrameData()
skeletonFrameData.load(join(skeletonDirectory, fileName+skeletonVideoSuffix+skeletonFilenameExtension))
skeletonFrameData.setDebug(DEBUG_MODE)
srcVideoPath = join(videoDirectory,fileName+depthVideoSuffix+videoFilenameExtension)
depthFrameData = VideoFrameData()
depthFrameData.load(srcVideoPath)
i = 0
resultImages = []
depthRetval,depthFrame = depthFrameData.readFrame()
labelRetval,labelFrame = labelFrameData.readFrame()
skeletonRetval,skeletonFrame = skeletonFrameData.readFrame()
if not depthRetval or not labelRetval or not skeletonRetval:
return
h,w = depthFrame.shape[0:2]
encodedFrame = frameConverter.encode(depthFrame, labelFrame, skeletonFrame)
if SAVE_RESULT_VIDEO:
videoPath = join(resultDirectory, 'croppedHand-'+fileName+videoFilenameExtension)
videoWriter = cv2.VideoWriter()
fourcc = cv2.cv.CV_FOURCC('m', 'p', '4', 'v')
videoWriter.open(videoPath, fourcc, 30, (w*2,h))
else:
videoWriter = None
while(labelRetval and depthRetval and skeletonRetval):
encodedFrame = frameConverter.encode(depthFrame, labelFrame, skeletonFrame)
res,resultImage = frameCallback(depthFrame, labelFrame, videoWriter)
resultImages.append(resultImage)
if VISUALIZE_RESULT:
if cv2.waitKey(1) & 0xFF == ord('q'):
depthRetval = False
continue
depthRetval,depthFrame = depthFrameData.readFrame()
labelRetval,labelFrame = labelFrameData.readFrame()
skeletonRetval,skeletonFrame = skeletonFrameData.readFrame()
depthFrameData.close()
labelFrameData.close()
skeletonFrameData.close()
if SAVE_RESULT_VIDEO & (videoWriter is not None):
message = "Saving Video..."
savingImage = np.zeros((h,w*2,3), np.uint8)
savingImageTextSize, _ = cv2.getTextSize(message, cv2.FONT_HERSHEY_SIMPLEX, 1, 2)
savingImageTextLocation = (w-savingImageTextSize[0]/2, h/2-savingImageTextSize[1]/2)
for i in xrange(0,len(resultImages)):
savingImage[:] = 0
cv2.rectangle(savingImage, (0,savingImageTextLocation[1]), (i*w*2/len(resultImages),savingImageTextLocation[1]+savingImageTextSize[1]), (0,255,0), cv2.cv.CV_FILLED)
cv2.putText(savingImage, message, savingImageTextLocation, cv2.FONT_HERSHEY_SIMPLEX, 1, (255,255,255), 2, cv2.CV_AA)
cv2.imshow('Depth Video', savingImage)
videoWriter.write(resultImages[i])
def __init__(self, host, port):
self.client = EchoClient(host, port, self)
self.converter = FrameConverter()
def __init__(self, port):
self.converter = FrameConverter()
self.slword=slWord.SlWord()
self.caffedl=caffeDL()
self.server = EchoServer(port, self.received_data)
class HandShapeServer(object):
def __init__(self, port):
self.converter = FrameConverter()
self.slword=slWord.SlWord()
self.caffedl=caffeDL()
self.server = EchoServer(port, self.received_data)
def received_data(self, received_data):
print (len(received_data), binascii.crc32(received_data))
#decompressed_data = bz2.decompress(received_data)
decoded_data = self.converter.decode(received_data)
#import pprint
#pprint.pprint(decoded_data)
return self.process_data(decoded_data)
def process_data(self, decoded_data):
sleep_time = random.randint(1, 10)
if(decoded_data["label"]=="End"):
self.process_all()
return "0"
self.slword.loadSkeleton(decoded_data["skeleton"])
if(decoded_data['depth_image']!=''):
print "skl"
img=decoded_data['depth_image']
sp=img.shape
print sp
img2=cv2.copyMakeBorder(img, 0,0, int(abs(sp[0]-sp[1])/2),int(abs(sp[0]-sp[1])/2), cv2.BORDER_CONSTANT, value=(0, 0, 0, 0))
img3=cv2.resize(img2,(128,128))
img3=img3/255.0
self.slword.imgset.append(img3)
return "0"
else:
self.slword.imgset.append([])
return "0"
def play(self,pred_labels):
modeldic={}
modelindex=open("/home/lzz/ModelIndex.txt","r")
for line in open("/home/lzz/ModelIndex.txt"):
line = modelindex.readline()
sep=line.split(" ")
modeldic[sep[0]]=sep[1]
wordname=modeldic[str(int(pred_labels))]
dirs=os.listdir("/media/lzz/Data1/Aaron/1-250/")
for dir in dirs:
if(dir.find(wordname)):
video=dir
break
filename = "/media/lzz/Data1/Aaron/1-250/"+video+"/"+video+"_c.avi"
print filename
win_name=filename
capture = cv2.cv.CaptureFromFile(filename)
cv2.cv.NamedWindow(win_name, cv2.cv.CV_WINDOW_AUTOSIZE)
while 1:
image = cv2.cv.QueryFrame(capture)
cv2.cv.ShowImage(win_name, image)
c = cv2.cv.WaitKey(33)
if(image==None):
break
cv2.cv.DestroyWindow(win_name)
def process_all(self):
self.slword.displacement,rightV,rightH=hodmodule.hod(self.slword.skeletons)
print self.slword.displacement,rightV
print len(self.slword.displacement),len(rightV)
self.slword.getVelocity(rightV,rightH)
self.slword.findTopHandshape()
self.caffedl.net.predict(self.slword.batch)
feature=[]
for s in range(len(self.slword.batch)):
feat = self.caffedl.net.blobs['ip1'].data[s].flatten().tolist()
feature.append(feat)
self.slword.handshape=self.slword.pooling(feature,1)
self.slword.combineFeature()
svmModel= svm_load_model("/home/lzz/svmModel")
print self.slword.combinedFeature
print len(self.slword.combinedFeature)
svm_res1=test_svm_model(0,self.slword.combinedFeature,svmModel)
pred_labels=svm_res1[0];
print "pred_labels",pred_labels
self.play(pred_labels)
def flatten(self, d, parent_key='', sep='.'):
items = []
for k, v in d.items():
new_key = parent_key + sep + k if parent_key else k
if isinstance(v, collections.MutableMapping):
items.extend(self.flatten(v, new_key, sep=sep).items())
else:
items.append((new_key, v))
return dict(items)
