def __init__(self, port): f=open('/media/lzz/65c50da0-a3a2-4117-8a72-7b37fd81b574/sign/project/src/EducationNew.pickle') self.pickle=pickle.load(f) f.close() self.fr=0 self.one=0 self.exist=0 self.lock = threading.Lock() self.condition=threading.Condition() self.cond=threading.Condition() self.classifier = EduClassifier() self.p_thres=0.4 self.h_thres=0.5 self.h_thres_test=0.3 self.converter = FrameConverter() self.caffedl=caffeDL('../../proto/lenet_test_edu.prototxt','../../model/lenet_iter_5000.caffemodel') self.caffedlInter=caffeDL('../../proto_inter/lenet_test_edu.prototxt','../../model/lenet__iter_400.caffemodel') self.signDic={} self.queue = Queue.Queue(maxsize = 10000) self.server = EchoServer(port, self.received_data) self.mode='' thread.start_new_thread(self.threadfunc,()) asyncore.loop()
def __init__(self, port): shutil.rmtree('/home/lzz/sign/realtime/handshape') shutil.rmtree('/home/lzz/sign/realtime/rawshape') os.mkdir('/home/lzz/sign/realtime/handshape') os.mkdir('/home/lzz/sign/realtime/rawshape') self.fr=0 self.one=0 self.exist=0 self.lock = threading.Lock() self.condition=threading.Condition() self.classifier=Classifier() self.classifier.filelist.append('/home/lzz/sign/svm/') self.classifier.dic['/home/lzz/sign/svm/']=SignWord('/home/lzz/sign/svm/',0) self.classifier.dic['/home/lzz/sign/svm/'].skeletons=[] self.classifier.dic['/home/lzz/sign/svm/'].positions=[] #f=self.dic[path].loadData() #self.classifier.buildDic('/home/lzz/svm/',1) self.converter = FrameConverter() self.caffedl=caffeDL('/home/lzz/sign/proto/lenet_test.prototxt','/home/lzz/sign/model/lenet_iter_5000.caffemodel') self.caffedlInter=caffeDL('/home/lzz/sign/proto_inter/lenet_test.prototxt','/home/lzz/sign/model/lenet__iter_400.caffemodel') self.signDic={} self.queue = Queue.Queue(maxsize = 10000) Label1=open('/home/lzz/sign/svm/dictionary.csv','rb') reader = csv.reader(Label1) for row in reader: self.signDic[int(row[0])]=row[1] self.server = EchoServer(port, self.received_data) thread.start_new_thread(self.threadfunc,()) asyncore.loop()
def readVideo(fileName, frameCallback, result): frameConverter = FrameConverter() labelFrameData = CSVFrameData() labelFrameData.load(join(labelDirectory, fileName)) labelFrameData.setDebug(DEBUG_MODE) fileName = fileName[:-4] skeletonFrameData = CSVFrameData() skeletonFrameData.load( join(skeletonDirectory, fileName + skeletonVideoSuffix + skeletonFilenameExtension)) skeletonFrameData.setDebug(DEBUG_MODE) srcVideoPath = join(videoDirectory, fileName + depthVideoSuffix + videoFilenameExtension) depthFrameData = VideoFrameData() depthFrameData.load(srcVideoPath) i = 0 resultImages = [] depthRetval, depthFrame = depthFrameData.readFrame() labelRetval, labelFrame = labelFrameData.readFrame() skeletonRetval, skeletonFrame = skeletonFrameData.readFrame() if not depthRetval or not labelRetval or not skeletonRetval: return h, w = depthFrame.shape[0:2] encodedFrame = frameConverter.encode(depthFrame, labelFrame, skeletonFrame) if SAVE_RESULT_VIDEO: videoPath = join(resultDirectory, 'croppedHand-' + fileName + videoFilenameExtension) videoWriter = cv2.VideoWriter() fourcc = cv2.cv.CV_FOURCC('m', 'p', '4', 'v') videoWriter.open(videoPath, fourcc, 30, (w * 2, h)) else: videoWriter = None while (labelRetval and depthRetval and skeletonRetval): encodedFrame = frameConverter.encode(depthFrame, labelFrame, skeletonFrame) res, resultImage = frameCallback(depthFrame, labelFrame, videoWriter) resultImages.append(resultImage) if VISUALIZE_RESULT: if cv2.waitKey(1) & 0xFF == ord('q'): depthRetval = False continue depthRetval, depthFrame = depthFrameData.readFrame() labelRetval, labelFrame = labelFrameData.readFrame() skeletonRetval, skeletonFrame = skeletonFrameData.readFrame() depthFrameData.close() labelFrameData.close() skeletonFrameData.close() if SAVE_RESULT_VIDEO & (videoWriter is not None): message = "Saving Video..." savingImage = np.zeros((h, w * 2, 3), np.uint8) savingImageTextSize, _ = cv2.getTextSize(message, cv2.FONT_HERSHEY_SIMPLEX, 1, 2) savingImageTextLocation = (w - savingImageTextSize[0] / 2, h / 2 - savingImageTextSize[1] / 2) for i in xrange(0, len(resultImages)): savingImage[:] = 0 cv2.rectangle( savingImage, (0, savingImageTextLocation[1]), (i * w * 2 / len(resultImages), savingImageTextLocation[1] + savingImageTextSize[1]), (0, 255, 0), cv2.cv.CV_FILLED) cv2.putText(savingImage, message, savingImageTextLocation, cv2.FONT_HERSHEY_SIMPLEX, 1, (255, 255, 255), 2, cv2.CV_AA) cv2.imshow('Depth Video', savingImage) videoWriter.write(resultImages[i])
class HandShapeServer(object): def __init__(self, port): shutil.rmtree('/home/lzz/sign/realtime/handshape') shutil.rmtree('/home/lzz/sign/realtime/rawshape') os.mkdir('/home/lzz/sign/realtime/handshape') os.mkdir('/home/lzz/sign/realtime/rawshape') self.fr=0 self.one=0 self.exist=0 self.lock = threading.Lock() self.condition=threading.Condition() self.classifier=Classifier() self.classifier.filelist.append('/home/lzz/sign/svm/') self.classifier.dic['/home/lzz/sign/svm/']=SignWord('/home/lzz/sign/svm/',0) self.classifier.dic['/home/lzz/sign/svm/'].skeletons=[] self.classifier.dic['/home/lzz/sign/svm/'].positions=[] #f=self.dic[path].loadData() #self.classifier.buildDic('/home/lzz/svm/',1) self.converter = FrameConverter() self.caffedl=caffeDL('/home/lzz/sign/proto/lenet_test.prototxt','/home/lzz/sign/model/lenet_iter_5000.caffemodel') self.caffedlInter=caffeDL('/home/lzz/sign/proto_inter/lenet_test.prototxt','/home/lzz/sign/model/lenet__iter_400.caffemodel') self.signDic={} self.queue = Queue.Queue(maxsize = 10000) Label1=open('/home/lzz/sign/svm/dictionary.csv','rb') reader = csv.reader(Label1) for row in reader: self.signDic[int(row[0])]=row[1] self.server = EchoServer(port, self.received_data) thread.start_new_thread(self.threadfunc,()) asyncore.loop() def received_data(self, received_data): #print (len(received_data), binascii.crc32(received_data)) #decompressed_data = bz2.decompress(received_data) decoded_data = self.converter.decode(received_data) return self.process_data(decoded_data) '''def process_data_handshape(self, decoded_data): if(self.queue.full()==0): self.condition.acquire() self.queue.put(decoded_data) self.condition.notify() self.condition.release() else: self.server.send("full"+"#TERMINATOR#") def threadfunc_handshape(self): while(1): if(self.queue.empty()==1): self.condition.acquire() self.condition.wait() self.condition.release() decoded_data=self.queue.get() img=decoded_data['right'] sp=img.shape if sp[0]>sp[1]: img2=cv2.copyMakeBorder(img, 0,0, int(abs(sp[0]-sp[1])/2),int(abs(sp[0]-sp[1])/2), cv2.BORDER_CONSTANT, value=(0, 0, 0, 0)) else: img2=cv2.copyMakeBorder(img, int(abs(sp[0]-sp[1])/2),int(abs(sp[0]-sp[1])/2),0,0, cv2.BORDER_CONSTANT, value=(0, 0, 0, 0)) img3=cv2.resize(img2,(128,128)) img3=img3/255.0 self.classifier.batch.append(img3) self.caffedl.net.predict(self.classifier.batch,False) feat = self.caffedl.net.blobs['prob'].data[0].flatten().tolist() maxi=0 result=-1 for i in range(len(feat)): if maxi<feat[i]: maxi=feat[i] result=i print result self.server.handler.send(str(result)+'#TERMINATOR#')''' def process_data(self, decoded_data): #print decoded_data if(self.queue.full()==0): if self.exist==0: if decoded_data["label"]=='End': if self.lock.acquire(): self.exist=1#exist means there is a End in the queue, that is, a complete sign exists self.lock.release() else: if self.lock.acquire(): self.queue.queue.clear() self.classifier.buffer=[] self.exist=0 self.lock.release() '''timedata={} timedata['label']='time' timedata['position']=datetime.datetime.now() #print "starttime",timedata['position'] self.queue.put(timedata)''' self.queue.put(decoded_data) self.condition.acquire() self.condition.notify() self.condition.release() else: self.server.handler.send("Redo"+'#TERMINATOR#') if self.lock.acquire(): self.queue.queue.clear() self.classifier.buffer=[] self.exist=0 self.lock.release() return "0" def threadfunc(self): while(1): self.condition.acquire() if(self.queue.empty()==1): self.condition.wait() self.condition.release() decoded_data=self.queue.get() if self.lock.acquire(): self.classifier.buffer.append(decoded_data) self.lock.release() if(decoded_data["label"]=="End"): #print len(self.classifier.buffer) if self.lock.acquire(): self.exist=0 self.lock.release() if len(self.classifier.buffer)<10: self.server.handler.send("Redo"+"#TERMINATOR#") else: self.process_all() if self.lock.acquire(): self.classifier.buffer=[] self.lock.release() '''starttimedic=self.queue.get() starttime=starttimedic['position'] endtime = datetime.datetime.now() print "endtime",endtime print (endtime - starttime).seconds if (endtime - starttime).seconds<60 and len(self.classifier.buffer)>10:''' def ret(self): self.fr=0 self.one=0 self.classifier.reset() def process_all(self): path="/home/lzz/sign/svm/" buf=self.classifier.buffer for decoded_data in buf: if decoded_data["label"]=="time": self.ret() return if(decoded_data["label"]=="End"): break read_index=[1,3,4,5,6,7,8,9] raw=decoded_data["skeleton"].split(",") posraw=decoded_data["position"].split(",") if raw=='untracked' or posraw=='': continue skeleton=[] position=[] if raw!='': raw=[0]+raw for index in read_index: skeleton.append(float(raw[index*7+1])) skeleton.append(float(raw[index*7+2])) skeleton.append(float(raw[index*7+3])) if self.one==0: #print self.classifier.dic self.classifier.dic[path].headpos=[int(raw[4]),int(raw[5])] self.classifier.dic[path].shoulder=abs(int(raw[11])-int(raw[25])) self.classifier.dic[path].hip=int(raw[89]) self.classifier.dic[path].tall=int(raw[89])-int(raw[5]) self.one=1 if len(position)>5: for i in range(4,6): position.append(int(posraw[i])) for i in range(4,6): position.append(int(posraw[i])) else: for i in range(4): position.append(int(posraw[i])) self.classifier.dic[path].positions.append(position) self.classifier.dic[path].skeletons.append(skeleton) else: if len(position)>5: for i in range(4,6): position.append(int(posraw[i])) for i in range(4,6): position.append(int(posraw[i])) else: for i in range(4): position.append(int(posraw[i])) self.classifier.dic[path].positions.append(position) rightimg=None leftimg=None ftype=decoded_data['label'] if decoded_data['right']!=None: img=decoded_data['right'] rightimg=img cv2.imwrite("/home/lzz/sing/svm/handshaperight/"+str(self.fr)+str(ftype)+".bmp",rightimg) if decoded_data['left']!=None: img=decoded_data['left'] leftimg=img cv2.imwrite("/home/lzz/sign/svm/handshape/"+str(self.fr)+"l.bmp",leftimg) self.classifier.dic[path].dict[self.fr]=frame(self.fr,skeleton,ftype,rightimg,leftimg,position,0) self.classifier.dic[path].framelist.append(self.fr) self.fr+=1 if self.classifier.dic[path].skeletons==[] or self.classifier.dic[path].positions==[]: self.fr=0 self.one=0 self.classifier.reset() return #self.classifier.dic[path].heightenough() self.classifier.dic[path].getBothSeparate() self.classifier.dic[path].getInter() self.classifier.dic[path].getVelo() self.classifier.dic[path].getVelocity() self.classifier.dic[path].findTopHandshape() self.classifier.getHogFeature() try: ret=self.classifier.dic[path].consTrajectory() except: print "Exception in user code:" print '-'*60 traceback.print_exc(file=sys.stdout) print '-'*60 ret=0 if ret==0: self.ret() return #self.slword.displacement=hodmodule.hod(self.slword.skeletons) #self.slword.checkDecisionTreeInter() #self.slword.getHogFeature() '''import glob imgset=glob.glob("/home/lzz/1/*.bmp") for i in imgset: img=cv2.imread(i) img=img/255.0 self.slword.batch.append(img)''' '''print "batch" print len(self.slword.batch) if len(self.slword.batch)==0: self.fr=0 del self.slword self.slword=slWord.SlWord() return self.caffedl.net.predict(self.slword.batch,False) feature=[] for s in range(self.slword.keyNo): feat = self.caffedl.net.blobs['ip1'].data[s].flatten().tolist() feature.append(feat) #print feature self.slword.handshape=self.slword.pooling(feature,1)''' self.classifier.getCaffeFeature(self.caffedl.net,self.caffedlInter.net) self.classifier.separateCaffeFeature() '''if self.slword.bothseparate==1: leftfeature=[] for s in range(self.slword.leftkeyNo): feat = self.caffedl.net.blobs['ip1'].data[self.slword.keyNo+s].flatten().tolist() leftfeature.append(feat) self.slword.lefthandshape=self.slword.pooling(leftfeature,1)''' self.classifier.dic[path].combineFeature() if self.classifier.dic[path].intersect==0 and self.classifier.dic[path].bothseparate==0: print "class 1" svmModel= svm_load_model("/home/lzz/sign/svm/single.model") elif self.classifier.dic[path].intersect==0 and self.classifier.dic[path].bothseparate==1: print "class 2" svmModel= svm_load_model("/home/lzz/sign/svm/both.model") elif self.classifier.dic[path].intersect==1 and self.classifier.dic[path].bothseparate==0: print "class 3" svmModel= svm_load_model("/home/lzz/sign/svm/inter.model") svm_res1=test_svm_model(svmModel,[0],[self.classifier.dic[path].combinedFeature]) #print "result",svm_res1 pred_labels=svm_res1[0] print self.signDic[int(pred_labels[0])] self.server.handler.send(self.signDic[int(pred_labels[0])]+'#TERMINATOR#') self.ret() def flatten(self, d, parent_key='', sep='.'): items = [] for k, v in d.items(): new_key = parent_key + sep + k if parent_key else k if isinstance(v, collections.MutableMapping): items.extend(self.flatten(v, new_key, sep=sep).items()) else: items.append((new_key, v)) return dict(items) def play(self,pred_labels): modeldic={} modelindex=open("/home/lzz/ModelIndex.txt","r") for line in open("/home/lzz/ModelIndex.txt"): line = modelindex.readline() sep=line.split(" ") modeldic[sep[0]]=sep[1] wordname=modeldic[str(int(pred_labels))] dirs=os.listdir("/media/lzz/Data1/Aaron/1-250/") for dir in dirs: if(dir.find(wordname)): video=dir break filename = "/media/lzz/Data1/Aaron/1-250/"+video+"/"+video+"_c.avi" #print filename win_name=filename capture = cv2.cv.CaptureFromFile(filename) cv2.cv.NamedWindow(win_name, cv2.cv.CV_WINDOW_AUTOSIZE) while 1: image = cv2.cv.QueryFrame(capture) cv2.cv.ShowImage(win_name, image) c = cv2.cv.WaitKey(33) if(image==None): break cv2.cv.DestroyWindow(win_name)
class Edu_Server(object): def __init__(self, port): f=open('/media/lzz/65c50da0-a3a2-4117-8a72-7b37fd81b574/sign/project/src/EducationNew.pickle') self.pickle=pickle.load(f) f.close() self.fr=0 self.one=0 self.exist=0 self.lock = threading.Lock() self.condition=threading.Condition() self.cond=threading.Condition() self.classifier = EduClassifier() self.p_thres=0.4 self.h_thres=0.5 self.h_thres_test=0.3 self.converter = FrameConverter() self.caffedl=caffeDL('../../proto/lenet_test_edu.prototxt','../../model/lenet_iter_5000.caffemodel') self.caffedlInter=caffeDL('../../proto_inter/lenet_test_edu.prototxt','../../model/lenet__iter_400.caffemodel') self.signDic={} self.queue = Queue.Queue(maxsize = 10000) self.server = EchoServer(port, self.received_data) self.mode='' thread.start_new_thread(self.threadfunc,()) asyncore.loop() def received_data(self, received_data): decoded_data = self.converter.decode(received_data) return self.process_data(decoded_data) def process_data(self, decoded_data): #print decoded_data if decoded_data["label"]=='guide': word_name=decoded_data['wordname'] self.mode='guide' self.standard=self.pickle[word_name] self.stepno=0 if self.lock.acquire(): self.queue.queue.clear() self.lock.release() self.fr=0 self.one=0 self.cond.acquire() self.cond.notify() self.cond.release() self.classifier.reset() return '0' elif decoded_data['label']=='evaluation': print 'evaluation' word_name=decoded_data['wordname'] self.mode='evaluation' self.standard=self.pickle[word_name] self.queue.queue.clear() self.buffer=[] self.fr=0 self.one=0 self.cond.acquire() self.cond.notify() self.cond.release() print word_name self.classifier.reset() return '0' elif self.mode=='guide': self.queue.put(decoded_data) self.condition.acquire() self.condition.notify() self.condition.release() #self.process_oneframe(decoded_data) return '0' elif self.mode=='evaluation': if(self.queue.full()==0): if self.exist==0: if decoded_data["label"]=='End': if self.lock.acquire(): self.exist=1#exist means there is a End in the queue, that is, a complete sign exists self.lock.release() else: if self.lock.acquire(): self.queue.queue.clear() self.classifier.buffer=[] self.exist=0 self.lock.release() self.queue.put(decoded_data) self.condition.acquire() self.condition.notify() self.condition.release() else: self.server.handler.send("Redo"+'#TERMINATOR#') if self.lock.acquire(): self.queue.queue.clear() self.classifier.buffer=[] self.exist=0 self.lock.release() return '0' def threadfunc(self): while(1): if(self.mode==''): self.cond.acquire() self.cond.wait() self.cond.release() if self.mode=='guide': self.condition.acquire() if(self.queue.empty()==1): self.condition.wait() self.condition.release() while self.queue._qsize()>1: self.queue.get() self.process_oneframe(self.queue.get()) if self.mode=='evaluation': self.condition.acquire() if(self.queue.empty()==1): self.condition.wait() self.condition.release() decoded_data=self.queue.get() if self.lock.acquire(): self.classifier.buffer.append(decoded_data) self.lock.release() if(decoded_data["label"]=="End"): if self.lock.acquire(): self.exist=0 self.lock.release() if len(self.classifier.buffer)<10: self.server.handler.send("Redo"+"#TERMINATOR#") else: self.process_all() if self.lock.acquire(): self.classifier.buffer=[] self.lock.release() def process_oneframe(self,decoded_data): path="/home/lzz/sign/svm/" read_index=[1,3,4,5,6,7,8,9] raw=decoded_data["skeleton"].split(",") posraw=decoded_data["position"].split(",") if raw=='untracked' or posraw=='': return skeleton=[] if raw!='': raw=[0]+raw for index in read_index: skeleton.append(float(raw[index*7+1])) skeleton.append(float(raw[index*7+2])) skeleton.append(float(raw[index*7+3])) if self.one==0: self.classifier.dic[path].headpos=[int(raw[4]),int(raw[5])] self.classifier.dic[path].shoulder=abs(int(raw[11])-int(raw[25])) self.classifier.dic[path].hip=int(raw[89]) self.classifier.dic[path].tall=int(raw[89])-int(raw[5]) self.one=1 if self.stepno>=len(self.standard): #print self.stepno,len(self.standard) return '0' step=self.standard[self.stepno] pos0=step['pos'] pos1=[(float(posraw[0])-self.classifier.dic[path].headpos[0])/self.classifier.dic[path].shoulder, (float(posraw[1])-self.classifier.dic[path].headpos[1])/self.classifier.dic[path].tall, (float(posraw[2])-self.classifier.dic[path].headpos[0])/self.classifier.dic[path].shoulder, (float(posraw[3])-self.classifier.dic[path].headpos[1])/self.classifier.dic[path].tall, ] tickle=0 if decoded_data['label']==step['type']: if step['type']=='Right': #print pos0,pos1 print 'pos',np.sqrt((pos1[0]-pos0[0])**2+(pos1[1]-pos0[1])**2) if np.sqrt((pos1[0]-pos0[0])**2+(pos1[1]-pos0[1])**2)<0.4: start=time.time() h0=step['handshape'] img=decoded_data['right'] if img!=None: sp=img.shape if sp[0]>sp[1]: img2=cv2.copyMakeBorder(img, 0,0, int(abs(sp[0]-sp[1])/2),int(abs(sp[0]-sp[1])/2), cv2.BORDER_CONSTANT, value=(0, 0, 0, 0)) else: img2=cv2.copyMakeBorder(img, int(abs(sp[0]-sp[1])/2),int(abs(sp[0]-sp[1])/2),0,0, cv2.BORDER_CONSTANT, value=(0, 0, 0, 0)) img3=cv2.resize(img2,(128,128)) img3=img3/255.0 self.caffedl.net.predict([img3],False) h1 = self.caffedl.net.blobs['ip1'].data[0].flatten().tolist() print 'hand',1 - spatial.distance.cosine(h0,h1) if 1 - spatial.distance.cosine(h0,h1)>self.h_thres: self.stepno+=1 tickle=1 print 'time:',time.time()-start elif step['type']=='Both': if np.sqrt((pos1[0]-pos0[0])**2+(pos1[1]-pos0[1])**2)<self.p_thres and np.sqrt((pos1[2]-pos0[2])**2+(pos1[3]-pos0[3])**2)<self.p_thres: h0=step['handshape'] lefth0=step['lefthandshape'] img=decoded_data['right'] leftimg=decoded_data['left'] if img!=None and leftimg!=None: sp=img.shape if sp[0]>sp[1]: img2=cv2.copyMakeBorder(img, 0,0, int(abs(sp[0]-sp[1])/2),int(abs(sp[0]-sp[1])/2), cv2.BORDER_CONSTANT, value=(0, 0, 0, 0)) else: img2=cv2.copyMakeBorder(img, int(abs(sp[0]-sp[1])/2),int(abs(sp[0]-sp[1])/2),0,0, cv2.BORDER_CONSTANT, value=(0, 0, 0, 0)) img3=cv2.resize(img2,(128,128)) imgr=img3/255.0 sp=leftimg.shape if sp[0]>sp[1]: img2=cv2.copyMakeBorder(leftimg, 0,0, int(abs(sp[0]-sp[1])/2),int(abs(sp[0]-sp[1])/2), cv2.BORDER_CONSTANT, value=(0, 0, 0, 0)) else: img2=cv2.copyMakeBorder(leftimg, int(abs(sp[0]-sp[1])/2),int(abs(sp[0]-sp[1])/2),0,0, cv2.BORDER_CONSTANT, value=(0, 0, 0, 0)) img3=cv2.resize(img2,(128,128)) imgl=img3/255.0 self.caffedl.net.predict([imgr,imgl],False) h1r = self.caffedl.net.blobs['ip1'].data[0].flatten().tolist() h1l = self.caffedl.net.blobs['ip1'].data[1].flatten().tolist() if 1 - spatial.distance.cosine(h0,h1r)>self.h_thres and 1 - spatial.distance.cosine(lefth0,h1l)>self.h_thres: self.stepno+=1 tickle=1 elif step['type']=='Intersect': if np.sqrt((pos1[0]-pos0[0])**2+(pos1[1]-pos0[1])**2)<self.p_thres: h0=step['handshape'] img=decoded_data['right'] if img!=None: sp=img.shape if sp[0]>sp[1]: img2=cv2.copyMakeBorder(img, 0,0, int(abs(sp[0]-sp[1])/2),int(abs(sp[0]-sp[1])/2), cv2.BORDER_CONSTANT, value=(0, 0, 0, 0)) else: img2=cv2.copyMakeBorder(img, int(abs(sp[0]-sp[1])/2),int(abs(sp[0]-sp[1])/2),0,0, cv2.BORDER_CONSTANT, value=(0, 0, 0, 0)) img3=cv2.resize(img2,(128,128)) img3=img3/255.0 self.caffedlInter.net.predict([img3],False) h1 = self.caffedlInter.net.blobs['ip1'].data[0].flatten().tolist() if 1 - spatial.distance.cosine(h0,h1)>self.h_thres: self.stepno+=1 tickle=1 if tickle==1: if self.stepno>=len(self.standard)-1: self.server.handler.send("Finish"+'#TERMINATOR#') else: self.server.handler.send("Next"+'#TERMINATOR#') def ret(self): self.fr=0 self.one=0 self.classifier.reset() def process_all(self): print 'process all' buf=self.classifier.buffer buffer=buf current=0 ticklelist={} cnt=0 for s in range(len(self.standard)): ticklelist[s]={} validno=[] belong=[] for i in range(len(buf)): belong.append(-1) for s in range(len(self.standard)): ticklelist[s]['position']=0 ticklelist[s]['handshape']=0 if self.standard[s]['type']=='Both': ticklelist[s]['lefthandshape']=0 for i in range(current,len(buf)-1): path="/home/lzz/sign/svm/" read_index=[1,3,4,5,6,7,8,9] raw=buf[i]["skeleton"].split(",") posraw=buf[i]["position"].split(",") if raw=='untracked' or posraw=='': return skeleton=[] if raw!='': raw=[0]+raw for index in read_index: skeleton.append(float(raw[index*7+1])) skeleton.append(float(raw[index*7+2])) skeleton.append(float(raw[index*7+3])) if self.one==0: self.classifier.dic[path].headpos=[int(raw[4]),int(raw[5])] self.classifier.dic[path].shoulder=abs(int(raw[11])-int(raw[25])) self.classifier.dic[path].hip=int(raw[89]) self.classifier.dic[path].tall=int(raw[89])-int(raw[5]) self.one=1 if s>=len(self.standard): print s,len(self.standard) return '0' pos1=[(float(posraw[0])-self.classifier.dic[path].headpos[0])/self.classifier.dic[path].shoulder, (float(posraw[1])-self.classifier.dic[path].headpos[1])/self.classifier.dic[path].tall, (float(posraw[2])-self.classifier.dic[path].headpos[0])/self.classifier.dic[path].shoulder, (float(posraw[3])-self.classifier.dic[path].headpos[1])/self.classifier.dic[path].tall, ] decoded_data=buf[i] step=self.standard[s] pos0=step['pos'] tickle=0 if decoded_data['label']==step['type']: if step['type']=='Right': print 'pos',np.sqrt((pos1[0]-pos0[0])**2+(pos1[1]-pos0[1])**2) if np.sqrt((pos1[0]-pos0[0])**2+(pos1[1]-pos0[1])**2)<self.p_thres: cnt+=1 ticklelist[s]['position']=1 validno.append(i) belong[i]=s tickle=1 elif step['type']=='Both': if np.sqrt((pos1[0]-pos0[0])**2+(pos1[1]-pos0[1])**2)<self.p_thres and np.sqrt((pos1[2]-pos0[2])**2+(pos1[3]-pos0[3])**2)<self.p_thres: ticklelist[s]['position']=1 validno.append(i) belong[i]=s tickle=1 elif step['type']=='Intersect': if np.sqrt((pos1[0]-pos0[0])**2+(pos1[1]-pos0[1])**2)<self.p_thres: ticklelist[s]['position']=1 validno.append(i) belong[i]=s tickle=1 if tickle==1: current=i if i==len(buf)-1: break for s in range(len(self.standard)): if ticklelist[s]['position']==0: self.server.handler.send(str(ticklelist)+'#TERMINATOR#') return totallen=cnt jump=max(1,int(totallen/30)) cnt=0 havetickled=[] print belong for u in range(len(validno)): i=validno[u] cnt+=1 step=self.standard[belong[u]] if belong[u] in havetickled: continue if cnt%jump!=0: continue tickle=0 decoded_data=buffer[i] if step['type']=='Right': img=buffer[i]['right'] h0=step['handshape'] if img!=None: sp=img.shape if sp[0]>sp[1]: img2=cv2.copyMakeBorder(img, 0,0, int(abs(sp[0]-sp[1])/2),int(abs(sp[0]-sp[1])/2), cv2.BORDER_CONSTANT, value=(0, 0, 0, 0)) else: img2=cv2.copyMakeBorder(img, int(abs(sp[0]-sp[1])/2),int(abs(sp[0]-sp[1])/2),0,0, cv2.BORDER_CONSTANT, value=(0, 0, 0, 0)) img3=cv2.resize(img2,(128,128)) img3=img3/255.0 self.caffedl.net.predict([img3],False) h1 = self.caffedl.net.blobs['ip1'].data[0].flatten().tolist() print 'hand', 1 - spatial.distance.cosine(h0,h1),belong[i] if 1 - spatial.distance.cosine(h0,h1)>self.h_thres_test: tickle=1 print 'tickle',belong[i] ticklelist[belong[i]]['handshape']=1 elif step['type']=='Both': righttickle=0 lefttickle=0 img=buffer[i]['right'] h0=step['handshape'] if img!=None: sp=img.shape if sp[0]>sp[1]: img2=cv2.copyMakeBorder(img, 0,0, int(abs(sp[0]-sp[1])/2),int(abs(sp[0]-sp[1])/2), cv2.BORDER_CONSTANT, value=(0, 0, 0, 0)) else: img2=cv2.copyMakeBorder(img, int(abs(sp[0]-sp[1])/2),int(abs(sp[0]-sp[1])/2),0,0, cv2.BORDER_CONSTANT, value=(0, 0, 0, 0)) img3=cv2.resize(img2,(128,128)) img3=img3/255.0 self.caffedl.net.predict([img3],False) h1 = self.caffedl.net.blobs['ip1'].data[0].flatten().tolist() if 1 - spatial.distance.cosine(h0,h1)>self.h_thres_test: righttickle=1 ticklelist[belong[i]]['handshape']=1 img=buffer[i]['left'] h0=step['lefthandshape'] if img!=None: sp=img.shape if sp[0]>sp[1]: img2=cv2.copyMakeBorder(img, 0,0, int(abs(sp[0]-sp[1])/2),int(abs(sp[0]-sp[1])/2), cv2.BORDER_CONSTANT, value=(0, 0, 0, 0)) else: img2=cv2.copyMakeBorder(img, int(abs(sp[0]-sp[1])/2),int(abs(sp[0]-sp[1])/2),0,0, cv2.BORDER_CONSTANT, value=(0, 0, 0, 0)) img3=cv2.resize(img2,(128,128)) img3=img3/255.0 self.caffedl.net.predict([img3],False) h1 = self.caffedl.net.blobs['ip1'].data[0].flatten().tolist() if 1 - spatial.distance.cosine(h0,h1)>self.h_thres_test: lefttickle=1 ticklelist[belong[i]]['lefthandshape']=1 if righttickle==1 and lefttickle==1: tickle=1 elif step['type']=='Intersect': img=buffer[i]['right'] h0=step['handshape'] if img!=None: sp=img.shape if sp[0]>sp[1]: img2=cv2.copyMakeBorder(img, 0,0, int(abs(sp[0]-sp[1])/2),int(abs(sp[0]-sp[1])/2), cv2.BORDER_CONSTANT, value=(0, 0, 0, 0)) else: img2=cv2.copyMakeBorder(img, int(abs(sp[0]-sp[1])/2),int(abs(sp[0]-sp[1])/2),0,0, cv2.BORDER_CONSTANT, value=(0, 0, 0, 0)) img3=cv2.resize(img2,(128,128)) img3=img3/255.0 self.caffedl.net.predict([img3],False) h1 = self.caffedl.net.blobs['ip1'].data[0].flatten().tolist() if 1 - spatial.distance.cosine(h0,h1)>self.h_thres_test: ticklelist[belong[i]]['handshape']=1 tickle=1 if tickle==1: havetickled.append(belong[i]) if i==len(buf)-1: break self.server.handler.send(str(ticklelist)+'#TERMINATOR#') def flatten(self, d, parent_key='', sep='.'): items = [] for k, v in d.items(): new_key = parent_key + sep + k if parent_key else k if isinstance(v, collections.MutableMapping): items.extend(self.flatten(v, new_key, sep=sep).items()) else: items.append((new_key, v)) return dict(items)
class HandShapeServer(object): def __init__(self, port): self.converter = FrameConverter() self.slword = slWord.SlWord() self.caffedl = caffeDL() self.server = EchoServer(port, self.received_data) def received_data(self, received_data): print(len(received_data), binascii.crc32(received_data)) #decompressed_data = bz2.decompress(received_data) decoded_data = self.converter.decode(received_data) #import pprint #pprint.pprint(decoded_data) return self.process_data(decoded_data) def process_data(self, decoded_data): sleep_time = random.randint(1, 10) if (decoded_data["label"] == "End"): self.process_all() return "0" self.slword.loadSkeleton(decoded_data["skeleton"]) if (decoded_data['depth_image'] != ''): print "skl" img = decoded_data['depth_image'] sp = img.shape print sp img2 = cv2.copyMakeBorder(img, 0, 0, int(abs(sp[0] - sp[1]) / 2), int(abs(sp[0] - sp[1]) / 2), cv2.BORDER_CONSTANT, value=(0, 0, 0, 0)) img3 = cv2.resize(img2, (128, 128)) img3 = img3 / 255.0 self.slword.imgset.append(img3) return "0" else: self.slword.imgset.append([]) return "0" def play(self, pred_labels): modeldic = {} modelindex = open("/home/lzz/ModelIndex.txt", "r") for line in open("/home/lzz/ModelIndex.txt"): line = modelindex.readline() sep = line.split(" ") modeldic[sep[0]] = sep[1] wordname = modeldic[str(int(pred_labels))] dirs = os.listdir("/media/lzz/Data1/Aaron/1-250/") for dir in dirs: if (dir.find(wordname)): video = dir break filename = "/media/lzz/Data1/Aaron/1-250/" + video + "/" + video + "_c.avi" print filename win_name = filename capture = cv2.cv.CaptureFromFile(filename) cv2.cv.NamedWindow(win_name, cv2.cv.CV_WINDOW_AUTOSIZE) while 1: image = cv2.cv.QueryFrame(capture) cv2.cv.ShowImage(win_name, image) c = cv2.cv.WaitKey(33) if (image == None): break cv2.cv.DestroyWindow(win_name) def process_all(self): self.slword.displacement, rightV, rightH = hodmodule.hod( self.slword.skeletons) print self.slword.displacement, rightV print len(self.slword.displacement), len(rightV) self.slword.getVelocity(rightV, rightH) self.slword.findTopHandshape() self.caffedl.net.predict(self.slword.batch) feature = [] for s in range(len(self.slword.batch)): feat = self.caffedl.net.blobs['ip1'].data[s].flatten().tolist() feature.append(feat) self.slword.handshape = self.slword.pooling(feature, 1) self.slword.combineFeature() svmModel = svm_load_model("/home/lzz/svmModel") print self.slword.combinedFeature print len(self.slword.combinedFeature) svm_res1 = test_svm_model(0, self.slword.combinedFeature, svmModel) pred_labels = svm_res1[0] print "pred_labels", pred_labels self.play(pred_labels) def flatten(self, d, parent_key='', sep='.'): items = [] for k, v in d.items(): new_key = parent_key + sep + k if parent_key else k if isinstance(v, collections.MutableMapping): items.extend(self.flatten(v, new_key, sep=sep).items()) else: items.append((new_key, v)) return dict(items)
def __init__(self, port): self.converter = FrameConverter() self.slword = slWord.SlWord() self.caffedl = caffeDL() self.server = EchoServer(port, self.received_data)
depthFrameData.load(srcVideoPath) srcVideoPath = join(videoDirectory,firstFile+colorVideoSuffix+videoFilenameExtension) colorFrameData = VideoFrameData() colorFrameData.load(srcVideoPath) i = 0 resultImages = [] depthRetval,depthFrame = depthFrameData.readFrame() colorRetval,colorFrame = colorFrameData.readFrame() labelRetval,labelFrame = labelFrameData.readFrame() skeletonRetval,skeletonFrame = skeletonFrameData.readFrame() if not depthRetval or not colorRetval or not labelRetval or not skeletonRetval: exit encodedFrame = FrameConverter().encode(depthFrame, colorFrame, labelFrame, skeletonFrame) plt.subplot(1,2,1), plt.imshow(depthFrame) plt.subplot(1,2,2), plt.imshow(colorFrame) plt.show() argc = len(sys.argv) if argc > 1: host = sys.argv[1] else: host = 'localhost' client = HandShapeClient(host, port) client.send_data(encodedFrame)
def readVideo(fileName, frameCallback, result): frameConverter = FrameConverter() labelFrameData = CSVFrameData() labelFrameData.load(join(labelDirectory, fileName)) labelFrameData.setDebug(DEBUG_MODE) fileName = fileName[:-4] skeletonFrameData = CSVFrameData() skeletonFrameData.load(join(skeletonDirectory, fileName+skeletonVideoSuffix+skeletonFilenameExtension)) skeletonFrameData.setDebug(DEBUG_MODE) srcVideoPath = join(videoDirectory,fileName+depthVideoSuffix+videoFilenameExtension) depthFrameData = VideoFrameData() depthFrameData.load(srcVideoPath) i = 0 resultImages = [] depthRetval,depthFrame = depthFrameData.readFrame() labelRetval,labelFrame = labelFrameData.readFrame() skeletonRetval,skeletonFrame = skeletonFrameData.readFrame() if not depthRetval or not labelRetval or not skeletonRetval: return h,w = depthFrame.shape[0:2] encodedFrame = frameConverter.encode(depthFrame, labelFrame, skeletonFrame) if SAVE_RESULT_VIDEO: videoPath = join(resultDirectory, 'croppedHand-'+fileName+videoFilenameExtension) videoWriter = cv2.VideoWriter() fourcc = cv2.cv.CV_FOURCC('m', 'p', '4', 'v') videoWriter.open(videoPath, fourcc, 30, (w*2,h)) else: videoWriter = None while(labelRetval and depthRetval and skeletonRetval): encodedFrame = frameConverter.encode(depthFrame, labelFrame, skeletonFrame) res,resultImage = frameCallback(depthFrame, labelFrame, videoWriter) resultImages.append(resultImage) if VISUALIZE_RESULT: if cv2.waitKey(1) & 0xFF == ord('q'): depthRetval = False continue depthRetval,depthFrame = depthFrameData.readFrame() labelRetval,labelFrame = labelFrameData.readFrame() skeletonRetval,skeletonFrame = skeletonFrameData.readFrame() depthFrameData.close() labelFrameData.close() skeletonFrameData.close() if SAVE_RESULT_VIDEO & (videoWriter is not None): message = "Saving Video..." savingImage = np.zeros((h,w*2,3), np.uint8) savingImageTextSize, _ = cv2.getTextSize(message, cv2.FONT_HERSHEY_SIMPLEX, 1, 2) savingImageTextLocation = (w-savingImageTextSize[0]/2, h/2-savingImageTextSize[1]/2) for i in xrange(0,len(resultImages)): savingImage[:] = 0 cv2.rectangle(savingImage, (0,savingImageTextLocation[1]), (i*w*2/len(resultImages),savingImageTextLocation[1]+savingImageTextSize[1]), (0,255,0), cv2.cv.CV_FILLED) cv2.putText(savingImage, message, savingImageTextLocation, cv2.FONT_HERSHEY_SIMPLEX, 1, (255,255,255), 2, cv2.CV_AA) cv2.imshow('Depth Video', savingImage) videoWriter.write(resultImages[i])
def __init__(self, host, port): self.client = EchoClient(host, port, self) self.converter = FrameConverter()
def __init__(self, port): self.converter = FrameConverter() self.slword=slWord.SlWord() self.caffedl=caffeDL() self.server = EchoServer(port, self.received_data)
class HandShapeServer(object): def __init__(self, port): self.converter = FrameConverter() self.slword=slWord.SlWord() self.caffedl=caffeDL() self.server = EchoServer(port, self.received_data) def received_data(self, received_data): print (len(received_data), binascii.crc32(received_data)) #decompressed_data = bz2.decompress(received_data) decoded_data = self.converter.decode(received_data) #import pprint #pprint.pprint(decoded_data) return self.process_data(decoded_data) def process_data(self, decoded_data): sleep_time = random.randint(1, 10) if(decoded_data["label"]=="End"): self.process_all() return "0" self.slword.loadSkeleton(decoded_data["skeleton"]) if(decoded_data['depth_image']!=''): print "skl" img=decoded_data['depth_image'] sp=img.shape print sp img2=cv2.copyMakeBorder(img, 0,0, int(abs(sp[0]-sp[1])/2),int(abs(sp[0]-sp[1])/2), cv2.BORDER_CONSTANT, value=(0, 0, 0, 0)) img3=cv2.resize(img2,(128,128)) img3=img3/255.0 self.slword.imgset.append(img3) return "0" else: self.slword.imgset.append([]) return "0" def play(self,pred_labels): modeldic={} modelindex=open("/home/lzz/ModelIndex.txt","r") for line in open("/home/lzz/ModelIndex.txt"): line = modelindex.readline() sep=line.split(" ") modeldic[sep[0]]=sep[1] wordname=modeldic[str(int(pred_labels))] dirs=os.listdir("/media/lzz/Data1/Aaron/1-250/") for dir in dirs: if(dir.find(wordname)): video=dir break filename = "/media/lzz/Data1/Aaron/1-250/"+video+"/"+video+"_c.avi" print filename win_name=filename capture = cv2.cv.CaptureFromFile(filename) cv2.cv.NamedWindow(win_name, cv2.cv.CV_WINDOW_AUTOSIZE) while 1: image = cv2.cv.QueryFrame(capture) cv2.cv.ShowImage(win_name, image) c = cv2.cv.WaitKey(33) if(image==None): break cv2.cv.DestroyWindow(win_name) def process_all(self): self.slword.displacement,rightV,rightH=hodmodule.hod(self.slword.skeletons) print self.slword.displacement,rightV print len(self.slword.displacement),len(rightV) self.slword.getVelocity(rightV,rightH) self.slword.findTopHandshape() self.caffedl.net.predict(self.slword.batch) feature=[] for s in range(len(self.slword.batch)): feat = self.caffedl.net.blobs['ip1'].data[s].flatten().tolist() feature.append(feat) self.slword.handshape=self.slword.pooling(feature,1) self.slword.combineFeature() svmModel= svm_load_model("/home/lzz/svmModel") print self.slword.combinedFeature print len(self.slword.combinedFeature) svm_res1=test_svm_model(0,self.slword.combinedFeature,svmModel) pred_labels=svm_res1[0]; print "pred_labels",pred_labels self.play(pred_labels) def flatten(self, d, parent_key='', sep='.'): items = [] for k, v in d.items(): new_key = parent_key + sep + k if parent_key else k if isinstance(v, collections.MutableMapping): items.extend(self.flatten(v, new_key, sep=sep).items()) else: items.append((new_key, v)) return dict(items)